def __init__(self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), output_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), bias: bool = True, activation: str = 'tanh', param_init: param_initializers.ParamInitializer = Ref("exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref("exp_global.bias_init", default=bare(param_initializers.ZeroInitializer))) -> None: self.bias = bias self.output_dim = output_dim self.input_dim = input_dim if activation == 'tanh': self.activation = dy.tanh elif activation == 'relu': self.activation = dy.rectify elif activation == 'sigmoid': self.activation = dy.sigmoid elif activation == 'elu': self.activation = dy.elu elif activation == 'selu': self.activation = dy.selu elif activation == 'asinh': self.activation = dy.asinh elif activation == 'identity': def identity(x): return x self.activation = identity else: raise ValueError('Unknown activation %s' % activation) model = param_collections.ParamManager.my_params(self) self.W1 = model.add_parameters((self.output_dim, self.input_dim), init=param_init.initializer((self.output_dim, self.input_dim))) if self.bias: self.b1 = model.add_parameters((self.output_dim,), init=bias_init.initializer((self.output_dim,)))
def __init__( self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), state_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), hidden_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref( "exp_global.bias_init", default=bare(param_initializers.ZeroInitializer)), truncate_dec_batches: bool = Ref("exp_global.truncate_dec_batches", default=False) ) -> None: self.input_dim = input_dim self.state_dim = state_dim self.hidden_dim = hidden_dim self.truncate_dec_batches = truncate_dec_batches param_collection = param_collections.ParamManager.my_params(self) self.pW = param_collection.add_parameters((hidden_dim, input_dim), init=param_init.initializer( (hidden_dim, input_dim))) self.pV = param_collection.add_parameters((hidden_dim, state_dim), init=param_init.initializer( (hidden_dim, state_dim))) self.pb = param_collection.add_parameters((hidden_dim, ), init=bias_init.initializer( (hidden_dim, ))) self.pU = param_collection.add_parameters((1, hidden_dim), init=param_init.initializer( (1, hidden_dim))) self.curr_sent = None
def __init__(self, input_dim: int, chn_dim: int = 3, num_filters: int = 32, param_init: param_initializers.ParamInitializer = Ref("exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref("exp_global.bias_init", default=bare(param_initializers.ZeroInitializer))): model = param_collections.ParamManager.my_params(self) if input_dim % chn_dim != 0: raise RuntimeError("input_dim must be divisible by chn_dim") self.input_dim = input_dim self.chn_dim = chn_dim self.freq_dim = input_dim / chn_dim self.num_filters = num_filters self.filter_size_time = 1 self.filter_size_freq = 3 self.params = {} for direction in ["fwd", "bwd"]: dim_x = (self.filter_size_time, self.filter_size_freq, self.chn_dim, self.num_filters * 4) self.params["x2all_" + direction] = \ model.add_parameters(dim=dim_x, init=param_init.initializer(dim_x, num_shared=4)) dim_h = (self.filter_size_time, self.filter_size_freq, self.num_filters, self.num_filters * 4) self.params["h2all_" + direction] = \ model.add_parameters(dim=dim_h, init=param_init.initializer(dim_h, num_shared=4)) dim_b = (self.num_filters * 4,) self.params["b_" + direction] = \ model.add_parameters(dim=dim_b, init=bias_init.initializer(dim_b, num_shared=4))
def __init__(self, emb_dim=Ref("exp_global.default_layer_dim"), weight_noise=Ref("exp_global.weight_noise", default=0.0), word_dropout=0.0, fix_norm=None, param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref( "exp_global.bias_init", default=bare(param_initializers.ZeroInitializer)), vocab_size=None, vocab=None, yaml_path=None, src_reader=Ref("model.src_reader", default=None), trg_reader=Ref("model.trg_reader", default=None)): self.fix_norm = fix_norm self.weight_noise = weight_noise self.word_dropout = word_dropout self.emb_dim = emb_dim param_collection = param_collections.ParamManager.my_params(self) self.vocab_size = self.choose_vocab_size(vocab_size, vocab, yaml_path, src_reader, trg_reader) self.save_processed_arg("vocab_size", self.vocab_size) self.embeddings = param_collection.add_parameters( (self.vocab_size, self.emb_dim), init=param_init.initializer((self.vocab_size, self.emb_dim), is_lookup=True)) self.bias = param_collection.add_parameters((self.vocab_size, ), init=bias_init.initializer( (self.vocab_size, )))
def __init__( self, layers: numbers.Integral, input_dim: numbers.Integral, hidden_dim: numbers.Integral, param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref( "exp_global.bias_init", default=bare(param_initializers.ZeroInitializer)) ) -> None: if layers != 1: raise RuntimeError( "CustomLSTMSeqTransducer supports only exactly one layer") self.input_dim = input_dim self.hidden_dim = hidden_dim model = param_collections.ParamManager.my_params(self) # [i; f; o; g] self.p_Wx = model.add_parameters(dim=(hidden_dim * 4, input_dim), init=param_init.initializer( (hidden_dim * 4, input_dim))) self.p_Wh = model.add_parameters(dim=(hidden_dim * 4, hidden_dim), init=param_init.initializer( (hidden_dim * 4, hidden_dim))) self.p_b = model.add_parameters(dim=(hidden_dim * 4, ), init=bias_init.initializer( (hidden_dim * 4, )))
def __init__( self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), dropout: numbers.Real = Ref("exp_global.dropout", default=0.0), param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref( "exp_global.bias_init", default=bare(param_initializers.ZeroInitializer)), num_heads: numbers.Integral = 8): assert (input_dim % num_heads == 0) self.dropout = dropout param_collection = param_collections.ParamManager.my_params(self) self.input_dim = input_dim self.num_heads = num_heads self.head_dim = input_dim // num_heads self.pWq, self.pWk, self.pWv, self.pWo = [ param_collection.add_parameters(dim=(input_dim, input_dim), init=param_init.initializer( (input_dim, input_dim))) for _ in range(4) ] self.pbq, self.pbk, self.pbv, self.pbo = [ param_collection.add_parameters(dim=(1, input_dim), init=bias_init.initializer(( 1, input_dim, ))) for _ in range(4) ]
def __init__( self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), output_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), bias: bool = True, activation: str = 'tanh', param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref( "exp_global.bias_init", default=bare(param_initializers.ZeroInitializer)) ) -> None: self.bias = bias self.output_dim = output_dim self.input_dim = input_dim self.activation = tt.activation_by_name(activation) my_params = param_collections.ParamManager.my_params(self) self.W1 = my_params.add_parameters( (self.output_dim, self.input_dim), init=param_init.initializer((self.output_dim, self.input_dim))) if self.bias: self.b1 = my_params.add_parameters((self.output_dim, ), init=bias_init.initializer( (self.output_dim, )))
def __init__(self, emb_dim: int = Ref("exp_global.default_layer_dim"), vocab_size: Optional[int] = None, vocab: Optional[vocabs.Vocab] = None, yaml_path: Path = Path(''), src_reader: Optional[input_readers.InputReader] = Ref( "model.src_reader", default=None), trg_reader: Optional[input_readers.InputReader] = Ref( "model.trg_reader", default=None), is_dense: bool = False, param_init: pinit.ParamInitializer = Ref( "exp_global.param_init", default=bare(pinit.GlorotInitializer)), bias_init: pinit.ParamInitializer = Ref( "exp_global.bias_init", default=bare(pinit.ZeroInitializer)), init_fastext: Optional[str] = None, weight_noise: float = Ref("exp_global.weight_noise", default=0.0), fix_norm: Optional[float] = None): super().__init__(emb_dim=emb_dim, weight_noise=weight_noise, fix_norm=fix_norm) # Embedding Parameters pcol = param_collections.ParamManager.my_params(self) self.vocab_size = self.choose_vocab_size(vocab_size, vocab, yaml_path, src_reader, trg_reader) emb_mtr_dim = (self.vocab_size, self.emb_dim) if init_fastext is not None: logger.info("Setting Dense to False because of init_fastext") is_dense = False if not is_dense: if init_fastext is not None: self.embeddings = pcol.lookup_parameters_from_numpy( self._read_fasttext_embeddings(vocab, init_fastext)) else: self.embeddings = pcol.add_lookup_parameters( emb_mtr_dim, init=param_init.initializer(emb_mtr_dim, is_lookup=True)) else: self.embeddings = pcol.add_parameters(emb_mtr_dim, init=param_init.initializer( emb_mtr_dim, is_lookup=True)) self.bias = pcol.add_parameters((self.vocab_size, ), init=bias_init.initializer( (self.vocab_size, ))) # Model States self.is_dense = is_dense self.train = False self.save_processed_arg("vocab_size", self.vocab_size)
def __init__( self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), state_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), hidden_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), bias_init: param_initializers.ParamInitializer = Ref( "exp_global.bias_init", default=bare(param_initializers.ZeroInitializer)) ) -> None: self.input_dim = input_dim self.state_dim = state_dim self.hidden_dim = hidden_dim my_params = param_collections.ParamManager.my_params(self) self.linear_context = my_params.add_parameters( (hidden_dim, input_dim), init=param_init[0].initializer((hidden_dim, input_dim))) self.linear_query = my_params.add_parameters( (hidden_dim, state_dim), init=param_init[1].initializer((hidden_dim, state_dim))) self.bias_context = my_params.add_parameters( (hidden_dim, ), init=bias_init.initializer((hidden_dim, ))) self.pU = my_params.add_parameters((1, hidden_dim), init=param_init[2].initializer( (1, hidden_dim))) self.curr_sent = None self.attention_vecs = None self.WI = None
def __init__( self, emb_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), weight_noise: numbers.Real = Ref("exp_global.weight_noise", default=0.0), word_dropout: numbers.Real = 0.0, fix_norm: Optional[numbers.Real] = None, param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), vocab_size: Optional[numbers.Integral] = None, vocab: Optional[vocabs.Vocab] = None, yaml_path=None, src_reader: Optional[input_readers.InputReader] = Ref( "model.src_reader", default=None), trg_reader: Optional[input_readers.InputReader] = Ref( "model.trg_reader", default=None) ) -> None: #print(f"embedder received param_init: {param_init}") self.emb_dim = emb_dim self.weight_noise = weight_noise self.word_dropout = word_dropout self.fix_norm = fix_norm self.word_id_mask = None self.train = False param_collection = param_collections.ParamManager.my_params(self) self.vocab_size = self.choose_vocab_size(vocab_size, vocab, yaml_path, src_reader, trg_reader) self.save_processed_arg("vocab_size", self.vocab_size) self.embeddings = param_collection.add_lookup_parameters( (self.vocab_size, self.emb_dim), init=param_init.initializer((self.vocab_size, self.emb_dim), is_lookup=True))
def __init__(self, max_pos: numbers.Integral, op: str = 'sum', emb_type: str = 'param', input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), dropout: numbers.Real = Ref("exp_global.dropout", default=0.0), param_init: param_initializers.ParamInitializer = Ref("exp_global.param_init", default=bare(param_initializers.GlorotInitializer))) \ -> None: """ max_pos: largest embedded position op: how to combine positional encodings with the original encodings, can be "sum" or "concat" type: what type of embddings to use, "param"=parameterized (others, such as the trigonometric embeddings are todo) input_dim: embedding size dropout: apply dropout to output of this transducer param_init: how to initialize embedding matrix """ self.max_pos = max_pos self.input_dim = input_dim self.dropout = dropout self.op = op self.emb_type = emb_type param_init = param_init dim = (self.input_dim, max_pos) my_params = param_collections.ParamManager.my_params(self) self.embedder = my_params.add_parameters(dim, init=param_init.initializer( dim, is_lookup=True))
def __init__(self, char_vocab: Optional[vocabs.CharVocab] = Ref( "model.src_reader.char_vocab", default=None), vocab_size: Optional[int] = None, emb_dim: int = Ref("exp_global.default_layer_dim"), weight_noise: float = Ref("exp_global.weight_noise", default=0.0), param_init: pinit.ParamInitializer = Ref( "exp_global.param_init", default=bare(pinit.GlorotInitializer)), bias_init: pinit.ParamInitializer = Ref( "exp_global.bias_init", default=bare(pinit.ZeroInitializer)), composer: seq_composer.SequenceComposer = bare( seq_composer.SumComposer), fix_norm: Optional[float] = None): super().__init__(emb_dim=emb_dim, weight_noise=weight_noise, fix_norm=fix_norm) self.composer = composer # Embedding Parameters pcol = param_collections.ParamManager.my_params(self) self.vocab_size = self.choose_vocab_size(vocab_size, char_vocab, '', None, None) self.embeddings = pcol.add_lookup_parameters( (self.vocab_size, self.emb_dim), init=param_init.initializer((self.vocab_size, self.emb_dim), is_lookup=True)) # Model States self.train = False self.save_processed_arg("vocab_size", self.vocab_size)
def __init__(self, weight_noise: float = Ref("exp_global.weight_noise", default=0.0), layers: int = 1, input_dim: int = 120, chn_dim: int = 3, num_filters: int = 32, stride: Tuple[int] = (2, 2), batch_norm: bool = False, nonlinearity: str = "rectify", pre_activation: bool = False, output_tensor: bool = False, transpose: bool = True, param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer))): assert layers > 0 if input_dim % chn_dim != 0: raise ValueError( f"StridedConvEncoder requires input_dim mod chn_dim == 0, got: {input_dim} and {chn_dim}" ) param_col = param_collections.ParamManager.my_params(self) self.layers = layers self.chn_dim = chn_dim self.freq_dim = input_dim / chn_dim self.num_filters = num_filters self.filter_size_time = 3 self.filter_size_freq = 3 self.stride = stride self.output_transposed_tensor = output_tensor self.nonlinearity = nonlinearity self.pre_activation = pre_activation self.use_bn = batch_norm self.train = True self.transpose = transpose self.weight_noise = regularizers.WeightNoise(weight_noise) self.bn_layers = [] self.filters_layers = [] for layer_i in range(layers): filter_dim = (self.filter_size_time, self.filter_size_freq, self.chn_dim if layer_i == 0 else self.num_filters, self.num_filters) filters = param_col.add_parameters( dim=filter_dim, init=param_init[layer_i].initializer(filter_dim) if isinstance( param_init, Sequence) else param_init.initializer(filter_dim)) if self.use_bn: self.bn_layers.append( norms.BatchNorm(param_col, (self.chn_dim if self.pre_activation else self.num_filters), 3)) self.filters_layers.append(filters)
def __init__(self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), state_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), param_init: param_initializers.ParamInitializer = Ref("exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), truncate_dec_batches: bool = Ref("exp_global.truncate_dec_batches", default=False)) -> None: if truncate_dec_batches: raise NotImplementedError("truncate_dec_batches not yet implemented for BilinearAttender") self.input_dim = input_dim self.state_dim = state_dim param_collection = param_collections.ParamManager.my_params(self) self.pWa = param_collection.add_parameters((input_dim, state_dim), init=param_init.initializer((input_dim, state_dim))) self.curr_sent = None
def __init__( self, input_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), state_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)) ) -> None: self.input_dim = input_dim self.state_dim = state_dim my_params = param_collections.ParamManager.my_params(self) self.pWa = my_params.add_parameters((input_dim, state_dim), init=param_init.initializer( (input_dim, state_dim))) self.curr_sent = None
def __init__(self, hidden_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), output_len: numbers.Integral = 32, pos_enc_max: Optional[numbers.Integral] = None, param_init: param_initializers.ParamInitializer = Ref("exp_global.param_init", default=bare(param_initializers.GlorotInitializer))) \ -> None: subcol = param_collections.ParamManager.my_params(self) self.output_len = output_len self.W = subcol.add_parameters(dim=(hidden_dim, output_len), init=param_init.initializer((hidden_dim, output_len))) self.pos_enc_max = pos_enc_max if self.pos_enc_max: self.pos_enc = np.zeros((self.pos_enc_max, self.output_len)) for k in range(self.output_len): for s in range(self.pos_enc_max): self.pos_enc[s, k] = (1.0 - k / self.output_len) * ( 1.0 - s / self.pos_enc_max) + k / self.output_len * s / self.pos_enc_max
def __init__(self, max_pos: numbers.Integral, emb_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), param_init: param_initializers.ParamInitializer = Ref("exp_global.param_init", default=bare(param_initializers.GlorotInitializer))) \ -> None: """ max_pos: largest embedded position emb_dim: embedding size param_init: how to initialize embedding matrix """ self.max_pos = max_pos self.emb_dim = emb_dim param_collection = param_collections.ParamManager.my_params(self) param_init = param_init dim = (self.emb_dim, max_pos) self.embeddings = param_collection.add_parameters(dim, init=param_init.initializer(dim, is_lookup=True))
def __init__(self, max_pos: int, op: str = 'sum', emb_type: str = 'param', input_dim: int = Ref("exp_global.default_layer_dim"), param_init: ParamInitializer = Ref("exp_global.param_init", default=bare(GlorotInitializer))): """ max_pos: largest embedded position op: how to combine positional encodings with the original encodings, can be "sum" or "concat" type: what type of embddings to use, "param"=parameterized (others, such as the trigonometric embeddings are todo) input_dim: embedding size param_init: how to initialize embedding matrix """ self.max_pos = max_pos self.input_dim = input_dim self.op = op self.emb_type = emb_type param_init = param_init dim = (self.input_dim, max_pos) param_collection = ParamManager.my_params(self) self.embedder = param_collection.add_parameters(dim, init=param_init.initializer(dim, is_lookup=True))
def __init__( self, emb_dim: numbers.Integral = Ref("exp_global.default_layer_dim"), weight_noise: numbers.Real = Ref("exp_global.weight_noise", default=0.0), word_dropout: numbers.Real = 0.0, fix_norm: Optional[numbers.Real] = None, param_init: param_initializers.ParamInitializer = Ref( "exp_global.param_init", default=bare(param_initializers.GlorotInitializer)), vocab_size: Optional[numbers.Integral] = None, vocab: Optional[vocabs.Vocab] = None, yaml_path: Path = Path(), src_reader: Optional[input_readers.InputReader] = Ref( "model.src_reader", default=None), trg_reader: Optional[input_readers.InputReader] = Ref( "model.trg_reader", default=None) ) -> None: self.emb_dim = emb_dim self.weight_noise = weight_noise self.word_dropout = word_dropout self.fix_norm = fix_norm self.word_id_mask = None self.train = False my_params = param_collections.ParamManager.my_params(self) self.vocab_size = self.choose_vocab_size(vocab_size, vocab, yaml_path, src_reader, trg_reader) self.save_processed_arg("vocab_size", self.vocab_size) if isinstance(param_init, param_initializers.NumpyInitializer): if param_init.array.shape != (self.vocab_size, self.emb_dim): raise ValueError( f"Expected numpy array of shape {(self.vocab_size, self.emb_dim)}, got {param_init.array.shape}" ) self.embeddings = my_params.lookup_parameters_from_numpy( param_init.array, name=self.xnmt_subcol_name) else: self.embeddings = my_params.add_lookup_parameters( (self.vocab_size, self.emb_dim), init=param_init.initializer((self.vocab_size, self.emb_dim), is_lookup=True))