def __init__(self,
layers,
input_dim,
hidden_dim,
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer))):
if layers != 1:
raise RuntimeError(
"CustomLSTMSeqTransducer supports only exactly one layer")
self.input_dim = input_dim
self.hidden_dim = hidden_dim
model = ParamManager.my_params(self)
# [i; f; o; g]
self.p_Wx = model.add_parameters(dim=(hidden_dim * 4, input_dim),
init=param_init.initializer(
(hidden_dim * 4, input_dim)))
self.p_Wh = model.add_parameters(dim=(hidden_dim * 4, hidden_dim),
init=param_init.initializer(
(hidden_dim * 4, hidden_dim)))
self.p_b = model.add_parameters(dim=(hidden_dim * 4, ),
init=bias_init.initializer(
(hidden_dim * 4, )))
def __init__(self,
input_dim=Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer)),
num_heads=8):
assert (input_dim % num_heads == 0)
param_collection = ParamManager.my_params(self)
self.input_dim = input_dim
self.num_heads = num_heads
self.head_dim = input_dim // num_heads
self.pWq, self.pWk, self.pWv, self.pWo = [
param_collection.add_parameters(dim=(input_dim, input_dim),
init=param_init.initializer(
(input_dim, input_dim)))
for _ in range(4)
]
self.pbq, self.pbk, self.pbv, self.pbo = [
param_collection.add_parameters(dim=(1, input_dim),
init=bias_init.initializer((
1,
input_dim,
))) for _ in range(4)
]
def __init__(self, filter_height, filter_width, channels, num_filters,
stride):
"""
Args:
num_layers: depth of the RNN
input_dim: size of the inputs
hidden_dim: size of the outputs (and intermediate RNN layer representations)
"""
model = ParamManager.my_params(self)
self.filter_height = filter_height
self.filter_width = filter_width
self.channels = channels
self.num_filters = num_filters
self.stride = stride # (2,2)
self.hidden_states = {}
normalInit = dy.NormalInitializer(0, 0.1)
self.filters1 = model.add_parameters(
dim=(self.filter_height[0], self.filter_width[0], self.channels[0],
self.num_filters[0]),
init=normalInit)
self.filters2 = model.add_parameters(
dim=(self.filter_height[1], self.filter_width[1], self.channels[1],
self.num_filters[1]),
init=normalInit)
self.filters3 = model.add_parameters(
dim=(self.filter_height[2], self.filter_width[2], self.channels[2],
self.num_filters[2]),
init=normalInit)
def __init__(self,
layers=1,
input_dim=Ref("exp_global.default_layer_dim"),
hidden_dim=Ref("exp_global.default_layer_dim"),
dropout=Ref("exp_global.dropout", default=0.0),
weightnoise_std=Ref("exp_global.weight_noise", default=0.0),
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer)),
yaml_path=None,
decoder_input_dim=Ref("exp_global.default_layer_dim",
default=None),
decoder_input_feeding=True):
self.num_layers = layers
model = ParamManager.my_params(self)
if yaml_path is not None and "decoder" in yaml_path:
if decoder_input_feeding:
input_dim += decoder_input_dim
self.hidden_dim = hidden_dim
self.dropout_rate = dropout
self.weightnoise_std = weightnoise_std
self.input_dim = input_dim
if not isinstance(param_init, Sequence):
param_init = [param_init] * layers
if not isinstance(bias_init, Sequence):
bias_init = [bias_init] * layers
# [i; f; o; g]
self.p_Wx = [
model.add_parameters(dim=(hidden_dim * 4, input_dim),
init=param_init[0].initializer(
(hidden_dim * 4, input_dim),
num_shared=4))
]
self.p_Wx += [
model.add_parameters(dim=(hidden_dim * 4, hidden_dim),
init=param_init[i].initializer(
(hidden_dim * 4, hidden_dim),
num_shared=4)) for i in range(1, layers)
]
self.p_Wh = [
model.add_parameters(dim=(hidden_dim * 4, hidden_dim),
init=param_init[i].initializer(
(hidden_dim * 4, hidden_dim),
num_shared=4)) for i in range(layers)
]
self.p_b = [
model.add_parameters(dim=(hidden_dim * 4, ),
init=bias_init[i].initializer(
(hidden_dim * 4, ), num_shared=4))
for i in range(layers)
]
self.dropout_mask_x = None
self.dropout_mask_h = None
def __init__(self, layers=1, input_dim=512, h=1,
dropout=0.0, attn_dropout=False, layer_norm=False, **kwargs):
dy_model = ParamManager.my_params(self)
self.layer_names = []
for i in range(1, layers + 1):
name = 'l{}'.format(i)
layer = EncoderLayer(dy_model, input_dim, h, attn_dropout, layer_norm)
self.layer_names.append((name, layer))
self.dropout_val = dropout
def __init__(self,
filter_height,
filter_width,
channels,
num_filters,
stride,
rhn_num_hidden_layers,
rhn_dim,
rhn_microsteps,
attention_dim,
residual=False):
self.filter_height = filter_height
self.filter_width = filter_width
self.channels = channels
self.num_filters = num_filters
self.stride = stride
self.rhn_num_hidden_layers = rhn_num_hidden_layers
self.rhn_dim = rhn_dim
self.rhn_microsteps = rhn_microsteps
self.attention_dim = attention_dim
self.residual = residual
model = ParamManager.my_params(self)
# Convolutional layer
self.filter_conv = model.add_parameters(dim=(self.filter_height,
self.filter_width,
self.channels,
self.num_filters))
# Recurrent highway layer
self.recur = []
self.linear = []
self.init = []
self.attention = []
input_dim = num_filters
for _ in range(rhn_num_hidden_layers):
self.init.append(model.add_parameters((rhn_dim, )))
self.linear.append((model.add_parameters((rhn_dim, input_dim)),
model.add_parameters((
rhn_dim,
input_dim,
))))
input_dim = rhn_dim
recur_layer = []
for _ in range(self.rhn_microsteps):
recur_layer.append((model.add_parameters(
(rhn_dim, rhn_dim)), model.add_parameters(
(rhn_dim, )), model.add_parameters((
rhn_dim,
rhn_dim,
)), model.add_parameters((rhn_dim, ))))
self.recur.append(recur_layer)
# Attention layer
self.attention.append((model.add_parameters(
(attention_dim, rhn_dim)), model.add_parameters(attention_dim, )))
def __init__(self,
child: SeqTransducer,
input_dim: int,
layer_norm: bool = False):
self.child = child
self.input_dim = input_dim
self.layer_norm = layer_norm
if layer_norm:
model = ParamManager.my_params(self)
self.ln_g = model.add_parameters(dim=(input_dim, ))
self.ln_b = model.add_parameters(dim=(input_dim, ))
def __init__(self,
ngram_size,
param_init=Ref("exp_global.param_init", default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.param_init", default=bare(ZeroInitializer)),
embed_dim=Ref("exp_global.default_layer_dim"),
hidden_dim=Ref("exp_global.default_layer_dim")):
model = ParamManager.my_params(self)
dim = (1, ngram_size, embed_dim, hidden_dim)
self.filter = model.add_parameters(dim=dim, init=param_init.initializer(dim))
self.bias = model.add_parameters(dim=(embed_dim,), init=bias_init.initializer(dim))
self.ngram_size = ngram_size
self.embed_dim = embed_dim
def __init__(self,
input_dim,
window_receptor,
output_dim,
num_layers,
internal_dim,
non_linearity='linear'):
"""
Args:
num_layers: num layers after first receptor conv
input_dim: size of the inputs
window_receptor: window for the receptor
ouput_dim: size of the outputs
internal_dim: size of hidden dimension, internal dimension
non_linearity: Non linearity to apply between layers
"""
model = ParamManager.my_params(self)
self.input_dim = input_dim
self.window_receptor = window_receptor
self.internal_dim = internal_dim
self.non_linearity = non_linearity
self.output_dim = output_dim
if self.non_linearity == 'linear':
self.gain = 1.0
elif self.non_linearity == 'tanh':
self.gain = 1.0
elif self.non_linearity == 'relu':
self.gain = 0.5
elif self.non_linearity == 'sigmoid':
self.gain = 4.0
normalInit = dy.NormalInitializer(0, 0.1)
self.pConv1 = model.add_parameters(dim=(self.input_dim,
self.window_receptor, 1,
self.internal_dim),
init=normalInit)
self.pBias1 = model.add_parameters(dim=(self.internal_dim, ))
self.builder_layers = []
for _ in range(num_layers):
conv = model.add_parameters(dim=(self.internal_dim, 1, 1,
self.internal_dim),
init=normalInit)
bias = model.add_parameters(dim=(self.internal_dim, ))
self.builder_layers.append((conv, bias))
self.last_conv = model.add_parameters(dim=(self.internal_dim, 1, 1,
self.output_dim),
init=normalInit)
self.last_bias = model.add_parameters(dim=(self.output_dim, ))
def __init__(self, layers=1, input_dim=512, h=1,
dropout=0.0, attn_dropout=False, layer_norm=False,
vocab_size = None, vocab = None,
trg_reader = Ref("model.trg_reader")):
dy_model = ParamManager.my_params(self)
self.layer_names = []
for i in range(1, layers + 1):
name = 'l{}'.format(i)
layer = DecoderLayer(dy_model, input_dim, h, attn_dropout, layer_norm)
self.layer_names.append((name, layer))
self.vocab_size = self.choose_vocab_size(vocab_size, vocab, trg_reader)
self.output_affine = LinearSent(dy_model, input_dim, self.vocab_size)
self.dropout_val = dropout
def __init__(self, in_height, out_height):
"""
Args:
num_layers: depth of the RNN
input_dim: size of the inputs
hidden_dim: size of the outputs (and intermediate RNN layer representations)
"""
model = ParamManager.my_params(self)
self.in_height = in_height
self.out_height = out_height
normalInit=dy.NormalInitializer(0, 0.1)
self.pW = model.add_parameters(dim = (self.out_height, self.in_height), init=normalInit)
self.pb = model.add_parameters(dim = self.out_height)
def __init__(self,
word_vocab=None,
src_vocab=Ref(Path("model.src_reader.vocab")),
hidden_dim=Ref("exp_global.default_layer_dim"),
vocab_size=25000):
super().__init__()
param_collection = ParamManager.my_params(self)
if word_vocab is None:
word_vocab = Vocab()
dict_entry = vocab_size
else:
dict_entry = len(word_vocab)
self.src_vocab = src_vocab
self.word_vocab = word_vocab
self.embedding = param_collection.add_lookup_parameters((dict_entry, hidden_dim))
def __init__(self,
hidden_dim: numbers.Integral = Ref("exp_global.default_layer_dim"),
output_len: numbers.Integral = 32,
pos_enc_max: Optional[numbers.Integral] = None,
param_init: ParamInitializer = Ref("exp_global.param_init",
default=bare(GlorotInitializer))) \
-> None:
subcol = ParamManager.my_params(self)
self.output_len = output_len
self.W = subcol.add_parameters(dim=(hidden_dim, output_len),
init=param_init.initializer((hidden_dim, output_len)))
self.pos_enc_max = pos_enc_max
if self.pos_enc_max:
self.pos_enc = np.zeros((self.pos_enc_max, self.output_len))
for k in range(self.output_len):
for s in range(self.pos_enc_max):
self.pos_enc[s, k] = (1.0 - k / self.output_len) * (
1.0 - s / self.pos_enc_max) + k / self.output_len * s / self.pos_enc_max
def __init__(self,
input_dim,
hidden_dim,
above_dim,
a,
hier,
last_layer,
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer))):
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.above_dim = above_dim
self.last_layer = last_layer
self.a = a #for slope annealing
self.hier = hier
# self.save_processed_arg("last_layer", self.last_layer)
model = ParamManager.my_params(self)
self.p_W_1l_r = model.add_parameters(dim=(hidden_dim * 4 + 1,
hidden_dim),
init=param_init.initializer(
(hidden_dim * 4 + 1,
hidden_dim)))
if not self.last_layer:
self.p_W_2l_td = model.add_parameters(dim=(hidden_dim * 4 + 1,
above_dim),
init=param_init.initializer(
(hidden_dim * 4 + 1,
above_dim)))
self.p_W_0l_bu = model.add_parameters(dim=(hidden_dim * 4 + 1,
input_dim),
init=param_init.initializer(
(hidden_dim * 4 + 1,
input_dim)))
self.p_bias = model.add_parameters(dim=(hidden_dim * 4 + 1, ),
init=bias_init.initializer(
(hidden_dim * 4 + 1, )))
# to track prev timestep c, h, & z values for this layer
self.c = None
self.h = None
self.z = None
def __init__(self,
max_pos: int,
op: str = 'sum',
emb_type: str = 'param',
input_dim: int = Ref("exp_global.default_layer_dim"),
param_init: ParamInitializer = Ref("exp_global.param_init", default=bare(GlorotInitializer))):
"""
max_pos: largest embedded position
op: how to combine positional encodings with the original encodings, can be "sum" or "concat"
type: what type of embddings to use, "param"=parameterized (others, such as the trigonometric embeddings are todo)
input_dim: embedding size
param_init: how to initialize embedding matrix
"""
self.max_pos = max_pos
self.input_dim = input_dim
self.op = op
self.emb_type = emb_type
param_init = param_init
dim = (self.input_dim, max_pos)
param_collection = ParamManager.my_params(self)
self.embedder = param_collection.add_parameters(dim, init=param_init.initializer(dim, is_lookup=True))
def __init__(self,
word_vocab=None,
vocab_size=32000,
cache_id_pool=None,
cache_word_table=None,
char_vocab=Ref(Path("model.src_reader.vocab")),
hidden_dim=Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init", default=bare(GlorotInitializer))):
super().__init__(word_vocab, vocab_size, cache_id_pool, cache_word_table)
param_collection = ParamManager.my_params(self)
# Attributes
if word_vocab is None:
self.dict_entry = vocab_size+1
else:
self.dict_entry = len(word_vocab)
self.char_vocab = char_vocab
self.hidden_dim = hidden_dim
# Word Embedding
embed_dim = (self.dict_entry, hidden_dim)
self.param_init = param_init.initializer(embed_dim, is_lookup=True)
self.embedding = param_collection.add_lookup_parameters(embed_dim, init=self.param_init)