def setUp(self):
layer_dim = 512
xnmt.events.clear()
ParamManager.init_param_col()
self.model = DefaultTranslator(
src_reader=PlainTextReader(),
trg_reader=PlainTextReader(),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
trg_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
decoder=MlpSoftmaxDecoder(input_dim=layer_dim,
lstm_dim=layer_dim,
mlp_hidden_dim=layer_dim,
trg_embed_dim=layer_dim,
vocab_size=100,
bridge=CopyBridge(dec_dim=layer_dim,
dec_layers=1)),
)
self.model.set_train(False)
self.model.initialize_generator()
self.src_data = list(
self.model.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(
self.model.trg_reader.read_sents("examples/data/head.en"))
def setUp(self):
layer_dim = 512
xnmt.events.clear()
ParamManager.init_param_col()
self.model = DefaultTranslator(
src_reader=PlainTextReader(),
trg_reader=PlainTextReader(),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
trg_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
decoder=AutoRegressiveDecoder(
input_dim=layer_dim,
trg_embed_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
decoder_input_dim=layer_dim,
yaml_path="model.decoder.rnn"),
transform=NonLinear(input_dim=layer_dim * 2,
output_dim=layer_dim),
scorer=Softmax(input_dim=layer_dim, vocab_size=100),
bridge=CopyBridge(dec_dim=layer_dim, dec_layers=1)),
)
self.model.set_train(False)
self.src_data = list(
self.model.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(
self.model.trg_reader.read_sents("examples/data/head.en"))
def setUp(self):
xnmt.events.clear()
ParamManager.init_param_col()
self.src_reader = PlainTextReader()
self.trg_reader = PlainTextReader()
self.src_data = list(self.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(self.trg_reader.read_sents("examples/data/head.en"))
def _load_referenced_serialized(experiment):
for path, node in traverse_tree(experiment):
if isinstance(node, LoadSerialized):
try:
with open(node.filename) as stream:
loaded_root = yaml.load(stream)
except IOError as e:
raise RuntimeError(f"Could not read configuration file {node.filename}: {e}")
ParamManager.add_load_path(f"{node.filename}.data")
cur_path = Path(getattr(node, "path", ""))
for _ in range(10): # follow references
loaded_trg = get_descendant(loaded_root, cur_path, redirect=True)
if isinstance(loaded_trg, Ref):
cur_path = loaded_trg.get_path()
else:
break
found_outside_ref = True
while found_outside_ref:
found_outside_ref = False
named_paths = get_named_paths(loaded_root)
replaced_paths = {}
for sub_path, sub_node in traverse_tree(loaded_trg, path_to_node=cur_path):
if isinstance(sub_node, Ref):
referenced_path = sub_node.resolve_path(named_paths)
if referenced_path.is_relative_path():
raise NotImplementedError("Handling of relative paths with LoadSerialized is not yet implemented.")
if referenced_path in replaced_paths:
set_descendant(loaded_trg, sub_path[len(cur_path):],
Ref(replaced_paths[referenced_path], default=sub_node.get_default()))
# if outside node:
elif not str(referenced_path).startswith(str(cur_path)):
found_outside_ref = True
referenced_obj = get_descendant(loaded_root, referenced_path)
set_descendant(loaded_trg, sub_path[len(cur_path):], referenced_obj)
replaced_paths[referenced_path] = sub_path
else:
set_descendant(loaded_trg, sub_path[len(cur_path):],
Ref(path.add_path(referenced_path[len(cur_path):]),
default=sub_node.get_default()))
for d in getattr(node, "overwrite", []):
overwrite_path = Path(d["path"])
set_descendant(loaded_trg, overwrite_path, d["val"])
if len(path) == 0:
experiment = loaded_trg
else:
set_descendant(experiment, path, loaded_trg)
return experiment
def __init__(self,
input_dim=Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer)),
num_heads=8):
assert (input_dim % num_heads == 0)
param_collection = ParamManager.my_params(self)
self.input_dim = input_dim
self.num_heads = num_heads
self.head_dim = input_dim // num_heads
self.pWq, self.pWk, self.pWv, self.pWo = [
param_collection.add_parameters(dim=(input_dim, input_dim),
init=param_init.initializer(
(input_dim, input_dim)))
for _ in range(4)
]
self.pbq, self.pbk, self.pbv, self.pbo = [
param_collection.add_parameters(dim=(1, input_dim),
init=bias_init.initializer((
1,
input_dim,
))) for _ in range(4)
]
def __init__(self,
input_dim=Ref("exp_global.default_layer_dim"),
state_dim=Ref("exp_global.default_layer_dim"),
hidden_dim=Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer))):
self.input_dim = input_dim
self.state_dim = state_dim
self.hidden_dim = hidden_dim
param_collection = ParamManager.my_params(self)
self.pW = param_collection.add_parameters((hidden_dim, input_dim),
init=param_init.initializer(
(hidden_dim, input_dim)))
self.pV = param_collection.add_parameters((hidden_dim, state_dim),
init=param_init.initializer(
(hidden_dim, state_dim)))
self.pb = param_collection.add_parameters((hidden_dim, ),
init=bias_init.initializer(
(hidden_dim, )))
self.pU = param_collection.add_parameters((1, hidden_dim),
init=param_init.initializer(
(1, hidden_dim)))
self.curr_sent = None
def __init__(self,
layers,
input_dim,
hidden_dim,
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init",
default=bare(ZeroInitializer))):
if layers != 1:
raise RuntimeError(
"CustomLSTMSeqTransducer supports only exactly one layer")
self.input_dim = input_dim
self.hidden_dim = hidden_dim
model = ParamManager.my_params(self)
# [i; f; o; g]
self.p_Wx = model.add_parameters(dim=(hidden_dim * 4, input_dim),
init=param_init.initializer(
(hidden_dim * 4, input_dim)))
self.p_Wh = model.add_parameters(dim=(hidden_dim * 4, hidden_dim),
init=param_init.initializer(
(hidden_dim * 4, hidden_dim)))
self.p_b = model.add_parameters(dim=(hidden_dim * 4, ),
init=bias_init.initializer(
(hidden_dim * 4, )))
def __init__(self,
## COMPONENTS
embed_encoder=None, segment_composer=None, final_transducer=None,
## OPTIONS
length_prior=3.3,
length_prior_alpha=None, # GeometricSequence
epsilon_greedy=None, # GeometricSequence
reinforce_scale=None, # GeometricSequence
confidence_penalty=None, # SegmentationConfidencePenalty
# For segmentation warmup (Always use the poisson prior)
segmentation_warmup=0,
## FLAGS
learn_delete = False,
use_baseline = True,
z_normalization = True,
learn_segmentation = True,
compose_char = False,
log_reward = True,
debug=False,
print_sample=False):
model = ParamManager.my_params(self)
# Sanity check
assert embed_encoder is not None
assert segment_composer is not None
assert final_transducer is not None
# The Embed Encoder transduces the embedding vectors to a sequence of vector
self.embed_encoder = embed_encoder
if not hasattr(embed_encoder, "hidden_dim"):
embed_encoder_dim = yaml_context.default_layer_dim
else:
embed_encoder_dim = embed_encoder.hidden_dim
# The Segment transducer produced word embeddings based on sequence of character embeddings
self.segment_composer = segment_composer
# The final transducer
self.final_transducer = final_transducer
# Decision layer of segmentation
self.segment_transform = linear.Linear(input_dim = embed_encoder_dim,
output_dim = 3 if learn_delete else 2)
# The baseline linear regression model
self.baseline = linear.Linear(input_dim = embed_encoder_dim,
output_dim = 1)
# Flags
self.use_baseline = use_baseline
self.learn_segmentation = learn_segmentation
self.learn_delete = learn_delete
self.z_normalization = z_normalization
self.debug = debug
self.compose_char = compose_char
self.print_sample = print_sample
self.log_reward = log_reward
# Fixed Parameters
self.length_prior = length_prior
self.segmentation_warmup = segmentation_warmup
# Variable Parameters
self.length_prior_alpha = length_prior_alpha
self.lmbd = reinforce_scale
self.eps = epsilon_greedy
self.confidence_penalty = confidence_penalty
# States of the object
self.train = False
def __init__(self,
filename,
emb_dim=Ref("exp_global.default_layer_dim"),
weight_noise=Ref("exp_global.weight_noise", default=0.0),
word_dropout=0.0,
fix_norm = None,
vocab = None,
yaml_path = None,
src_reader = Ref("model.src_reader", default=None),
trg_reader = Ref("model.trg_reader", default=None)):
self.emb_dim = emb_dim
self.weight_noise = weight_noise
self.word_dropout = word_dropout
self.word_id_mask = None
self.train = False
self.fix_norm = fix_norm
self.pretrained_filename = filename
param_collection = ParamManager.my_params(self)
self.vocab = self.choose_vocab(vocab, yaml_path, src_reader, trg_reader)
self.vocab_size = len(vocab)
self.save_processed_arg("vocab", self.vocab)
with open(self.pretrained_filename, encoding='utf-8') as embeddings_file:
total_embs, in_vocab, missing, initial_embeddings = self._read_fasttext_embeddings(vocab, embeddings_file)
self.embeddings = param_collection.lookup_parameters_from_numpy(initial_embeddings)
logger.info(f"{in_vocab} vocabulary matches out of {total_embs} total embeddings; "
f"{missing} vocabulary words without a pretrained embedding out of {self.vocab_size}")
def __init__(
self,
input_dim: int = Ref("exp_global.default_layer_dim"),
state_dim: int = Ref("exp_global.default_layer_dim"),
hidden_dim: int = Ref("exp_global.default_layer_dim"),
param_init: ParamInitializer = Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init: ParamInitializer = Ref("exp_global.bias_init",
default=bare(ZeroInitializer)),
truncate_dec_batches: bool = Ref("exp_global.truncate_dec_batches",
default=False)
) -> None:
self.input_dim = input_dim
self.state_dim = state_dim
self.hidden_dim = hidden_dim
self.truncate_dec_batches = truncate_dec_batches
param_collection = ParamManager.my_params(self)
self.pW = param_collection.add_parameters((hidden_dim, input_dim),
init=param_init.initializer(
(hidden_dim, input_dim)))
self.pV = param_collection.add_parameters((hidden_dim, state_dim),
init=param_init.initializer(
(hidden_dim, state_dim)))
self.pb = param_collection.add_parameters((hidden_dim, ),
init=bias_init.initializer(
(hidden_dim, )))
self.pU = param_collection.add_parameters((1, hidden_dim),
init=param_init.initializer(
(1, hidden_dim)))
self.curr_sent = None
def __init__(self,
layers=1,
input_dim=Ref("exp_global.default_layer_dim"),
hidden_dim=Ref("exp_global.default_layer_dim"),
dropout = Ref("exp_global.dropout", default=0.0),
weightnoise_std=Ref("exp_global.weight_noise", default=0.0),
param_init=Ref("exp_global.param_init", default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init", default=bare(ZeroInitializer)),
yaml_path=None,
decoder_input_dim=Ref("exp_global.default_layer_dim", default=None),
decoder_input_feeding=True):
self.num_layers = layers
model = ParamManager.my_params(self)
if yaml_path is not None and "decoder" in yaml_path:
if decoder_input_feeding:
input_dim += decoder_input_dim
self.hidden_dim = hidden_dim
self.dropout_rate = dropout
self.weightnoise_std = weightnoise_std
self.input_dim = input_dim
if not isinstance(param_init, Sequence):
param_init = [param_init] * layers
if not isinstance(bias_init, Sequence):
bias_init = [bias_init] * layers
# [i; f; o; g]
self.p_Wx = [model.add_parameters(dim=(hidden_dim*4, input_dim), init=param_init[0].initializer((hidden_dim*4, input_dim), num_shared=4))]
self.p_Wx += [model.add_parameters(dim=(hidden_dim*4, hidden_dim), init=param_init[i].initializer((hidden_dim*4, hidden_dim), num_shared=4)) for i in range(1, layers)]
self.p_Wh = [model.add_parameters(dim=(hidden_dim*4, hidden_dim), init=param_init[i].initializer((hidden_dim*4, hidden_dim), num_shared=4)) for i in range(layers)]
self.p_b = [model.add_parameters(dim=(hidden_dim*4,), init=bias_init[i].initializer((hidden_dim*4,), num_shared=4)) for i in range(layers)]
self.dropout_mask_x = None
self.dropout_mask_h = None
def __init__(
self,
input_dim: int = Ref("exp_global.default_layer_dim"),
trg_embed_dim: int = Ref("exp_global.default_layer_dim"),
input_feeding: bool = True,
bridge: Bridge = bare(CopyBridge),
rnn: UniLSTMSeqTransducer = bare(UniLSTMSeqTransducer),
transform: Transform = bare(AuxNonLinear),
scorer: Scorer = bare(Softmax),
truncate_dec_batches: bool = Ref("exp_global.truncate_dec_batches",
default=False)
) -> None:
self.param_col = ParamManager.my_params(self)
self.input_dim = input_dim
self.truncate_dec_batches = truncate_dec_batches
self.bridge = bridge
self.rnn = rnn
self.transform = transform
self.scorer = scorer
# Input feeding
self.input_feeding = input_feeding
rnn_input_dim = trg_embed_dim
if input_feeding:
rnn_input_dim += input_dim
assert rnn_input_dim == rnn.input_dim, "Wrong input dimension in RNN layer: {} != {}".format(
rnn_input_dim, rnn.input_dim)
def __init__(self, filter_height, filter_width, channels, num_filters,
stride):
"""
Args:
num_layers: depth of the RNN
input_dim: size of the inputs
hidden_dim: size of the outputs (and intermediate RNN layer representations)
"""
model = ParamManager.my_params(self)
self.filter_height = filter_height
self.filter_width = filter_width
self.channels = channels
self.num_filters = num_filters
self.stride = stride # (2,2)
self.hidden_states = {}
normalInit = dy.NormalInitializer(0, 0.1)
self.filters1 = model.add_parameters(
dim=(self.filter_height[0], self.filter_width[0], self.channels[0],
self.num_filters[0]),
init=normalInit)
self.filters2 = model.add_parameters(
dim=(self.filter_height[1], self.filter_width[1], self.channels[1],
self.num_filters[1]),
init=normalInit)
self.filters3 = model.add_parameters(
dim=(self.filter_height[2], self.filter_width[2], self.channels[2],
self.num_filters[2]),
init=normalInit)
def __init__(self,
input_dim=Ref("exp_global.default_layer_dim"),
trg_embed_dim=Ref("exp_global.default_layer_dim"),
input_feeding=True,
rnn_layer=bare(UniLSTMSeqTransducer),
mlp_layer=bare(MLP),
bridge=bare(CopyBridge),
label_smoothing=0.0):
self.param_col = ParamManager.my_params(self)
self.input_dim = input_dim
self.label_smoothing = label_smoothing
# Input feeding
self.input_feeding = input_feeding
rnn_input_dim = trg_embed_dim
if input_feeding:
rnn_input_dim += input_dim
assert rnn_input_dim == rnn_layer.input_dim, "Wrong input dimension in RNN layer"
# Bridge
self.bridge = bridge
# LSTM
self.rnn_layer = rnn_layer
# MLP
self.mlp_layer = mlp_layer
def __init__(self,
emb_dim=Ref("exp_global.default_layer_dim"),
weight_noise=Ref("exp_global.weight_noise", default=0.0),
word_dropout=0.0,
fix_norm=None,
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
vocab_size=None,
vocab=None,
yaml_path=None,
src_reader=Ref("model.src_reader", default=None),
trg_reader=Ref("model.trg_reader", default=None)):
#print(f"embedder received param_init: {param_init}")
self.emb_dim = emb_dim
self.weight_noise = weight_noise
self.word_dropout = word_dropout
self.fix_norm = fix_norm
self.word_id_mask = None
self.train = False
param_collection = ParamManager.my_params(self)
self.vocab_size = self.choose_vocab_size(vocab_size, vocab, yaml_path,
src_reader, trg_reader)
self.save_processed_arg("vocab_size", self.vocab_size)
self.embeddings = param_collection.add_lookup_parameters(
(self.vocab_size, self.emb_dim),
init=param_init.initializer((self.vocab_size, self.emb_dim),
is_lookup=True))
def __init__(self, child: SeqTransducer, input_dim: int, layer_norm: bool = False):
self.child = child
self.input_dim = input_dim
self.layer_norm = layer_norm
if layer_norm:
model = ParamManager.my_params(self)
self.ln_g = model.add_parameters(dim=(input_dim,))
self.ln_b = model.add_parameters(dim=(input_dim,))
def __init__(self,
alpha=0.001,
beta_1=0.9,
beta_2=0.999,
eps=1e-8,
update_every: int = 1,
skip_noisy: bool = False):
super().__init__(optimizer=dy.AdamTrainer(
ParamManager.global_collection(), alpha, beta_1, beta_2, eps),
skip_noisy=skip_noisy)
def __init__(self,
filter_height,
filter_width,
channels,
num_filters,
stride,
rhn_num_hidden_layers,
rhn_dim,
rhn_microsteps,
attention_dim,
residual=False):
self.filter_height = filter_height
self.filter_width = filter_width
self.channels = channels
self.num_filters = num_filters
self.stride = stride
self.rhn_num_hidden_layers = rhn_num_hidden_layers
self.rhn_dim = rhn_dim
self.rhn_microsteps = rhn_microsteps
self.attention_dim = attention_dim
self.residual = residual
model = ParamManager.my_params(self)
# Convolutional layer
self.filter_conv = model.add_parameters(dim=(self.filter_height,
self.filter_width,
self.channels,
self.num_filters))
# Recurrent highway layer
self.recur = []
self.linear = []
self.init = []
self.attention = []
input_dim = num_filters
for _ in range(rhn_num_hidden_layers):
self.init.append(model.add_parameters((rhn_dim, )))
self.linear.append((model.add_parameters((rhn_dim, input_dim)),
model.add_parameters((
rhn_dim,
input_dim,
))))
input_dim = rhn_dim
recur_layer = []
for _ in range(self.rhn_microsteps):
recur_layer.append((model.add_parameters(
(rhn_dim, rhn_dim)), model.add_parameters(
(rhn_dim, )), model.add_parameters((
rhn_dim,
rhn_dim,
)), model.add_parameters((rhn_dim, ))))
self.recur.append(recur_layer)
# Attention layer
self.attention.append((model.add_parameters(
(attention_dim, rhn_dim)), model.add_parameters(attention_dim, )))
def __init__(self,
input_dim=Ref("exp_global.default_layer_dim"),
state_dim=Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer))):
self.input_dim = input_dim
self.state_dim = state_dim
param_collection = ParamManager.my_params(self)
self.pWa = param_collection.add_parameters((input_dim, state_dim),
init=param_init.initializer(
(input_dim, state_dim)))
self.curr_sent = None
def __init__(self, max_pos: int, emb_dim: int = Ref("exp_global.default_layer_dim"),
param_init: ParamInitializer = Ref("exp_global.param_init", default=bare(GlorotInitializer))):
"""
max_pos: largest embedded position
emb_dim: embedding size
param_init: how to initialize embedding matrix
"""
self.max_pos = max_pos
self.emb_dim = emb_dim
param_collection = ParamManager.my_params(self)
param_init = param_init
dim = (self.emb_dim, max_pos)
self.embeddings = param_collection.add_parameters(dim, init=param_init.initializer(dim, is_lookup=True))
def __init__(self,
input_dim,
window_receptor,
output_dim,
num_layers,
internal_dim,
non_linearity='linear'):
"""
Args:
num_layers: num layers after first receptor conv
input_dim: size of the inputs
window_receptor: window for the receptor
ouput_dim: size of the outputs
internal_dim: size of hidden dimension, internal dimension
non_linearity: Non linearity to apply between layers
"""
model = ParamManager.my_params(self)
self.input_dim = input_dim
self.window_receptor = window_receptor
self.internal_dim = internal_dim
self.non_linearity = non_linearity
self.output_dim = output_dim
if self.non_linearity == 'linear':
self.gain = 1.0
elif self.non_linearity == 'tanh':
self.gain = 1.0
elif self.non_linearity == 'relu':
self.gain = 0.5
elif self.non_linearity == 'sigmoid':
self.gain = 4.0
normalInit = dy.NormalInitializer(0, 0.1)
self.pConv1 = model.add_parameters(dim=(self.input_dim,
self.window_receptor, 1,
self.internal_dim),
init=normalInit)
self.pBias1 = model.add_parameters(dim=(self.internal_dim, ))
self.builder_layers = []
for _ in range(num_layers):
conv = model.add_parameters(dim=(self.internal_dim, 1, 1,
self.internal_dim),
init=normalInit)
bias = model.add_parameters(dim=(self.internal_dim, ))
self.builder_layers.append((conv, bias))
self.last_conv = model.add_parameters(dim=(self.internal_dim, 1, 1,
self.output_dim),
init=normalInit)
self.last_bias = model.add_parameters(dim=(self.output_dim, ))
def __init__(self,
dec_layers = 1,
enc_dim = Ref("exp_global.default_layer_dim"),
dec_dim = Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init", default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init", default=bare(ZeroInitializer))):
param_col = ParamManager.my_params(self)
self.dec_layers = dec_layers
self.enc_dim = enc_dim
self.dec_dim = dec_dim
self.projector = xnmt.linear.Linear(input_dim=enc_dim,
output_dim=dec_dim,
param_init=param_init,
bias_init=bias_init)
def __init__(self,
input_dim=Ref("exp_global.default_layer_dim"),
hidden_dim=Ref("exp_global.default_layer_dim"),
output_dim=Ref("exp_global.default_layer_dim", default=None),
param_init_hidden=Ref("exp_global.param_init", default=bare(GlorotInitializer)),
bias_init_hidden=Ref("exp_global.bias_init", default=bare(ZeroInitializer)),
param_init_output=Ref("exp_global.param_init", default=bare(GlorotInitializer)),
bias_init_output=Ref("exp_global.bias_init", default=bare(ZeroInitializer)),
activation='tanh',
hidden_layer=None,
output_projector=None,
yaml_path=None,
vocab_size=None,
vocab=None,
trg_reader=Ref("model.trg_reader", default=None),
decoder_rnn_dim=Ref("exp_global.default_layer_dim", default=None)):
model = ParamManager.my_params(self)
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.output_dim = output_dim
if yaml_path is not None and "decoder" in yaml_path:
self.input_dim += decoder_rnn_dim
self.output_dim = self.choose_vocab_size(vocab_size, vocab, trg_reader)
self.save_processed_arg("vocab_size", self.output_dim)
self.hidden_layer = self.add_serializable_component("hidden_layer", hidden_layer,
lambda: xnmt.linear.Linear(input_dim=self.input_dim,
output_dim=self.hidden_dim,
param_init=param_init_hidden,
bias_init=bias_init_hidden))
if activation == 'tanh':
self.activation = dy.tanh
elif activation == 'relu':
self.activation = dy.rectify
elif activation == 'sigmoid':
self.activation = dy.sigmoid
elif activation == 'elu':
self.activation = dy.elu
elif activation == 'selu':
self.activation = dy.selu
elif activation == 'asinh':
self.activation = dy.asinh
else:
raise ValueError('Unknown activation %s' % activation)
self.output_projector = self.add_serializable_component("output_projector", output_projector,
lambda: output_projector or xnmt.linear.Linear(
input_dim=self.hidden_dim, output_dim=self.output_dim,
param_init=param_init_output, bias_init=bias_init_output))
def __init__(self,
alpha=1.0,
dim=512,
warmup_steps=4000,
beta_1=0.9,
beta_2=0.98,
eps=1e-9):
self.optimizer = dy.AdamTrainer(ParamManager.global_collection(),
alpha=alpha,
beta_1=beta_1,
beta_2=beta_2,
eps=eps)
self.dim = dim
self.warmup_steps = warmup_steps
self.steps = 0
def __init__(self, in_height, out_height):
"""
Args:
num_layers: depth of the RNN
input_dim: size of the inputs
hidden_dim: size of the outputs (and intermediate RNN layer representations)
"""
model = ParamManager.my_params(self)
self.in_height = in_height
self.out_height = out_height
normalInit=dy.NormalInitializer(0, 0.1)
self.pW = model.add_parameters(dim = (self.out_height, self.in_height), init=normalInit)
self.pb = model.add_parameters(dim = self.out_height)
def __init__(self,
ngram_size,
param_init=Ref("exp_global.param_init",
default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.param_init",
default=bare(ZeroInitializer)),
embed_dim=Ref("exp_global.default_layer_dim"),
hidden_dim=Ref("exp_global.default_layer_dim")):
model = ParamManager.my_params(self)
dim = (1, ngram_size, embed_dim, hidden_dim)
self.filter = model.add_parameters(dim=dim,
init=param_init.initializer(dim))
self.bias = model.add_parameters(dim=(embed_dim, ),
init=bias_init.initializer(dim))
self.ngram_size = ngram_size
self.embed_dim = embed_dim
def __init__(self, in_height, out_height, nonlinearity='linear'):
"""
Args:
in_height: input dimension of the affine transform
out_height: output dimension of the affine transform
nonlinearity: nonlinear activation function
"""
model = ParamManager.my_params(self)
self.in_height = in_height
self.out_height = out_height
self.nonlinearity = nonlinearity
normalInit = dy.NormalInitializer(0, 0.1)
self.pW = model.add_parameters(dim=(self.out_height, self.in_height),
init=normalInit)
self.pb = model.add_parameters(dim=self.out_height)
def __init__(self,
layers=1,
input_dim=512,
h=1,
dropout=0.0,
attn_dropout=False,
layer_norm=False,
**kwargs):
dy_model = ParamManager.my_params(self)
self.layer_names = []
for i in range(1, layers + 1):
name = 'l{}'.format(i)
layer = EncoderLayer(dy_model, input_dim, h, attn_dropout,
layer_norm)
self.layer_names.append((name, layer))
self.dropout_val = dropout
def __init__(self,
dec_layers = 1,
enc_dim = Ref("exp_global.default_layer_dim"),
dec_dim = Ref("exp_global.default_layer_dim"),
param_init=Ref("exp_global.param_init", default=bare(GlorotInitializer)),
bias_init=Ref("exp_global.bias_init", default=bare(ZeroInitializer)),
projector=None):
param_col = ParamManager.my_params(self)
self.dec_layers = dec_layers
self.enc_dim = enc_dim
self.dec_dim = dec_dim
self.projector = self.add_serializable_component("projector",
projector,
lambda: xnmt.linear.Linear(input_dim=self.enc_dim,
output_dim=self.dec_dim,
param_init=param_init,
bias_init=bias_init))
def __init__(self,
alpha=1.0,
dim=512,
warmup_steps=4000,
beta_1=0.9,
beta_2=0.98,
eps=1e-9,
skip_noisy: bool = False):
super().__init__(optimizer=dy.AdamTrainer(
ParamManager.global_collection(),
alpha=alpha,
beta_1=beta_1,
beta_2=beta_2,
eps=eps),
skip_noisy=skip_noisy)
self.dim = dim
self.warmup_steps = warmup_steps
self.steps = 0
