def _create_decoding_layers(self):
logger.debug("_create_decoding_layers")
self.decode_inputers = [lambda x : 0] * self.num_levels
self.decode_reseters = [lambda x : 0] * self.num_levels
self.decode_updaters = [lambda x : 0] * self.num_levels
self.back_decode_inputers = [lambda x : 0] * self.num_levels
self.back_decode_reseters = [lambda x : 0] * self.num_levels
self.back_decode_updaters = [lambda x : 0] * self.num_levels
decoding_kwargs = dict(self.default_kwargs)
decoding_kwargs.update(dict(
n_in=self.state['c_dim'],
n_hids=self.state['dim'] * self.state['dim_mult'],
activation=['lambda x:x'],
learn_bias=False))
if self.state['decoding_inputs']:
for level in range(self.num_levels):
# Input contributions
self.decode_inputers[level] = MultiLayer(
self.rng,
name='{}_dec_inputter_{}'.format(self.prefix, level),
**decoding_kwargs)
# Update gate contributions
if prefix_lookup(self.state, 'dec', 'rec_gating'):
self.decode_updaters[level] = MultiLayer(
self.rng,
name='{}_dec_updater_{}'.format(self.prefix, level),
**decoding_kwargs)
# Reset gate contributions
if prefix_lookup(self.state, 'dec', 'rec_reseting'):
self.decode_reseters[level] = MultiLayer(
self.rng,
name='{}_dec_reseter_{}'.format(self.prefix, level),
**decoding_kwargs)
def _create_transition_layers(self):
logger.debug("_create_transition_layers")
self.transitions = []
rec_layer = eval(prefix_lookup(self.state, self.prefix, 'dec_rec_layer'))
add_args = dict()
if rec_layer == RecurrentLayerWithSearch:#if the decoder uses search RNN, then there are multiple contexts, the real dimension of c is times of hid num
add_args = dict(c_dim=self.state['c_dim'])
for level in range(self.num_levels):
self.transitions.append(rec_layer(#create a recurrent hidden layer for each level (there could be multiple hidden layers).
self.rng,
n_hids=self.state['dim'],
activation=prefix_lookup(self.state, self.prefix, 'activ'),
bias_scale=self.state['bias'],
init_fn=(self.state['rec_weight_init_fn']
if not self.skip_init
else "sample_zeros"),
scale=prefix_lookup(self.state, self.prefix, 'rec_weight_scale'),
weight_noise=self.state['weight_noise_rec'],
dropout=self.state['dropout_rec'],
gating=prefix_lookup(self.state, self.prefix, 'dec_rec_gating'),
gater_activation=prefix_lookup(self.state, self.prefix, 'dec_rec_gater'),
reseting=prefix_lookup(self.state, self.prefix, 'dec_rec_reseting'),
reseter_activation=prefix_lookup(self.state, self.prefix, 'dec_rec_reseter'),
name='{}_transition_{}'.format(self.prefix, level),
**add_args))
def _create_transition_layers(self):
logger.debug("_create_transition_layers")
self.transitions = []
rec_layer = eval(prefix_lookup(self.state, self.prefix, 'enc_rec_layer'))
add_args = dict()
if rec_layer == RecurrentLayerWithSearch:
add_args = dict(c_dim=self.state['c_dim'])
for level in range(self.num_levels):
self.transitions.append(rec_layer(
self.rng,
n_hids=self.state['dim'],
activation=prefix_lookup(self.state, self.prefix, 'activ'),
bias_scale=self.state['bias'],
init_fn=(self.state['rec_weight_init_fn']
if not self.skip_init
else "sample_zeros"),
scale=prefix_lookup(self.state, self.prefix, 'rec_weight_scale'),
weight_noise=self.state['weight_noise_rec'],
dropout=self.state['dropout_rec'],
gating=prefix_lookup(self.state, self.prefix, 'enc_rec_gating'),
gater_activation=prefix_lookup(self.state, self.prefix, 'enc_rec_gater'),
reseting=prefix_lookup(self.state, self.prefix, 'enc_rec_reseting'),
reseter_activation=prefix_lookup(self.state, self.prefix, 'enc_rec_reseter'),
name='{}_transition_{}'.format(self.prefix, level),
**add_args))
def _create_embedding_layers(self):
logger.debug("_create_embedding_layers")
self.approx_embedder = MultiLayer(
#word embedder is a MLP that maps words of n_sym dimensions to a vector of rank_n_approx dimensions
self.rng,
n_in=self.state['n_sym_source']
if self.prefix.find("enc") >= 0
else self.state['n_sym_target'],
n_hids=[self.state['rank_n_approx']],
activation=[self.state['rank_n_activ']],
name='{}_approx_embdr'.format(self.prefix),
**self.default_kwargs)
# We have 3 embeddings for each word in each level,
# input_embedder - the one used as input,
# reset_embedder - the one used to control resetting gate,
# update_embedder - the one used to control update gate.
self.input_embedders = [lambda x : 0] * self.num_levels
self.reset_embedders = [lambda x : 0] * self.num_levels
self.update_embedders = [lambda x : 0] * self.num_levels
embedder_kwargs = dict(self.default_kwargs)
embedder_kwargs.update(dict(
n_in=self.state['rank_n_approx'],
n_hids=[self.state['dim'] * self.state['dim_mult']],
activation=['lambda x:x']))
for level in range(self.num_levels):
self.input_embedders[level] = MultiLayer(
self.rng,
name='{}_input_embdr_{}'.format(self.prefix, level),
**embedder_kwargs)
if prefix_lookup(self.state, self.prefix, 'rec_gating'):
self.update_embedders[level] = MultiLayer(
self.rng,
learn_bias=False,
name='{}_update_embdr_{}'.format(self.prefix, level),
**embedder_kwargs)
if prefix_lookup(self.state, self.prefix, 'rec_reseting'):
self.reset_embedders[level] = MultiLayer(
self.rng,
learn_bias=False,
name='{}_reset_embdr_{}'.format(self.prefix, level),
**embedder_kwargs)
def _create_initialization_layers(self):
logger.debug("_create_initialization_layers")
self.initializers = [ZeroLayer()] * self.num_levels
if self.state['bias_code']:
for level in range(self.num_levels):
self.initializers[level] = MultiLayer(
self.rng,
n_in=self.state['dim'],
n_hids=[self.state['dim'] * self.state['hid_mult']],
activation=[prefix_lookup(self.state, 'dec', 'activ')],
bias_scale=[self.state['bias']],
name='{}_initializer_{}'.format(self.prefix, level),
**self.default_kwargs)
def _create_inter_level_layers(self):
logger.debug("_create_inter_level_layers")
inter_level_kwargs = dict(self.default_kwargs)
inter_level_kwargs.update(
n_in=self.state['dim'],
n_hids=self.state['dim'] * self.state['dim_mult'],
activation=['lambda x:x'])
self.inputers = [0] * self.num_levels
self.reseters = [0] * self.num_levels
self.updaters = [0] * self.num_levels
for level in range(1, self.num_levels):
self.inputers[level] = MultiLayer(self.rng,
name="{}_inputer_{}".format(self.prefix, level),
**inter_level_kwargs)
if prefix_lookup(self.state, self.prefix, 'rec_reseting'):
self.reseters[level] = MultiLayer(self.rng,
name="{}_reseter_{}".format(self.prefix, level),
**inter_level_kwargs)
if prefix_lookup(self.state, self.prefix, 'rec_gating'):
self.updaters[level] = MultiLayer(self.rng,
name="{}_updater_{}".format(self.prefix, level),
**inter_level_kwargs)