def _create_decoding_layers(self): logger.debug("_create_decoding_layers") self.decode_inputers = [lambda x : 0] * self.num_levels self.decode_reseters = [lambda x : 0] * self.num_levels self.decode_updaters = [lambda x : 0] * self.num_levels self.back_decode_inputers = [lambda x : 0] * self.num_levels self.back_decode_reseters = [lambda x : 0] * self.num_levels self.back_decode_updaters = [lambda x : 0] * self.num_levels decoding_kwargs = dict(self.default_kwargs) decoding_kwargs.update(dict( n_in=self.state['c_dim'], n_hids=self.state['dim'] * self.state['dim_mult'], activation=['lambda x:x'], learn_bias=False)) if self.state['decoding_inputs']: for level in range(self.num_levels): # Input contributions self.decode_inputers[level] = MultiLayer( self.rng, name='{}_dec_inputter_{}'.format(self.prefix, level), **decoding_kwargs) # Update gate contributions if prefix_lookup(self.state, 'dec', 'rec_gating'): self.decode_updaters[level] = MultiLayer( self.rng, name='{}_dec_updater_{}'.format(self.prefix, level), **decoding_kwargs) # Reset gate contributions if prefix_lookup(self.state, 'dec', 'rec_reseting'): self.decode_reseters[level] = MultiLayer( self.rng, name='{}_dec_reseter_{}'.format(self.prefix, level), **decoding_kwargs)
def _create_transition_layers(self): logger.debug("_create_transition_layers") self.transitions = [] rec_layer = eval(prefix_lookup(self.state, self.prefix, 'dec_rec_layer')) add_args = dict() if rec_layer == RecurrentLayerWithSearch:#if the decoder uses search RNN, then there are multiple contexts, the real dimension of c is times of hid num add_args = dict(c_dim=self.state['c_dim']) for level in range(self.num_levels): self.transitions.append(rec_layer(#create a recurrent hidden layer for each level (there could be multiple hidden layers). self.rng, n_hids=self.state['dim'], activation=prefix_lookup(self.state, self.prefix, 'activ'), bias_scale=self.state['bias'], init_fn=(self.state['rec_weight_init_fn'] if not self.skip_init else "sample_zeros"), scale=prefix_lookup(self.state, self.prefix, 'rec_weight_scale'), weight_noise=self.state['weight_noise_rec'], dropout=self.state['dropout_rec'], gating=prefix_lookup(self.state, self.prefix, 'dec_rec_gating'), gater_activation=prefix_lookup(self.state, self.prefix, 'dec_rec_gater'), reseting=prefix_lookup(self.state, self.prefix, 'dec_rec_reseting'), reseter_activation=prefix_lookup(self.state, self.prefix, 'dec_rec_reseter'), name='{}_transition_{}'.format(self.prefix, level), **add_args))
def _create_transition_layers(self): logger.debug("_create_transition_layers") self.transitions = [] rec_layer = eval(prefix_lookup(self.state, self.prefix, 'enc_rec_layer')) add_args = dict() if rec_layer == RecurrentLayerWithSearch: add_args = dict(c_dim=self.state['c_dim']) for level in range(self.num_levels): self.transitions.append(rec_layer( self.rng, n_hids=self.state['dim'], activation=prefix_lookup(self.state, self.prefix, 'activ'), bias_scale=self.state['bias'], init_fn=(self.state['rec_weight_init_fn'] if not self.skip_init else "sample_zeros"), scale=prefix_lookup(self.state, self.prefix, 'rec_weight_scale'), weight_noise=self.state['weight_noise_rec'], dropout=self.state['dropout_rec'], gating=prefix_lookup(self.state, self.prefix, 'enc_rec_gating'), gater_activation=prefix_lookup(self.state, self.prefix, 'enc_rec_gater'), reseting=prefix_lookup(self.state, self.prefix, 'enc_rec_reseting'), reseter_activation=prefix_lookup(self.state, self.prefix, 'enc_rec_reseter'), name='{}_transition_{}'.format(self.prefix, level), **add_args))
def _create_embedding_layers(self): logger.debug("_create_embedding_layers") self.approx_embedder = MultiLayer( #word embedder is a MLP that maps words of n_sym dimensions to a vector of rank_n_approx dimensions self.rng, n_in=self.state['n_sym_source'] if self.prefix.find("enc") >= 0 else self.state['n_sym_target'], n_hids=[self.state['rank_n_approx']], activation=[self.state['rank_n_activ']], name='{}_approx_embdr'.format(self.prefix), **self.default_kwargs) # We have 3 embeddings for each word in each level, # input_embedder - the one used as input, # reset_embedder - the one used to control resetting gate, # update_embedder - the one used to control update gate. self.input_embedders = [lambda x : 0] * self.num_levels self.reset_embedders = [lambda x : 0] * self.num_levels self.update_embedders = [lambda x : 0] * self.num_levels embedder_kwargs = dict(self.default_kwargs) embedder_kwargs.update(dict( n_in=self.state['rank_n_approx'], n_hids=[self.state['dim'] * self.state['dim_mult']], activation=['lambda x:x'])) for level in range(self.num_levels): self.input_embedders[level] = MultiLayer( self.rng, name='{}_input_embdr_{}'.format(self.prefix, level), **embedder_kwargs) if prefix_lookup(self.state, self.prefix, 'rec_gating'): self.update_embedders[level] = MultiLayer( self.rng, learn_bias=False, name='{}_update_embdr_{}'.format(self.prefix, level), **embedder_kwargs) if prefix_lookup(self.state, self.prefix, 'rec_reseting'): self.reset_embedders[level] = MultiLayer( self.rng, learn_bias=False, name='{}_reset_embdr_{}'.format(self.prefix, level), **embedder_kwargs)
def _create_initialization_layers(self): logger.debug("_create_initialization_layers") self.initializers = [ZeroLayer()] * self.num_levels if self.state['bias_code']: for level in range(self.num_levels): self.initializers[level] = MultiLayer( self.rng, n_in=self.state['dim'], n_hids=[self.state['dim'] * self.state['hid_mult']], activation=[prefix_lookup(self.state, 'dec', 'activ')], bias_scale=[self.state['bias']], name='{}_initializer_{}'.format(self.prefix, level), **self.default_kwargs)
def _create_inter_level_layers(self): logger.debug("_create_inter_level_layers") inter_level_kwargs = dict(self.default_kwargs) inter_level_kwargs.update( n_in=self.state['dim'], n_hids=self.state['dim'] * self.state['dim_mult'], activation=['lambda x:x']) self.inputers = [0] * self.num_levels self.reseters = [0] * self.num_levels self.updaters = [0] * self.num_levels for level in range(1, self.num_levels): self.inputers[level] = MultiLayer(self.rng, name="{}_inputer_{}".format(self.prefix, level), **inter_level_kwargs) if prefix_lookup(self.state, self.prefix, 'rec_reseting'): self.reseters[level] = MultiLayer(self.rng, name="{}_reseter_{}".format(self.prefix, level), **inter_level_kwargs) if prefix_lookup(self.state, self.prefix, 'rec_gating'): self.updaters[level] = MultiLayer(self.rng, name="{}_updater_{}".format(self.prefix, level), **inter_level_kwargs)