def apply(self, is_train, x, mask=None): return fully_connected(x, x.shape.as_list()[-1], use_bias=self.bias, activation=activations.get(self.activation), kernel_initializer=_wrap_init( initializers.get(self.w_init)))
def get_keras_activation(name): return activations.get(name)
syll_embedder = tf.get_variable('syll_embedder', (syll_size, syll_dim)) syll_embed = tf.nn.embedding_lookup(syll_embedder, sylls) from core_layer import han1_syll_cnn_char_rnn, han1_syll_cnn_char_cnn core_layer_output = han1_syll_cnn_char_cnn(config, word_embed, sent_len, char_embed, word_len, syll_embed, None, fc_dim, is_training) with tf.variable_scope("output"): output = fully_connected( core_layer_output, fc_dim, use_bias=True, activation=activations.get("relu"), kernel_initializer=initializers.get("glorot_uniform")) output = layers.dropout(output, keep_prob=config.keep_prob, is_training=is_training) output = fully_connected( output, 1, use_bias=True, activation=None, kernel_initializer=initializers.get("glorot_uniform")) y_logits = tf.sigmoid(output) * 9 + 1 predictions = y_logits acc = tf.reduce_mean( tf.to_float(tf.equal(tf.round(predictions), tf.round(y_))))
def __init__(self, stack_sizes, R_stack_sizes, A_filt_sizes, Ahat_filt_sizes, R_filt_sizes, pixel_max=1., error_activation='relu', A_activation='relu', LSTM_activation='tanh', LSTM_inner_activation='hard_sigmoid', output_mode='error', extrap_start_time=None, use_roi_loss=False, threshold=None, **kwargs): self.stack_sizes = stack_sizes # output_dim for each layer self.nb_layers = len(stack_sizes) # layer num assert len( R_stack_sizes ) == self.nb_layers, 'len(R_stack_sizes) must equal len(stack_sizes)' self.R_stack_sizes = R_stack_sizes # R state dim assert len(A_filt_sizes) == ( self.nb_layers - 1), 'len(A_filt_sizes) must equal len(stack_sizes) - 1' self.A_filt_sizes = A_filt_sizes # A2E filter assert len( Ahat_filt_sizes ) == self.nb_layers, 'len(Ahat_filt_sizes) must equal len(stack_sizes)' self.Ahat_filt_sizes = Ahat_filt_sizes # Ahat2E filter assert len(R_filt_sizes) == ( self.nb_layers), 'len(R_filt_sizes) must equal len(stack_sizes)' self.R_filt_sizes = R_filt_sizes # R_l+1, El 2 R_l filter self.extrap_start_time = extrap_start_time self.use_roi_loss = use_roi_loss self.threshold = threshold self.pixel_max = pixel_max self.error_activation = activations.get(error_activation) self.A_activation = activations.get(A_activation) self.LSTM_activation = activations.get(LSTM_activation) self.LSTM_inner_activation = activations.get(LSTM_inner_activation) default_output_modes = ['prediction', 'error', 'all'] layer_output_modes = [ layer + str(n) for n in range(self.nb_layers) for layer in ['R', 'E', 'A', 'Ahat'] ] assert output_mode in default_output_modes + layer_output_modes, 'Invalid output_mode: ' + str( output_mode) self.output_mode = output_mode if self.output_mode in layer_output_modes: self.output_layer_type = self.output_mode[:-1] self.output_layer_num = int(self.output_mode[-1]) else: self.output_layer_type = None self.output_layer_num = None # self.dim_ordering = 'tf' self.channel_axis = -1 self.row_axis = -3 self.column_axis = -2 super(PredNet, self).__init__(**kwargs) self.input_spec = [InputSpec(ndim=5) ] # (batch_num, time_step, row, column, channel)
name='cx_') sx_ = tf.placeholder(tf.int32, (None, max_word_num, max_syll_num), name='sx_') y_ = tf.placeholder(tf.int32, (None), name='y_') c_embed = tf.get_variable('c_embed', (character_size, char_dim)) s_embed = tf.get_variable('s_embed', (syllable_size, syll_dim)) cx = tf.nn.embedding_lookup(c_embed, cx_) sx = tf.nn.embedding_lookup(s_embed, sx_) core_output = cnn_char_syll(config, wx, cx, sx, is_training) preds = fully_connected( core_output, 10, activation=activations.get('relu'), kernel_initializer=initializers.get('glorot_uniform')) pred = tf.argmax(preds, axis=1, output_type=tf.int32) + 1 y_arr = tf.one_hot(y_, 10) acc = tf.reduce_mean(tf.to_float(tf.equal(pred, y_))) loss = tf.losses.mean_squared_error(y_arr, preds) mse = tf.losses.mean_squared_error(y_, pred) train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss) ############################################################################################################## sess = tf.InteractiveSession() tf.global_variables_initializer().run()
def rr_han(config, word_embed, sent_len, char_embed, word_len, syll_embed, syll_len, n_unit, is_training): ''' HAN 1 layer with char rnn @ Input spec word_embed [batch_size, max_sent_len, word_dim] sent_len [batch_size] char_embed [batch_size, max_sent_len, max_word_len, char_dim] word_len [batch_size, max_sent_len] syll_embed [batch_size, max_sent_len, max_syll_len, syll_dim] syll_len [batch_size, max_sent_len] @ Output spec return [batch, n_unit] ''' char_dim = config.char_dim syll_dim = config.syll_dim max_sent_len = config.max_sentence_length max_word_len = config.max_word_length max_syll_num = config.max_syll_num keep_prob = config.keep_prob rnn_dim = config.rnn_dim with tf.variable_scope('syll_rnn') as scope: cell_stack_count = 2 syll_cell = MultiRNNCell([GRUCell(syll_dim)] * cell_stack_count) syll_embed = tf.cast( tf.reshape(syll_embed, [-1, max_syll_num, syll_dim]), tf.float32) syll_len = tf.reshape(syll_len, [-1]) _, syll_rnn_embed = bidirectional_rnn(syll_cell, syll_cell, syll_embed, syll_len, scope=scope) syll_rnn_embed = tf.reshape( syll_rnn_embed, [-1, max_sent_len, syll_dim * 2 * cell_stack_count]) with tf.variable_scope('char_rnn') as scope: cell_stack_count = 2 char_cell = MultiRNNCell([GRUCell(char_dim)] * cell_stack_count) char_embed = tf.cast( tf.reshape(char_embed, [-1, max_word_len, char_dim]), tf.float32) word_len = tf.reshape(word_len, [-1]) _, char_rnn_embed = bidirectional_rnn(char_cell, char_cell, char_embed, word_len, scope=scope) char_rnn_embed = tf.reshape( char_rnn_embed, [-1, max_sent_len, char_dim * 2 * cell_stack_count]) word_char_concat = tf.concat([word_embed, char_rnn_embed, syll_rnn_embed], axis=2) with tf.variable_scope('embedding') as scope: word_char_embed = fully_connected( word_char_concat, rnn_dim, use_bias=True, activation=activations.get("relu"), kernel_initializer=initializers.get("glorot_uniform")) with tf.variable_scope('dropout'): word_char_embed = layers.dropout( word_char_embed, keep_prob=keep_prob, is_training=is_training, ) with tf.variable_scope('encoder') as scope: cell = MultiRNNCell([GRUCell(rnn_dim)] * 3) encoder_output, _ = bidirectional_rnn(cell, cell, word_char_embed, sent_len, scope=scope) with tf.variable_scope('attention') as scope: attn_sum_output = task_specific_attention(encoder_output, n_unit, scope=scope) with tf.variable_scope('dropout'): attn_sum_output = layers.dropout( attn_sum_output, keep_prob=keep_prob, is_training=is_training, ) return attn_sum_output