def apply(self, is_train, x, mask=None):
return fully_connected(x,
x.shape.as_list()[-1],
use_bias=self.bias,
activation=activations.get(self.activation),
kernel_initializer=_wrap_init(
initializers.get(self.w_init)))
def get_keras_activation(name):
return activations.get(name)
syll_embedder = tf.get_variable('syll_embedder', (syll_size, syll_dim))
syll_embed = tf.nn.embedding_lookup(syll_embedder, sylls)
from core_layer import han1_syll_cnn_char_rnn, han1_syll_cnn_char_cnn
core_layer_output = han1_syll_cnn_char_cnn(config, word_embed, sent_len,
char_embed, word_len,
syll_embed, None, fc_dim,
is_training)
with tf.variable_scope("output"):
output = fully_connected(
core_layer_output,
fc_dim,
use_bias=True,
activation=activations.get("relu"),
kernel_initializer=initializers.get("glorot_uniform"))
output = layers.dropout(output,
keep_prob=config.keep_prob,
is_training=is_training)
output = fully_connected(
output,
1,
use_bias=True,
activation=None,
kernel_initializer=initializers.get("glorot_uniform"))
y_logits = tf.sigmoid(output) * 9 + 1
predictions = y_logits
acc = tf.reduce_mean(
tf.to_float(tf.equal(tf.round(predictions), tf.round(y_))))
def __init__(self,
stack_sizes,
R_stack_sizes,
A_filt_sizes,
Ahat_filt_sizes,
R_filt_sizes,
pixel_max=1.,
error_activation='relu',
A_activation='relu',
LSTM_activation='tanh',
LSTM_inner_activation='hard_sigmoid',
output_mode='error',
extrap_start_time=None,
use_roi_loss=False,
threshold=None,
**kwargs):
self.stack_sizes = stack_sizes # output_dim for each layer
self.nb_layers = len(stack_sizes) # layer num
assert len(
R_stack_sizes
) == self.nb_layers, 'len(R_stack_sizes) must equal len(stack_sizes)'
self.R_stack_sizes = R_stack_sizes # R state dim
assert len(A_filt_sizes) == (
self.nb_layers -
1), 'len(A_filt_sizes) must equal len(stack_sizes) - 1'
self.A_filt_sizes = A_filt_sizes # A2E filter
assert len(
Ahat_filt_sizes
) == self.nb_layers, 'len(Ahat_filt_sizes) must equal len(stack_sizes)'
self.Ahat_filt_sizes = Ahat_filt_sizes # Ahat2E filter
assert len(R_filt_sizes) == (
self.nb_layers), 'len(R_filt_sizes) must equal len(stack_sizes)'
self.R_filt_sizes = R_filt_sizes # R_l+1, El 2 R_l filter
self.extrap_start_time = extrap_start_time
self.use_roi_loss = use_roi_loss
self.threshold = threshold
self.pixel_max = pixel_max
self.error_activation = activations.get(error_activation)
self.A_activation = activations.get(A_activation)
self.LSTM_activation = activations.get(LSTM_activation)
self.LSTM_inner_activation = activations.get(LSTM_inner_activation)
default_output_modes = ['prediction', 'error', 'all']
layer_output_modes = [
layer + str(n) for n in range(self.nb_layers)
for layer in ['R', 'E', 'A', 'Ahat']
]
assert output_mode in default_output_modes + layer_output_modes, 'Invalid output_mode: ' + str(
output_mode)
self.output_mode = output_mode
if self.output_mode in layer_output_modes:
self.output_layer_type = self.output_mode[:-1]
self.output_layer_num = int(self.output_mode[-1])
else:
self.output_layer_type = None
self.output_layer_num = None
# self.dim_ordering = 'tf'
self.channel_axis = -1
self.row_axis = -3
self.column_axis = -2
super(PredNet, self).__init__(**kwargs)
self.input_spec = [InputSpec(ndim=5)
] # (batch_num, time_step, row, column, channel)
name='cx_')
sx_ = tf.placeholder(tf.int32, (None, max_word_num, max_syll_num),
name='sx_')
y_ = tf.placeholder(tf.int32, (None), name='y_')
c_embed = tf.get_variable('c_embed', (character_size, char_dim))
s_embed = tf.get_variable('s_embed', (syllable_size, syll_dim))
cx = tf.nn.embedding_lookup(c_embed, cx_)
sx = tf.nn.embedding_lookup(s_embed, sx_)
core_output = cnn_char_syll(config, wx, cx, sx, is_training)
preds = fully_connected(
core_output,
10,
activation=activations.get('relu'),
kernel_initializer=initializers.get('glorot_uniform'))
pred = tf.argmax(preds, axis=1, output_type=tf.int32) + 1
y_arr = tf.one_hot(y_, 10)
acc = tf.reduce_mean(tf.to_float(tf.equal(pred, y_)))
loss = tf.losses.mean_squared_error(y_arr, preds)
mse = tf.losses.mean_squared_error(y_, pred)
train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)
##############################################################################################################
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
def rr_han(config, word_embed, sent_len, char_embed, word_len, syll_embed,
syll_len, n_unit, is_training):
'''
HAN 1 layer with char rnn
@ Input spec
word_embed [batch_size, max_sent_len, word_dim]
sent_len [batch_size]
char_embed [batch_size, max_sent_len, max_word_len, char_dim]
word_len [batch_size, max_sent_len]
syll_embed [batch_size, max_sent_len, max_syll_len, syll_dim]
syll_len [batch_size, max_sent_len]
@ Output spec
return [batch, n_unit]
'''
char_dim = config.char_dim
syll_dim = config.syll_dim
max_sent_len = config.max_sentence_length
max_word_len = config.max_word_length
max_syll_num = config.max_syll_num
keep_prob = config.keep_prob
rnn_dim = config.rnn_dim
with tf.variable_scope('syll_rnn') as scope:
cell_stack_count = 2
syll_cell = MultiRNNCell([GRUCell(syll_dim)] * cell_stack_count)
syll_embed = tf.cast(
tf.reshape(syll_embed, [-1, max_syll_num, syll_dim]), tf.float32)
syll_len = tf.reshape(syll_len, [-1])
_, syll_rnn_embed = bidirectional_rnn(syll_cell,
syll_cell,
syll_embed,
syll_len,
scope=scope)
syll_rnn_embed = tf.reshape(
syll_rnn_embed,
[-1, max_sent_len, syll_dim * 2 * cell_stack_count])
with tf.variable_scope('char_rnn') as scope:
cell_stack_count = 2
char_cell = MultiRNNCell([GRUCell(char_dim)] * cell_stack_count)
char_embed = tf.cast(
tf.reshape(char_embed, [-1, max_word_len, char_dim]), tf.float32)
word_len = tf.reshape(word_len, [-1])
_, char_rnn_embed = bidirectional_rnn(char_cell,
char_cell,
char_embed,
word_len,
scope=scope)
char_rnn_embed = tf.reshape(
char_rnn_embed,
[-1, max_sent_len, char_dim * 2 * cell_stack_count])
word_char_concat = tf.concat([word_embed, char_rnn_embed, syll_rnn_embed],
axis=2)
with tf.variable_scope('embedding') as scope:
word_char_embed = fully_connected(
word_char_concat,
rnn_dim,
use_bias=True,
activation=activations.get("relu"),
kernel_initializer=initializers.get("glorot_uniform"))
with tf.variable_scope('dropout'):
word_char_embed = layers.dropout(
word_char_embed,
keep_prob=keep_prob,
is_training=is_training,
)
with tf.variable_scope('encoder') as scope:
cell = MultiRNNCell([GRUCell(rnn_dim)] * 3)
encoder_output, _ = bidirectional_rnn(cell,
cell,
word_char_embed,
sent_len,
scope=scope)
with tf.variable_scope('attention') as scope:
attn_sum_output = task_specific_attention(encoder_output,
n_unit,
scope=scope)
with tf.variable_scope('dropout'):
attn_sum_output = layers.dropout(
attn_sum_output,
keep_prob=keep_prob,
is_training=is_training,
)
return attn_sum_output
