def build_ff_neural_net(nn_input,
n_inputs,
hidden_layers,
nonlinearity,
scope_name,
variable_name,
collect_summary,
logit_weights=None,
initializer=layers.xavier_initializer(),
dropout=False):
assert len(hidden_layers) == len(nonlinearity)
name_scope = '%s/%s' % (scope_name, variable_name)
h = nn_input
n_hiddens = n_inputs
n_hiddens_next = hidden_layers[0]
for i in range(len(hidden_layers)):
w = get_scope_variable(scope_name,
"%s/layer%d/weights" %
(variable_name, i),
shape=(n_hiddens, n_hiddens_next),
initializer=initializer)
b = get_scope_variable(scope_name,
"%s/layer%d/biases" %
(variable_name, i),
shape=(n_hiddens_next),
initializer=initializer)
if collect_summary:
with tf.name_scope(name_scope + '/layer%d' % i):
with tf.name_scope('weights'):
variable_summaries(w)
with tf.name_scope('biases'):
variable_summaries(b)
with tf.name_scope('Wx_plus_b'):
pre_h = tf.matmul(h, w) + b
# Yunfei: dropout option is useless now
if dropout:
# if i == 0:
# pre_h = tf.nn.dropout(tf.matmul(h,w), keep_prob=0.8) + b
# else:
pre_h = tf.nn.dropout(tf.matmul(h, w),
keep_prob=dropout) + b
tf.summary.histogram('pre_activations', pre_h)
h = nonlinearity[i](pre_h, name='activation')
tf.summary.histogram('activations', h)
else:
pre_h = tf.matmul(h, w) + b
h = nonlinearity[i](pre_h, name='activation')
n_hiddens = hidden_layers[i]
if i + 1 < len(hidden_layers):
n_hiddens_next = hidden_layers[i + 1]
if logit_weights is not None and i == len(hidden_layers) - 2:
h *= logit_weights
return h
def get_regularizer_loss(scope_name, variable_name):
if params['dynamics_model']['regularization']['method'] in [
None, ''
]:
return tf.constant(0.0, dtype=tf.float32)
constant = params['dynamics_model']['regularization']['constant']
regularizer = eval(
params['dynamics_model']['regularization']['method'])
hidden_layers = params['dynamics_model']['hidden_layers']
reg_loss = 0.0
for i in range(len(hidden_layers) + 1):
w = get_scope_variable(
scope_name, "%s/layer%d/weights" % (variable_name, i))
b = get_scope_variable(
scope_name, "%s/layer%d/biases" % (variable_name, i))
reg_loss += regularizer(w) + regularizer(b)
return constant * reg_loss
def HMN_Batch(U, h_i, u_s_i, u_e_i, batch_size, scope = None, FLAGS = None, dropout_rate = 1, iter_number=-1):
#U = tf.transpose(U_T)
maxout_pooling_size = FLAGS.maxout_pooling_size
#batch_size = FLAGS.train_batch_size
max_sequence_length = FLAGS.max_sequence_length
lstm_size = FLAGS.lstm_size
scope = scope or tf.get_variable_scope()
b_1 = ut.get_scope_variable(scope, 'hmn_b_1', [1, 1, lstm_size, maxout_pooling_size])
B_1 = tf.tile(b_1, [batch_size, max_sequence_length, 1, 1])
w_1 = ut.get_scope_variable(scope, 'hmn_w_1', [maxout_pooling_size, lstm_size, 3 * lstm_size])
w_d = ut.get_scope_variable(scope, 'hmn_w_d', [5*lstm_size, lstm_size])
b_2 = ut.get_scope_variable(scope, 'hmn_b_2', [1, 1, maxout_pooling_size, lstm_size])
B_2 = tf.tile(b_2, [batch_size, max_sequence_length, 1, 1])
w_2 = ut.get_scope_variable(scope, 'hmn_w_2', [maxout_pooling_size, lstm_size, lstm_size])
b_3 = ut.get_scope_variable(scope, 'hmn_b_3', [1, 1, maxout_pooling_size])
B_3 = tf.tile(b_3, [batch_size, max_sequence_length, 1])
w_3 = ut.get_scope_variable(scope, 'hmn_w_3', [maxout_pooling_size, 2 * lstm_size])
#if iter_number == 4:
# tf.summary.histogram(scope + '/hmn_b_1', b_1)
# tf.summary.histogram(scope + '/hmn_w_1', w_1)
# tf.summary.histogram(scope + '/hmn_w_d', w_d)
# tf.summary.histogram(scope + '/hmn_w_3', w_3)
# r is shape of (B, L)
r = tf.tanh(tf.matmul(tf.concat([h_i, u_s_i, u_e_i], axis = 1), w_d));
r = tf.reshape(r, [batch_size, 1, lstm_size])
# R is shape (B, D, L)
R = tf.tile(r, [1, max_sequence_length, 1])
# m_1_1 is shape of (B, D, L, p)
dropout_input_m_1 = tf.nn.dropout(tf.concat([U, R], axis = 2), dropout_rate)
m_1_1 = tf.tensordot(dropout_input_m_1, tf.transpose(w_1), axes=[[2], [0]])
# m_1_2 is shape of (B, D, L, p)
m_1_2 = tf.add(m_1_1, B_1)
# m_1 is shape of (B, D, L)
m_1 = maxout(m_1_2, axis = 3 )
# m_2_1 is shape of (B, D, p, L)
dropout_input_m_2 = tf.nn.dropout(m_1, dropout_rate)
m_2_1 = tf.tensordot(dropout_input_m_2, w_2, axes=[[2], [2]])
m_2_2 = tf.add(m_2_1, B_2)
# m_2 is shape of (B, D, L)
m_2 = maxout(m_2_2, axis = 2 )
# m_3_1 is shape of (B, D, p)
dropout_input_m_3 = tf.nn.dropout(tf.concat([m_1, m_2], axis = 2), dropout_rate)
m_3_1 = tf.tensordot(dropout_input_m_3, w_3, axes=[[2], [1]])
m_3_2 = tf.add(m_3_1, B_3)
m_3 = maxout(m_3_2, axis = 2)
if iter_number == 3:
tf.summary.histogram(scope + '/hmn_b_1', b_1)
tf.summary.histogram(scope + '/hmn_w_1', w_1)
tf.summary.histogram(scope + '/hmn_w_d', w_d)
tf.summary.histogram(scope + '/hmn_w_3', w_3)
tf.summary.histogram(scope + '/hmn_m_3', m_3)
return m_3