def __call__(self, inputs, state, scope=None):
""""""
with tf.variable_scope(scope or type(self).__name__):
cell_tm1, hidden_tm1 = tf.split(state, 2, axis=1)
input_list = [inputs, hidden_tm1]
with tf.variable_scope('Gates'):
gates = linear(inputs_list,
self.output_size,
add_bias=True,
n_splits=2,
moving_params=self.moving_params)
update_act, reset_act = gates
update_gate = gate(update_act - self.forget_bias)
reset_gate = gate(reset_act)
reset_state = reset_gate * hidden_tm1
input_list = [inputs, reset_state]
with tf.variable_scope('Candidate'):
hidden_act = linear(input_list,
self.output_size,
add_bias=True,
moving_params=self.moving_params)
hidden_tilde = self.recur_func(hidden_act)
cell_t = update_gate * cell_tm1 + (1 - update_gate) * hidden_tilde
return cell_t, tf.concat([cell_t, cell_t], 1)
def linear(self, inputs, output_size, keep_prob=None, n_splits=1, add_bias=True, initializer=tf.zeros_initializer):
""""""
print("===nn.py: START linear", ' name=', self.name)
#print ("nn.py,linear,keep:{}".format(keep_prob))
if isinstance(inputs, (list, tuple)):
n_dims = len(inputs[0].get_shape().as_list())
inputs = tf.concat(inputs, n_dims-1)
else:
n_dims = len(inputs.get_shape().as_list())
input_size = inputs.get_shape().as_list()[-1]
if self.moving_params is None:
keep_prob = keep_prob or self.mlp_keep_prob
else:
keep_prob = 1
if keep_prob < 1:
noise_shape = tf.stack([self.batch_size] + [1]*(n_dims-2) + [input_size])
inputs = tf.nn.dropout(inputs, keep_prob, noise_shape=noise_shape)
lin = linalg.linear(inputs,
output_size,
n_splits=n_splits,
add_bias=add_bias,
initializer=initializer,
moving_params=self.moving_params)
if output_size == 1:
if isinstance(lin, list):
lin = [tf.squeeze(x, axis=(n_dims-1)) for x in lin]
else:
lin = tf.squeeze(lin, axis=(n_dims-1))
print("===nn.py: END linear", ' name=', self.name, ' input_size=',input_size, ' output_size=', output_size, ' keep prob=',keep_prob)
return lin
def __call__(self, placeholder=None, moving_params=None):
""""""
print('===tagrep_vocab.py: START Call ', ' name=', self.name,
' embed_size=', self._embed_size)
embeddings = super(TagRepVocab,
self).__call__(placeholder,
moving_params=moving_params)
#tf.summary.histogram('tagreph',embeddings)
# (n x b x d') -> (n x b x d)
with tf.variable_scope(self.name.title()):
matrix = linalg.linear(embeddings,
self._embed_size,
moving_params=moving_params)
if moving_params is None:
with tf.variable_scope('Linear', reuse=True):
weights = tf.get_variable('Weights')
tf.losses.add_loss(
tf.nn.l2_loss(
tf.matmul(tf.transpose(weights), weights) -
tf.eye(self._embed_size)))
print('===tagrep_vocab.py: END Call ', ' name=', self.name)
return matrix
def __call__(self, inputs, state, scope=None):
""""""
print('===lstm_cell.py: Call for LSTMCell', ' name=', self.name)
with tf.variable_scope(scope or type(self).__name__):
cell_tm1, hidden_tm1 = tf.split(state, 2, axis=1)
input_list = [inputs, hidden_tm1]
lin = linear(input_list,
self.output_size,
add_bias=True,
n_splits=4,
moving_params=self.moving_params)
cell_act, input_act, forget_act, output_act = lin
cell_tilde_t = tanh(cell_act)
input_gate = gate(input_act)
forget_gate = gate(forget_act - self.forget_bias)
output_gate = gate(output_act)
cell_t = input_gate * cell_tilde_t + (1 - forget_gate) * cell_tm1
hidden_tilde_t = self.recur_func(cell_t)
hidden_t = hidden_tilde_t * output_gate
print("===lstm_cell.py: END Call for LSTMCell", " hidden_t_size=",
hidden_t.get_shape().as_list()[-1], " state_size=",
tf.concat([cell_t, hidden_t], 1).get_shape())
return hidden_t, tf.concat([cell_t, hidden_t], 1)
def __call__(self, inputs, state, scope=None):
""""""
with tf.variable_scope(scope or type(self).__name__):
inputs_list = [inputs, state]
hidden_act = linear(inputs_list,
self.output_size,
add_bias=True,
moving_params=self.moving_params)
hidden = self.recur_func(hidden_act)
return hidden, hidden
def __call__(self, placeholder=None, moving_params=None):
""""""
embeddings = super(PretrainedVocab,
self).__call__(placeholder,
moving_params=moving_params)
# (n x b x d') -> (n x b x d)
with tf.variable_scope(self.name.title()):
matrix = linalg.linear(embeddings,
self.token_embed_size,
moving_params=moving_params)
if moving_params is None:
with tf.variable_scope('Linear', reuse=True):
weights = tf.get_variable('Weights')
tf.losses.add_loss(
tf.nn.l2_loss(
tf.matmul(tf.transpose(weights), weights) -
tf.eye(self.token_embed_size)))
return matrix
def __call__(self, inputs, state, scope=None):
""""""
with tf.variable_scope(scope or type(self).__name__):
cell_tm1, hidden_tm1 = tf.split(state, 2, axis=1)
input_list = [inputs, hidden_tm1]
lin = linear(input_list,
self.output_size,
add_bias=True,
n_splits=3,
moving_params=self.moving_params)
cell_act, update_act, output_act = lin
cell_tilde_t = cell_act
update_gate = gate(update_act - self.forget_bias)
output_gate = gate(output_act)
cell_t = update_gate * cell_tilde_t + (1 - update_gate) * cell_tm1
hidden_tilde_t = self.recur_func(cell_t)
hidden_t = hidden_tilde_t * output_gate
return hidden_t, tf.concat([cell_t, hidden_t], 1)
