def export_ops(self, name):
"""Exports ops to collections."""
self._name = name
ops = {util.with_prefix(self._name, "cost"): self._cost}
if self._is_training:
ops.update(lr=self._lr, new_lr=self._new_lr, lr_update=self._lr_update)
if self._rnn_params:
ops.update(rnn_params=self._rnn_params)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, "initial")
self._final_state_name = util.with_prefix(self._name, "final")
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def import_ops(self):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
rnn_params = tf.get_collection_ref("rnn_params")
if self._cell and rnn_params:
params_saveable = tf.contrib.cudnn_rnn.RNNParamsSaveable(
self._cell,
self._cell.params_to_canonical,
self._cell.canonical_to_params,
rnn_params,
base_variable_scope="Model/RNN")
tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS,
params_saveable)
self._cost = tf.get_collection_ref(util.with_prefix(
self._name, "cost"))[0]
num_replicas = 1
self._initial_state = util.import_state_tuples(
self._initial_state, self._initial_state_name, num_replicas)
self._final_state = util.import_state_tuples(self._final_state,
self._final_state_name,
num_replicas)
def export_ops(self, name):
"""Exports ops to collections."""
self._name = name
ops = {util.with_prefix(self._name, "cost"): self._cost}
if self._is_training:
ops.update(lr=self._lr,
new_lr=self._new_lr,
lr_update=self._lr_update)
if self._rnn_params:
ops.update(rnn_params=self._rnn_params)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, "initial")
self._final_state_name = util.with_prefix(self._name, "final")
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def import_ops(self):
"""Import ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref('train_op')[0]
self._lr = tf.get_collection_ref('lr')[0]
self._new_lr = tf.get_collection_ref('new_lr')[0]
self._lr_update = tf.get_collection_ref('lr_update')[0]
rnn_params = tf.get_collection_ref('rnn_params')
if self._cell and rnn_params:
params_saveable = tf.contrib.cudnn_rnn.RNNParamsSaveable(
self._cell,
self._cell.params_to_canonical,
self._cell.canonical_to_params,
rnn_params,
base_variable_scope='Model/RNN')
tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS,
params_saveable)
self._cost = tf.get_collection_ref(util.with_prefix(
self._name, 'cost'))[0]
num_replicas = FLAGS.num_gpus if self._name == 'Train' else 1
self._initial_state = util.import_state_tuples(
self._initial_state, self._initial_state_name, num_replicas)
self._final_state = util.import_state_tuples(self._final_state,
self._final_state_name,
num_replicas)
def import_ops(self):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
self._params_size = tf.get_collection_ref("params_size")[0]
if FLAGS.num_gpus:
self._memory_use = tf.get_collection_ref("memory_use")[0]
rnn_params = tf.get_collection_ref("rnn_params")
# if self._cell and rnn_params:
# params_saveable = tf.contrib.cudnn_rnn.RNNParamsSaveable(
# self._cell,
# self._cell.params_to_canonical,
# self._cell.canonical_to_params,
# rnn_params,
# base_variable_scope="Model/RNN")
# params_saveable = tf.contrib.cudnn_rnn.CudnnLSTMSaveable(
# rnn_params,
# self._cell.num_layers,
# self._cell.num_units,
# self._cell.input_size,
# self._cell.input_mode,
# self._cell.direction,
# scope="Model/RNN")
# tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, params_saveable)
self._cost = tf.get_collection_ref(util.with_prefix(
self._name, "cost"))[0]
num_replicas = FLAGS.num_gpus if self._name == "Train" else 1
self._initial_state = util.import_state_tuples(
self._initial_state, self._initial_state_name, num_replicas)
self._final_state = util.import_state_tuples(self._final_state,
self._final_state_name,
num_replicas)
def export_ops(self, name):
self._name = name
ops = {util.with_prefix(self._name, 'cost'): self._cost}
tf.add_to_collection(util.with_prefix(self._name, 'logits'),
self._logits)
if self._is_training:
ops.update(lr=self._lr,
new_lr=self._new_lr,
lr_update=self._lr_update)
if self._rnn_params:
ops.update(rnn_params=self._rnn_params)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, 'initial')
self._final_state_name = util.with_prefix(self._name, 'final')
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def export_ops(self, name):
"""Exports ops to collections."""
self._name = name
self._exported_ops = {}
ops = {}
ops[util.with_prefix(self._name, 'clfs_padd')] = self._padding
ops[util.with_prefix(self._name, 'last_step')] = self._last_step
for key, preds in self._predictions.items():
ops[util.with_prefix(self._name, key)] = preds
self._exported_ops.update({'predictions': [f'{self._name}/{k}' for k in self._predictions]})
for key, loss in self._losses.items():
ops[util.with_prefix(self._name, key)] = loss
for key, loss in self._l2_losses.items():
ops[util.with_prefix(self._name, key)] = loss
self._exported_ops.update({'losses': [f'{self._name}/{k}' for k in self._losses.keys()],
'l2_losses': [f'{self._name}/{k}' for k in self._l2_losses.keys()]})
ops.update({f'{name}/cur_epoch': self._cur_epoch,
f'{name}/global_step': self._global_step})
self._exported_ops.update({'epoch_and_step': [f'{name}/cur_epoch', f'{name}/global_step']})
if self._is_training:
for key, train_op in self._train_ops.items():
ops[key] = train_op
self._exported_ops.update({'train_ops': self._train_ops.keys()})
ops.update(lr=self._lr, new_lr=self._new_lr, lr_update=self._lr_update,
new_val=self._new_val, update_epoch=self._update_epoch)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, "initial")
self._final_state_name = util.with_prefix(self._name, "final")
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def import_ops(self):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
self._cost = tf.get_collection_ref(util.with_prefix(self._name, "cost"))[0]
self._final_state = util.import_state_tuples(self._final_state, self._final_state_name)
def export_ops(self, name):
"""Exports ops to collections."""
self._name = name
ops = {util.with_prefix(self._name, 'cost'): self._cost}
if self._is_training:
ops.update(lr=self._lr,
new_lr=self._new_lr,
lr_update=self._lr_update)
if self._rnn_params:
ops.update(rnn_params=self._rnn_params)
if self.model_type == 'test':
ops.update(logits=self.logits, y=self.y)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, 'initial')
self._final_state_name = util.with_prefix(self._name, 'final')
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def import_ops(self, num_gpus = 1):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
self._cost = tf.get_collection_ref(util.with_prefix(self._name, "cost"))[0]
self._kl_loss = tf.get_collection_ref(util.with_prefix(self._name, "kl_div"))[0]
self._input_data = tf.get_collection_ref(util.with_prefix(self._name, "input_data"))[0]
self._output = tf.get_collection_ref(util.with_prefix(self._name, "output"))[0]
self._targets = tf.get_collection_ref(util.with_prefix(self._name, "targets"))[0]
num_replicas = num_gpus if self._name == "Train" else 1
self._initial_state = util.import_state_tuples(
self._initial_state, self._initial_state_name, num_replicas)
self._final_state = util.import_state_tuples(
self._final_state, self._final_state_name, num_replicas)
def import_ops(self):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
rnn_params = tf.get_collection_ref("rnn_params")
if self._cell and rnn_params:
params_saveable = tf.contrib.cudnn_rnn.RNNParamsSaveable(
self._cell,
self._cell.params_to_canonical,
self._cell.canonical_to_params,
rnn_params,
base_variable_scope="Model/RNN")
tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, params_saveable)
self._cost = tf.get_collection_ref(util.with_prefix(self._name, "cost"))[0]
self._tranM = tf.get_collection_ref(util.with_prefix(self._name, "tranM"))[0]
self._logits = tf.get_collection_ref(util.with_prefix(self._name, "logits"))[0]
self._tar_test = tf.get_collection_ref(util.with_prefix(self._name, "tar_test"))[0]
self._wei_test = tf.get_collection_ref(util.with_prefix(self._name, "wei_test"))[0]
num_replicas = FLAGS.num_gpus if self._name == "Train" else 1
def export_ops(self, name):
"""Exports ops to collections."""
self._name = name
ops = {util.with_prefix(self._name, "cost"): self._cost,
util.with_prefix(self._name, "kl_div"): self._kl_loss,
util.with_prefix(self._name, "input_data"): self._input_data,
util.with_prefix(self._name, "output"): self._output,
util.with_prefix(self._name, "targets"): self._targets,
}
if self._is_training:
ops.update(lr=self._lr, new_lr=self._new_lr, lr_update=self._lr_update)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, "initial")
self._final_state_name = util.with_prefix(self._name, "final")
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def import_ops(self):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
rnn_params = tf.get_collection_ref("rnn_params")
if self._cell and rnn_params:
params_saveable = tf.contrib.cudnn_rnn.RNNParamsSaveable(
self._cell,
self._cell.params_to_canonical,
self._cell.canonical_to_params,
rnn_params,
base_variable_scope="Model/RNN")
tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, params_saveable)
self._cost = tf.get_collection_ref(util.with_prefix(self._name, "cost"))[0]
num_replicas = FLAGS.num_gpus if self._name == "Train" else 1
self._initial_state = util.import_state_tuples(
self._initial_state, self._initial_state_name, num_replicas)
self._final_state = util.import_state_tuples(
self._final_state, self._final_state_name, num_replicas)
def import_ops(self,config):
"""Imports ops from collections."""
if self._is_training:
self._train_op = tf.get_collection_ref("train_op")[0]
self._lr = tf.get_collection_ref("lr")[0]
self._new_lr = tf.get_collection_ref("new_lr")[0]
self._lr_update = tf.get_collection_ref("lr_update")[0]
rnn_params = tf.get_collection_ref("rnn_params")
if self._cell and rnn_params:
params_saveable = tf.contrib.cudnn_rnn.CudnnLSTMSaveable(
rnn_params,
config.num_layers,
config.hidden_size,
config.hidden_size,
scope="Model/RNN")
tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, params_saveable)
self._cost = tf.get_collection_ref(util.with_prefix(self._name, "cost"))[0]
num_replicas = FLAGS.num_gpus if self._name == "Train" else 1
self._initial_state = util.import_state_tuples(
self._initial_state, self._initial_state_name, num_replicas)
self._final_state = util.import_state_tuples(
self._final_state, self._final_state_name, num_replicas)
def export_ops(self, name):
"""Exports ops to collections."""
self._name = name
ops = {
util.with_prefix(self._name, "cost"): self._cost,
util.with_prefix(self._name, "kl_div"): self._kl_loss,
util.with_prefix(self._name, "input_data"): self._input_data,
util.with_prefix(self._name, "output"): self._output,
util.with_prefix(self._name, "targets"): self._targets,
}
if self._is_training:
ops.update(lr=self._lr,
new_lr=self._new_lr,
lr_update=self._lr_update)
for name, op in ops.items():
tf.add_to_collection(name, op)
self._initial_state_name = util.with_prefix(self._name, "initial")
self._final_state_name = util.with_prefix(self._name, "final")
util.export_state_tuples(self._initial_state, self._initial_state_name)
util.export_state_tuples(self._final_state, self._final_state_name)
def __init__(self, is_training, config, input_, name=None):
self._is_training = is_training
if name:
self._name = name
self._initial_state_name = util.with_prefix(self._name, "initial")
self._final_state_name = util.with_prefix(self._name, "final")
self._input = input_
self._rnn_params = None
self._cell = None
self.batch_size = input_.batch_size
self.num_steps = input_.num_steps
size = config.hidden_size
vocab_size = config.vocab_size
if not is_training:
self.x = tf.placeholder(tf.int32, shape=[1, 1])
self.y = tf.placeholder(tf.int32, shape=[1, 1])
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size],
dtype=data_type())
inputs = tf.nn.embedding_lookup(embedding, self.x)
else:
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size],
dtype=data_type())
inputs = tf.nn.embedding_lookup(embedding, input_.input_data)
if is_training and config.keep_prob < 1:
inputs = tf.nn.dropout(inputs, config.keep_prob)
output, state = self._build_rnn_graph(inputs, config, is_training)
softmax_w = tf.get_variable("softmax_w", [size, vocab_size],
dtype=data_type())
softmax_b = tf.get_variable("softmax_b", [vocab_size],
dtype=data_type())
logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)
# Reshape logits to be a 3-D tensor for sequence loss
logits = tf.reshape(logits,
[self.batch_size, self.num_steps, vocab_size])
#tmp = tf.reduce_sum(logits,1)
#self._logits = tf.nn.softmax(tmp, name="LOGITS")
#if not is_training:
# self._cost = tf.constant(0.0)
# self._final_state = state
# return
if is_training:
# Use the contrib sequence loss and average over the batches
loss = tf.contrib.seq2seq.sequence_loss(
logits,
input_.targets,
tf.ones([self.batch_size, self.num_steps], dtype=data_type()),
average_across_timesteps=False,
average_across_batch=True)
else:
# Use the contrib sequence loss and average over the batches
loss = tf.contrib.seq2seq.sequence_loss(
logits,
self.y,
tf.ones([self.batch_size, self.num_steps], dtype=data_type()),
average_across_timesteps=False,
average_across_batch=True)
# Update the cost
self._cost = tf.reduce_sum(loss)
self._final_state = state
if not is_training:
return
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self._cost, tvars),
config.max_grad_norm)
optimizer = tf.train.GradientDescentOptimizer(self._lr)
self._train_op = optimizer.apply_gradients(
zip(grads, tvars),
global_step=tf.contrib.framework.get_or_create_global_step())
self._new_lr = tf.placeholder(tf.float32,
shape=[],
name="new_learning_rate")
self._lr_update = tf.assign(self._lr, self._new_lr)
from __future__ import absolute_import
