def _build_clip_fn(self):
clip_class = self._params[self.CLIP_GRADS_CLASS_PK]
clip_params = self._params[self.CLIP_GRADS_PARAMS_PK]
if clip_class:
return configurable.factory(
clip_class, self.mode, clip_params, sys.modules[__name__])
return None
def __init__(self, config):
"""Initializes a new experiment instance."""
self._logdir = config[self.LOGDIR_KEY]
self._train_files = config[self.TRAIN_FILES_KEY]
self._train_steps = config[self.TRAIN_STEPS_KEY]
self._train_dev = config[self.TRAIN_DEVICE_KEY]
self._train_ckpt_every = config[self.TRAIN_CKPT_EVERY_KEY]
self._eval_files = config[self.EVAL_FILES_KEY]
self._eval_dev = config[self.EVAL_DEVICE_KEY]
self._eval_dump = config[self.EVAL_DUMP_KEY]
self._params = config[self.MODEL_PARAMS_KEY]
# build the training model.
t_params = copy.deepcopy(self._params)
t_params[Model.INPUT_PARAMS_PK][
ModelInputs.FILES_PK] = self._train_files
clz = config[self.MODEL_CLASS_KEY]
with tf.device(self._train_dev):
t_model = configurable.factory(clz, self._TRAIN, t_params,
dket.model)
t_logdir = os.path.join(self._logdir, self._TRAIN)
# build the eval model.
e_params = copy.deepcopy(self._params)
e_params[Model.INPUT_PARAMS_PK][
ModelInputs.FILES_PK] = self._eval_files
e_params[Model.INPUT_PARAMS_PK][ModelInputs.EPOCHS_PK] = 1
with tf.device(self._eval_dev):
e_model = configurable.factory(clz, self._EVAL, e_params,
dket.model)
e_logdir = os.path.join(self._logdir, self._EVAL)
e_dumpdir = os.path.join(e_logdir,
'dump') if self._eval_dump else ''
self._eval = Evaluation(model=e_model,
logdir=e_logdir,
steps=0,
metrics=get_metrics(),
dumpdir=e_dumpdir)
self._training = Training(model=t_model,
logdir=t_logdir,
steps=self._train_steps,
checkpoint_every=self._train_ckpt_every,
metrics=get_metrics(),
evaluation=self._eval)
def _build_lr(self, global_step):
lr = self._params[self.LR_PK] # pylint: disable=I0011,C0103
if self._params[self.LR_DECAY_CLASS_PK]:
lr_decay_class = self._params[self.LR_DECAY_CLASS_PK]
lr_decay_params = self._params[self.LR_DECAY_PARAMS_PK]
lr_decay_fn = configurable.factory(
lr_decay_class, self.mode, lr_decay_params, sys.modules[__name__])
lr = lr_decay_fn(lr, global_step) # pylint: disable=I0011,C0103
return lr
def _build_train_op(self):
if self.mode != tf.contrib.learn.ModeKeys.TRAIN:
logging.debug('mode is `%s`: skipping the loss calculation.', self.mode)
return
opt_class = self._params[self.OPTIMIZER_CLASS_PK]
opt_params = self._params[self.OPTIMIZER_PARAMS_PK]
self._optimizer = configurable.factory(opt_class, self.mode, opt_params, train)
self._train_op = self._optimizer.minimize(
self._loss_op, global_step=self._global_step)
def _build_loss(self):
if self.mode != tf.contrib.learn.ModeKeys.TRAIN:
logging.debug('mode is `%s`: skipping the loss calculation.', self.mode)
return
targets = self.inputs.get(ModelInputs.FORMULA_KEY)
lengths = self.inputs.get(ModelInputs.FORMULA_LENGTH_KEY)
weights = tf.sequence_mask(lengths, dtype=tf.float32)
predictions = self._predictions
clz = self._params[self.LOSS_NAME_PK]
loss = configurable.factory(clz, self.mode, {}, train)
self._loss_op = loss.compute(targets, predictions, weights=weights)
def test_factory(self):
"""Test the configurable.factory method."""
clz = __name__ + '.' + DummyConf.__name__
module = sys.modules[__name__]
mode = tf.contrib.learn.ModeKeys.TRAIN
params = DummyConf.get_default_params()
params['seed'] = 23
params['label'] = 'ciaone proprio!'
instance = configurable.factory(clz, mode, params, module)
self.assertIsInstance(instance, DummyConf)
self.assertEqual(mode, instance.mode)
self.assertEqual(params, instance.get_params())
self.assertEqual(1, len(DummyConf.get_create_call_args()))
self.assertEqual((mode, params), DummyConf.call_args())
def test_factory(self):
"""Test the configurable.factory method."""
clz = __name__ + '.' + DummyConf.__name__
module = sys.modules[__name__]
mode = tf.contrib.learn.ModeKeys.TRAIN
params = DummyConf.get_default_params()
params['seed'] = 23
params['label'] = 'ciaone proprio!'
instance = configurable.factory(clz, mode, params, module)
self.assertIsInstance(instance, DummyConf)
self.assertEqual(mode, instance.mode)
self.assertEqual(params, instance.get_params())
self.assertEqual(1, len(DummyConf.get_create_call_args()))
self.assertEqual((mode, params), DummyConf.call_args())
def _do_test(self, ctype):
clz = ctype.__module__ + '.' + ctype.__name__
params = ctype.get_default_params()
instance = configurable.factory(clz, TRAIN, params)
self.assertIsNotNone(instance)
def _build_graph(self):
trainable = self.mode == tf.contrib.learn.ModeKeys.TRAIN
words = self.inputs.get(self.inputs.WORDS_KEY)
slengths = self.inputs.get(self.inputs.SENTENCE_LENGTH_KEY)
targets = self.inputs.get(self.inputs.FORMULA_KEY)
flengths = self.inputs.get(self.inputs.FORMULA_LENGTH_KEY)
with self._graph.as_default(): # pylint: disable=E1129
with tf.variable_scope('Embedding'): # pylint: disable=E1129
with tf.device('CPU:0'):
embedding_size = self._params['embedding_size']
vocabulary_size = self._params[self.INPUT_VOC_SIZE_PK]
embeddings = tf.get_variable(
'E', [vocabulary_size, embedding_size])
inputs = tf.nn.embedding_lookup(embeddings, words)
batch_dim = utils.get_dimension(words, 0)
with tf.variable_scope('Encoder'): # pylint: disable=E1129
encoder_params = self._params['encoder']
encoder_cell_type = encoder_params['cell.type']
encoder_cell_params = encoder_params['cell.params']
encoder_cell = configurable.factory(encoder_cell_type,
self._mode,
encoder_cell_params, rnn)
state = encoder_cell.zero_state(batch_dim, tf.float32)
encoder_out, _ = tf.nn.dynamic_rnn(
cell=encoder_cell,
initial_state=state,
inputs=inputs,
sequence_length=slengths,
parallel_iterations=self._params['parallel_iterations'])
with tf.variable_scope('Decoder'): # pylint: disable=E1129
decoder_params = self._params['decoder']
decoder_cell_type = decoder_params['cell.type']
decoder_cell_params = decoder_params['cell.params']
decoder_cell = configurable.factory(decoder_cell_type,
self._mode,
decoder_cell_params, rnn)
attention = layers.BahdanauAttention(
states=encoder_out,
inner_size=self._params['attention_size'],
trainable=trainable)
location = layers.LocationSoftmax(attention=attention,
sequence_length=slengths)
output = layers.PointingSoftmaxOutput(
shortlist_size=self._params[self.OUTPUT_VOC_SIZE_PK],
decoder_out_size=decoder_cell.output_size,
state_size=encoder_out.shape[-1].value,
trainable=trainable)
self._decoder_inputs = None
if trainable:
location_size = utils.get_dimension(words, 1)
output_size = self._params[
self.OUTPUT_VOC_SIZE_PK] + location_size
self._decoder_inputs = tf.one_hot(
targets,
output_size,
dtype=tf.float32,
name='decoder_training_input')
ps_decoder = layers.PointingSoftmaxDecoder(
cell=decoder_cell,
location_softmax=location,
pointing_output=output,
input_size=self._params['feedback_size'],
decoder_inputs=self._decoder_inputs,
trainable=trainable)
eos = None if trainable else self.EOS_IDX
pad_to = None if trainable else utils.get_dimension(targets, 1)
helper = layers.TerminationHelper(lengths=flengths, EOS=eos)
decoder = layers.DynamicDecoder(
decoder=ps_decoder,
helper=helper,
pad_to=pad_to,
parallel_iterations=self._params['parallel_iterations'],
swap_memory=False)
self._predictions, _ = decoder.decode()
def _build_inputs(self):
clz = self._params[self.INPUT_CLASS_PK]
params = self._params[self.INPUT_PARAMS_PK]
self._inputs = configurable.factory(clz, self.mode, params)
def _build_graph(self):
trainable = self.mode == tf.contrib.learn.ModeKeys.TRAIN
words = self.inputs.get(self.inputs.WORDS_KEY)
slengths = self.inputs.get(self.inputs.SENTENCE_LENGTH_KEY)
targets = self.inputs.get(self.inputs.FORMULA_KEY)
flengths = self.inputs.get(self.inputs.FORMULA_LENGTH_KEY)
with self._graph.as_default(): # pylint: disable=E1129
if self._seed:
tf.set_random_seed(self._seed)
with tf.variable_scope('Embedding'): # pylint: disable=E1129
with tf.device('CPU:0'):
embedding_size = self._params['embedding_size']
vocabulary_size = self._params[self.INPUT_VOC_SIZE_PK]
embeddings = tf.get_variable(
'E', [vocabulary_size, embedding_size])
inputs = tf.nn.embedding_lookup(embeddings, words)
batch_dim = utils.get_dimension(words, 0)
with tf.variable_scope('Encoder'): # pylint: disable=E1129
encoder_params = self._params['encoder']
encoder_cell_type = encoder_params['cell.type']
encoder_cell_params = encoder_params['cell.params']
encoder_cell = configurable.factory(encoder_cell_type, self._mode, encoder_cell_params, rnn)
state = encoder_cell.zero_state(batch_dim, tf.float32)
encoder_out, _ = tf.nn.dynamic_rnn(
cell=encoder_cell,
initial_state=state,
inputs=inputs,
sequence_length=slengths,
parallel_iterations=self._params['parallel_iterations'])
with tf.variable_scope('Decoder'): # pylint: disable=E1129
decoder_params = self._params['decoder']
decoder_cell_type = decoder_params['cell.type']
decoder_cell_params = decoder_params['cell.params']
decoder_cell = configurable.factory(decoder_cell_type, self._mode, decoder_cell_params, rnn)
attention = layers.BahdanauAttention(
states=encoder_out,
inner_size=self._params['attention_size'],
trainable=trainable)
location = layers.LocationSoftmax(
attention=attention,
sequence_length=slengths)
output = layers.PointingSoftmaxOutput(
shortlist_size=self._params[self.OUTPUT_VOC_SIZE_PK],
decoder_out_size=decoder_cell.output_size,
state_size=encoder_out.shape[-1].value,
trainable=trainable)
self._decoder_inputs = None
if trainable:
location_size = utils.get_dimension(words, 1)
output_size = self._params[self.OUTPUT_VOC_SIZE_PK] + location_size
self._decoder_inputs = tf.one_hot(
targets, output_size, dtype=tf.float32,
name='decoder_training_input')
ps_decoder = layers.PointingSoftmaxDecoder(
cell=decoder_cell,
location_softmax=location,
pointing_output=output,
input_size=self._params['feedback_size'],
decoder_inputs=self._decoder_inputs,
trainable=trainable)
eos = None if trainable else self.EOS_IDX
pad_to = None if trainable else utils.get_dimension(targets, 1)
helper = layers.TerminationHelper(
lengths=flengths, EOS=eos)
decoder = layers.DynamicDecoder(
decoder=ps_decoder, helper=helper, pad_to=pad_to,
parallel_iterations=self._params['parallel_iterations'],
swap_memory=False)
self._predictions, _ = decoder.decode()
def _build_inputs(self):
clz = self._params[self.INPUT_CLASS_PK]
params = self._params[self.INPUT_PARAMS_PK]
self._inputs = configurable.factory(clz, self.mode, params)
def _do_test(self, ctype):
clz = ctype.__module__ + '.' + ctype.__name__
params = ctype.get_default_params()
instance = configurable.factory(clz, TRAIN, params)
self.assertIsNotNone(instance)
