def test_train_max_steps_is_not_incremental(self):
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
learn.graph_actions.train(
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=constant_op.constant(2.0),
max_steps=10)
step = checkpoint_utils.load_variable(
self._output_dir, variables_lib.get_global_step().name)
self.assertEqual(10, step)
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
learn.graph_actions.train(
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=constant_op.constant(2.0),
max_steps=15)
step = checkpoint_utils.load_variable(
self._output_dir, variables_lib.get_global_step().name)
self.assertEqual(15, step)
def test_train_skip_train_if_max_step_already_saved(self):
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
learn.graph_actions._monitored_train( # pylint: disable=protected-access
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=constant_op.constant(2.0),
max_steps=10)
step = checkpoint_utils.load_variable(
self._output_dir, variables_lib.get_global_step().name)
self.assertEqual(10, step)
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
learn.graph_actions._monitored_train( # pylint: disable=protected-access
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=constant_op.constant(2.0),
max_steps=10)
step = checkpoint_utils.load_variable(
self._output_dir, variables_lib.get_global_step().name)
self.assertEqual(10, step)
def _make_training_op(training_loss):
"""Training op for the DNN linear combined model."""
train_ops = []
if dnn_logits is not None:
train_ops.append(
optimizers.optimize_loss(
loss=training_loss,
global_step=contrib_variables.get_global_step(),
learning_rate=_DNN_LEARNING_RATE,
optimizer=_get_optimizer(dnn_optimizer),
gradient_multipliers=_extract_embedding_lr_multipliers( # pylint: disable=protected-access
embedding_lr_multipliers, dnn_parent_scope,
dnn_input_scope.name),
clip_gradients=gradient_clip_norm,
variables=ops.get_collection(dnn_parent_scope),
name=dnn_parent_scope,
# Empty summaries, because head already logs "loss" summary.
summaries=[]))
if linear_logits is not None:
train_ops.append(
optimizers.optimize_loss(
loss=training_loss,
global_step=contrib_variables.get_global_step(),
learning_rate=_linear_learning_rate(len(linear_feature_columns)),
optimizer=_get_optimizer(linear_optimizer),
clip_gradients=gradient_clip_norm,
variables=ops.get_collection(linear_parent_scope),
name=linear_parent_scope,
# Empty summaries, because head already logs "loss" summary.
summaries=[]))
return control_flow_ops.group(*train_ops)
def test_get_global_step(self):
with ops.Graph().as_default() as g:
self.assertEquals(None, variables_lib2.get_global_step())
variables_lib.Variable(
0,
trainable=False,
dtype=dtypes.int32,
name=ops.GraphKeys.GLOBAL_STEP)
self._assert_global_step(
variables_lib2.get_global_step(), expected_dtype=dtypes.int32)
self._assert_global_step(
variables_lib2.get_global_step(g), expected_dtype=dtypes.int32)
def _get_train_ops(self, features, targets):
"""See base class."""
if self._linear_feature_columns is None:
self._linear_feature_columns = layers.infer_real_valued_columns(
features)
if not isinstance(self._linear_optimizer,
sdca_optimizer.SDCAOptimizer):
return super(LinearClassifier,
self)._get_train_ops(features, targets)
# SDCA currently supports binary classification only.
if self._n_classes > 2:
raise ValueError(
"SDCA does not currently support multi-class classification.")
global_step = contrib_variables.get_global_step()
assert global_step
logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=self._linear_feature_columns,
num_outputs=self._num_label_columns(),
weight_collections=[self._linear_weight_collection],
name="linear")
with ops.control_dependencies([self._centered_bias()]):
loss = self._loss(logits, targets,
self._get_weight_tensor(features))
logging_ops.scalar_summary("loss", loss)
train_ops = self._linear_optimizer.get_train_step(
self._linear_feature_columns, self._weight_column_name,
"logistic_loss", features, targets, columns_to_variables,
global_step)
return train_ops, loss
def gmm_cluster_model_fn(features, labels, mode, params, config=None):
"""Model function."""
assert labels is None, labels
update_params = ''
for i in ["w", "m", 'c']:
if i in params["update_params"]:
update_params += i
(all_scores,
model_predictions,
losses, training_op) = gmm_ops.gmm(parse_tensor_or_dict(features),
"random",
params["num_clusters"],
params["random_seed"],
params["covariance_type"],
update_params,
)
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
'all_scores': all_scores[0],
'assignments': model_predictions[0][0],
}
eval_metric_ops = {
'scores': streaming_sum(loss),
}
return model_fn_lib.ModelFnOps(mode=mode, predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss, train_op=training_op)
def begin(self):
self._last_saved_step = None
self._request_summary = True
self._global_step_tensor = contrib_variables.get_global_step()
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use SummarySaverHook.")
def begin(self):
self._last_reported_time = None
self._last_reported_step = None
self._global_step_tensor = contrib_variables.get_global_step()
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use StepCounterHook.")
def begin(self):
self._last_saved_time = None
self._last_saved_step = None
self._global_step_tensor = contrib_variables.get_global_step()
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use CheckpointSaverHook.")
def _model_fn(features, labels, mode):
"""Model function."""
assert labels is None, labels
(all_scores, model_predictions, losses,
training_op) = clustering_ops.KMeans(
self._parse_tensor_or_dict(features),
self._num_clusters,
self._training_initial_clusters,
self._distance_metric,
self._use_mini_batch,
random_seed=self._random_seed,
kmeans_plus_plus_num_retries=self.
_kmeans_plus_plus_num_retries).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
logging_ops.scalar_summary('loss/raw', loss)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
KMeansClustering.ALL_SCORES: all_scores[0],
KMeansClustering.CLUSTER_IDX: model_predictions[0],
}
eval_metric_ops = {KMeansClustering.SCORES: loss,}
return ModelFnOps(
mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op)
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._logits(features, is_training=True)
with ops.control_dependencies([
self._centered_bias_step(targets,
self._get_weight_tensor(features))
]):
loss = self._loss(logits, targets,
self._get_weight_tensor(features))
logging_ops.scalar_summary("loss", loss)
linear_vars = self._get_linear_vars()
dnn_vars = self._get_dnn_vars()
grads = gradients.gradients(loss, dnn_vars + linear_vars)
dnn_grads = grads[0:len(dnn_vars)]
linear_grads = grads[len(dnn_vars):]
train_ops = self._get_linear_training_ops(
linear_grads, linear_vars) + self._get_dnn_training_ops(
dnn_grads, dnn_vars)
train_step = control_flow_ops.group(*train_ops,
name="combined_training_op")
with ops.control_dependencies([train_step]):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def begin(self):
self._last_reported_time = None
self._last_reported_step = None
self._global_step_tensor = contrib_variables.get_global_step()
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use StepCounterHook.")
def _get_train_ops(self, features, targets):
"""See base class."""
self._validate_linear_feature_columns(features)
if not isinstance(self._linear_optimizer, sdca_optimizer.SDCAOptimizer):
return super(LinearClassifier, self)._get_train_ops(features, targets)
# SDCA currently supports binary classification only.
if self._target_column.num_label_columns > 2:
raise ValueError(
"SDCA does not currently support multi-class classification.")
global_step = contrib_variables.get_global_step()
assert global_step
logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=self._linear_feature_columns,
num_outputs=self._target_column.num_label_columns,
weight_collections=[self._linear_weight_collection],
name="linear")
with ops.control_dependencies([self._centered_bias()]):
loss = self._loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
train_ops = self._linear_optimizer.get_train_step(
self._linear_feature_columns, self._target_column.weight_column_name,
"logistic_loss", features, targets, columns_to_variables, global_step)
return train_ops, loss
def begin(self):
self._last_saved_step = None
self._request_summary = True
self._global_step_tensor = contrib_variables.get_global_step()
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use SummarySaverHook.")
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
features = self._get_feature_dict(features)
logits = self._logits(features, is_training=True)
if self._enable_centered_bias:
centered_bias_step = [self._centered_bias_step(targets, features)]
else:
centered_bias_step = []
with ops.control_dependencies(centered_bias_step):
training_loss = self._target_column.training_loss(logits, targets,
features)
weighted_average_loss = self._target_column.loss(logits, targets,
features)
logging_ops.scalar_summary("loss", weighted_average_loss)
linear_train_step = self._linear_model.get_train_step(training_loss)
dnn_train_step = (self._dnn_model.get_train_step(training_loss) if
self._dnn_model else [])
with ops.control_dependencies(linear_train_step + dnn_train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, weighted_average_loss
def begin(self):
self._last_saved_time = None
self._last_saved_step = None
self._global_step_tensor = contrib_variables.get_global_step()
if self._global_step_tensor is None:
raise RuntimeError(
"Global step should be created to use CheckpointSaverHook.")
def test_train_chief_monitor(self):
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(
variables_lib.get_global_step(), 1)
loss_op = constant_op.constant(2.0)
summary.scalar('loss', loss_op)
chief_exclusive_monitor = _BaseMonitorWrapper(False)
all_workers_monitor = _BaseMonitorWrapper(True)
loss = learn.graph_actions.train(
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=loss_op,
supervisor_is_chief=True,
steps=1,
monitors=[chief_exclusive_monitor, all_workers_monitor])
self.assertEqual(2.0, loss)
self.assertTrue(
chief_exclusive_monitor.is_active
and all_workers_monitor.is_active,
'All monitors must have been active.')
self.assertTrue(
chief_exclusive_monitor.has_step
and all_workers_monitor.has_step,
'All monitors must have a step.')
def _model_fn(features, labels, mode, config):
"""Model function."""
assert labels is None, labels
(all_scores,
model_predictions,
losses, training_op,
init_op,
is_initialized) = gmm_ops.gmm(self._parse_tensor_or_dict(features),
self._training_initial_clusters,
self._num_clusters, self._random_seed,
self._covariance_type,
self._params)
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses)
training_op = with_dependencies([training_op, incr_step], loss)
training_hooks = [_InitializeClustersHook(
init_op, is_initialized, config.is_chief)]
predictions = {
GMM.ALL_SCORES: all_scores[0],
GMM.ASSIGNMENTS: model_predictions[0][0],
}
eval_metric_ops = {
GMM.SCORES: _streaming_sum(loss),
}
return model_fn_lib.ModelFnOps(mode=mode, predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss, train_op=training_op,
training_hooks=training_hooks)
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._logits(features, is_training=True)
if self._enable_centered_bias:
centered_bias_step = [self._centered_bias_step(targets, features)]
else:
centered_bias_step = []
with ops.control_dependencies(centered_bias_step):
loss = self._loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
linear_vars = self._get_linear_vars()
dnn_vars = self._get_dnn_vars()
grads = gradients.gradients(loss, dnn_vars + linear_vars)
if self._gradient_clip_norm:
grads, _ = clip_ops.clip_by_global_norm(grads, self._gradient_clip_norm)
dnn_grads = grads[0 : len(dnn_vars)]
linear_grads = grads[len(dnn_vars) :]
train_ops = self._get_linear_training_ops(linear_grads, linear_vars) + self._get_dnn_training_ops(
dnn_grads, dnn_vars
)
train_step = control_flow_ops.group(*train_ops, name="combined_training_op")
with ops.control_dependencies([train_step]):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def _invalid_model_fn(features, labels):
# pylint: disable=unused-argument
w = variables_lib.Variable(42.0, 'weight')
update_global_step = variables.get_global_step().assign_add(1)
with control_flow_ops.control_dependencies([update_global_step]):
loss = 100.0 - w
return None, loss, None
def test_train_summaries(self):
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(
variables_lib.get_global_step(), 1)
loss_op = constant_op.constant(2.0)
summary.scalar('loss', loss_op)
writer = learn.graph_actions.get_summary_writer(self._output_dir)
self._assert_summaries(self._output_dir, writer)
self._assert_ckpt(self._output_dir, False)
loss = learn.graph_actions._monitored_train( # pylint: disable=protected-access
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=loss_op,
steps=1)
meta_graph_def = meta_graph.create_meta_graph_def(
graph_def=g.as_graph_def(add_shapes=True),
saver_def=monitored_session.Scaffold().finalize(
).saver.saver_def)
self.assertEqual(2.0, loss)
self._assert_summaries(self._output_dir,
writer,
expected_graphs=[g],
expected_meta_graphs=[meta_graph_def],
expected_summaries={1: {
'loss': 2.0
}})
self._assert_ckpt(self._output_dir, True)
def test_train_summaries(self):
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
loss_op = constant_op.constant(2.0)
summary.scalar('loss', loss_op)
self._assert_summaries(self._output_dir)
self._assert_ckpt(self._output_dir, False)
loss = learn.graph_actions.train(
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=loss_op,
steps=1)
# TODO(ebrevdo,ptucker,ispir): this meta_graph_def lacks the
# SaverDef, so we can't add it to the summary assertion test below.
# meta_graph_def = meta_graph.create_meta_graph_def()
self.assertEqual(2.0, loss)
self._assert_summaries(
self._output_dir,
expected_graphs=[g],
expected_summaries={1: {
'loss': 2.0
}})
self._assert_ckpt(self._output_dir, True)
def _model_fn(features, labels, mode):
"""Model function."""
assert labels is None, labels
(all_scores, model_predictions, losses,
training_op) = clustering_ops.KMeans(
self._parse_tensor_or_dict(features),
self._num_clusters,
self._training_initial_clusters,
self._distance_metric,
self._use_mini_batch,
random_seed=self._random_seed,
kmeans_plus_plus_num_retries=self.
_kmeans_plus_plus_num_retries).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
logging_ops.scalar_summary('loss/raw', loss)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
KMeansClustering.ALL_SCORES: all_scores[0],
KMeansClustering.CLUSTER_IDX: model_predictions[0],
}
eval_metric_ops = {KMeansClustering.SCORES: loss,}
if self._relative_tolerance is not None:
training_hooks = [self.LossRelativeChangeHook(self._relative_tolerance)]
else:
training_hooks = None
return ModelFnOps(
mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op,
training_hooks=training_hooks)
def test_train_summaries(self):
with ops.Graph().as_default() as g, self.test_session(g):
with ops.control_dependencies(self._build_inference_graph()):
train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
loss_op = constant_op.constant(2.0)
summary.scalar('loss', loss_op)
writer = learn.graph_actions.get_summary_writer(self._output_dir)
self._assert_summaries(self._output_dir, writer)
self._assert_ckpt(self._output_dir, False)
loss = learn.graph_actions._monitored_train( # pylint: disable=protected-access
g,
output_dir=self._output_dir,
train_op=train_op,
loss_op=loss_op,
steps=1)
meta_graph_def = meta_graph.create_meta_graph_def(
graph_def=g.as_graph_def(add_shapes=True),
saver_def=monitored_session.Scaffold().finalize().saver.saver_def)
self.assertEqual(2.0, loss)
self._assert_summaries(
self._output_dir,
writer,
expected_graphs=[g],
expected_meta_graphs=[meta_graph_def],
expected_summaries={1: {
'loss': 2.0
}})
self._assert_ckpt(self._output_dir, True)
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
features = self._get_feature_dict(features)
logits = self._logits(features, is_training=True)
if self._enable_centered_bias:
centered_bias_step = [self._centered_bias_step(targets, features)]
else:
centered_bias_step = []
with ops.control_dependencies(centered_bias_step):
training_loss = self._target_column.training_loss(logits, targets,
features)
weighted_average_loss = self._target_column.loss(logits, targets,
features)
logging_ops.scalar_summary("loss", weighted_average_loss)
linear_train_step = self._linear_model.get_train_step(training_loss)
dnn_train_step = (self._dnn_model.get_train_step(training_loss) if
self._dnn_model else [])
with ops.control_dependencies(linear_train_step + dnn_train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, weighted_average_loss
def _argument_checker(features, labels, mode, params, config=None,
model_dir=None):
_, _, _ = features, labels, config
self.assertEqual(model_fn.ModeKeys.TRAIN, mode)
self.assertEqual(expected_param, params)
self.assertEqual(model_dir, expected_model_dir)
return (constant_op.constant(0.), constant_op.constant(0.),
variables.get_global_step().assign_add(1))
def _train_op_fn(loss):
global_step = contrib_variables.get_global_step()
my_vars = ops.get_collection(parent_scope)
grads = gradients.gradients(loss, my_vars)
if gradient_clip_norm:
grads, _ = clip_ops.clip_by_global_norm(grads, gradient_clip_norm)
return (_get_optimizer(optimizer).apply_gradients(
zip(grads, my_vars), global_step=global_step))
def model_fn(features, labels):
# dummy variable:
_ = variables_lib.Variable([0.])
_ = labels
predictions = features["x"]
loss = constant_op.constant([2.])
update_global_step = variables.get_global_step().assign_add(1)
return predictions, loss, update_global_step
def _train_op_fn(loss):
global_step = contrib_variables.get_global_step()
assert global_step
train_step = model.get_train_step(loss)
with ops.control_dependencies(train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op
def _train_op_fn(loss):
global_step = contrib_variables.get_global_step()
my_vars = ops.get_collection("linear")
grads = gradients.gradients(loss, my_vars)
if gradient_clip_norm:
grads, _ = clip_ops.clip_by_global_norm(grads, gradient_clip_norm)
return (_get_optimizer(optimizer).apply_gradients(
zip(grads, my_vars), global_step=global_step))
def _train_op_fn(loss):
global_step = contrib_variables.get_global_step()
assert global_step
train_step = model.get_train_step(loss)
with ops.control_dependencies(train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op
def _train_op_fn(unused_loss):
global_step = contrib_variables.get_global_step()
sdca_model, train_op = optimizer.get_train_step(
columns_to_variables, weight_column_name, loss_type, features,
targets, global_step)
if update_weights_hook is not None:
update_weights_hook.set_parameters(sdca_model, train_op)
return train_op
def _model_fn_scaffold(features, labels, mode):
_, _ = features, labels
return model_fn.ModelFnOps(
mode=mode,
predictions=constant_op.constant(0.),
loss=constant_op.constant(0.),
train_op=variables.get_global_step().assign_add(1),
scaffold=monitored_session.Scaffold(init_fn=_init_fn))
def _train_op_fn(unused_loss):
global_step = contrib_variables.get_global_step()
sdca_model, train_op = optimizer.get_train_step(
columns_to_variables, weight_column_name, loss_type, features, labels,
global_step)
if update_weights_hook is not None:
update_weights_hook.set_parameters(sdca_model, train_op)
return train_op
def model_fn(features, labels): # dummy variable: _ = variables_lib.Variable([0.]) _ = labels predictions = features["x"] loss = constant_op.constant([2.]) update_global_step = variables.get_global_step().assign_add(1) return predictions, loss, update_global_step
def dynamic_model_fn(features, targets, mode, params):
VOCAB_SIZE = len(params['token_map'])
embeddings = embedding_matrix(VOCAB_SIZE, params['embedding_dim'])
image = features['image']
visual_features = convolutional_features(tf.expand_dims(image, -1),
params['vdim'])
lstm = LSTMSingleton.get_instance(params['hdim'])
if mode is tf.contrib.learn.ModeKeys.TRAIN:
target_tokens = targets['target']
# Decoder inputs should be in time-major order.
decoder_inputs = tf.to_int64(array_ops.transpose(target_tokens, [1, 0]))
sequence_length = targets['sequence_length']
feed_prev_output = params['feed_prev_output']
else:
# Doesn't matter how long the sequence is since we will be feeding in
# tokens predicted from the previous iteration.
decoder_inputs = tf.zeros((1, params['batch_size']), dtype=tf.int64)
sequence_length = tf.constant(np.full((params['batch_size'],),
MAX_LENGTH), dtype=tf.int32)
feed_prev_output = True
# Builds the actual model.
initial_state = get_initial_lstm_state(
tf.reduce_mean(visual_features, (1,2)), params)
init_token = start_sequence_token(params['batch_size'])
fn = get_loop_fn(decoder_inputs, sequence_length, visual_features, lstm,
initial_state, attention, init_token, embeddings,
feed_prev_output=feed_prev_output,
hparams=params)
token_logits, final_state, _ = tf.nn.raw_rnn(lstm, fn)
# 'token_logits' holds the log-probability of each token for each
# iteration of the decoding sequence.
# Get logits as packed tensor in batch-major order
token_logits = array_ops.transpose(token_logits.pack(), [1, 0, 2])
if mode is tf.contrib.learn.ModeKeys.TRAIN:
loss = sequence_loss(token_logits, targets['target'],
targets['weights'])
global_step = variables.get_global_step()
learning_rate = tf.train.exponential_decay(
params['starter_learning_rate'],
global_step, 20000,
0.96, staircase=True)
optimizer = tf.train.AdamOptimizer(params['starter_learning_rate'],
epsilon=params['epsilon'])
train_op = optimizer.minimize(loss, global_step=global_step)
return (None, loss, train_op)
else:
predicted_tokens = tf.argmax(token_logits, 2)
print "pred tokens shape:", predicted_tokens.get_shape()
return (predicted_tokens, None, None)
def linear_model_fn_with_model_fn_ops(features, labels, mode):
"""Same as linear_model_fn, but returns `ModelFnOps`."""
assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL,
model_fn.ModeKeys.INFER)
prediction, loss = (models.linear_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(
loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1)
return model_fn.ModelFnOps(
mode=mode, predictions=prediction, loss=loss, train_op=train_op)
def linear_model_fn_with_model_fn_ops(features, labels, mode):
"""Same as linear_model_fn, but returns `ModelFnOps`."""
assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL,
model_fn.ModeKeys.INFER)
prediction, loss = (models.linear_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(
loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1)
return model_fn.ModelFnOps(
mode=mode, predictions=prediction, loss=loss, train_op=train_op)
def _get_train_ops(self, features, _):
(_, _, losses, training_op) = gmm_ops.gmm(
self._parse_tensor_or_dict(features), self._training_initial_clusters,
self._num_clusters, self._random_seed, self._covariance_type,
self._params)
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses)
training_op = with_dependencies([training_op, incr_step], loss)
return training_op, loss
def _get_train_ops(self, features, _):
(_, _, losses,
training_op) = gmm_ops.gmm(self._parse_tensor_or_dict(features),
self._training_initial_clusters,
self._num_clusters, self._random_seed,
self._covariance_type, self._params)
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses)
training_op = with_dependencies([training_op, incr_step], loss)
return training_op, loss
def test_create_global_step(self):
self.assertEquals(None, variables_lib2.get_global_step())
with ops.Graph().as_default() as g:
global_step = variables_lib2.create_global_step()
self._assert_global_step(global_step)
self.assertRaisesRegexp(ValueError, 'already exists',
variables_lib2.create_global_step)
self.assertRaisesRegexp(ValueError, 'already exists',
variables_lib2.create_global_step, g)
self._assert_global_step(variables_lib2.create_global_step(ops.Graph()))
def _make_training_op(training_loss):
global_step = contrib_variables.get_global_step()
assert global_step
linear_train_step = self._linear_model.get_train_step(training_loss)
dnn_train_step = (self._dnn_model.get_train_step(training_loss) if
self._dnn_model else [])
with ops.control_dependencies(linear_train_step + dnn_train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op
def _make_training_op(training_loss):
global_step = contrib_variables.get_global_step()
assert global_step
linear_train_step = self._linear_model.get_train_step(training_loss)
dnn_train_step = (self._dnn_model.get_train_step(training_loss) if
self._dnn_model else [])
with ops.control_dependencies(linear_train_step + dnn_train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op
def logistic_model_no_mode_fn(features, labels):
features = extract(features, 'input')
labels = extract(labels, 'labels')
labels = array_ops.one_hot(labels, 3, 1, 0)
prediction, loss = (models.logistic_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(
loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1)
return {
'class': math_ops.argmax(prediction, 1),
'prob': prediction
}, loss, train_op
def linear_model_fn(features, labels, mode):
features = extract(features, 'input')
labels = extract(labels, 'labels')
assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL,
model_fn.ModeKeys.INFER)
if isinstance(features, dict):
(_, features), = features.items()
prediction, loss = (models.linear_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(
loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1)
return prediction, loss, train_op
def logistic_model_no_mode_fn(features, labels):
features = extract(features, 'input')
labels = extract(labels, 'labels')
labels = array_ops.one_hot(labels, 3, 1, 0)
prediction, loss = (models.logistic_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(
loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1)
return {
'class': math_ops.argmax(prediction, 1),
'prob': prediction
}, loss, train_op
def linear_model_fn(features, labels, mode):
features = extract(features, 'input')
labels = extract(labels, 'labels')
assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL,
model_fn.ModeKeys.INFER)
if isinstance(features, dict):
(_, features), = features.items()
prediction, loss = (models.linear_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(
loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1)
return prediction, loss, train_op
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
return optimizers.optimize_loss(
loss=loss,
global_step=contrib_variables.get_global_step(),
learning_rate=_LEARNING_RATE,
optimizer=_get_optimizer(optimizer),
clip_gradients=gradient_clip_norm,
name=parent_scope,
# Empty summaries to prevent optimizers from logging the training_loss.
summaries=[])
def _get_train_ops(self, features, targets):
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._model.build_model(features, self._feature_columns, is_training=True)
loss = self._target_column.loss(logits, targets, features)
train_step = self._model.get_train_step(loss)
with ops.control_dependencies(train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def _invalid_model_fn(features, labels, mode):
# pylint: disable=unused-argument
w = variables_lib.Variable(42.0, 'weight')
loss = 100.0 - w
update_global_step = variables.get_global_step().assign_add(1)
with control_flow_ops.control_dependencies([update_global_step]):
train_op = w.assign_add(loss / 100.0)
predictions = loss
if mode == model_fn.ModeKeys.EVAL:
loss = None
return predictions, loss, train_op
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
return optimizers.optimize_loss(
loss=loss,
global_step=contrib_variables.get_global_step(),
learning_rate=_LEARNING_RATE,
optimizer=_get_optimizer(optimizer),
clip_gradients=gradient_clip_norm,
name=parent_scope,
# Empty summaries to prevent optimizers from logging the training_loss.
summaries=[])
def kmeans_cluster_model_fn(features, labels, mode, params, config):
"""Model function for KMeansClustering estimator."""
# https://wiki.python.org/moin/UsingAssertionsEffectively
assert labels is None, "labels are not needed: " + labels
# clustering_ops.KMeans 是重要的算法实现过程
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow
# /contrib/factorization/python/ops/clustering_ops.py
(all_scores, model_predictions, losses,
is_initialized, cluster_centers_var, init_op, training_op) = \
clustering_ops.KMeans(
_parse_tensor_or_dict(features),
params.get('num_clusters'),
initial_clusters=clustering_ops.RANDOM_INIT,
distance_metric=clustering_ops.SQUARED_EUCLIDEAN_DISTANCE,
use_mini_batch=False,
mini_batch_steps_per_iteration=1,
# use_mini_batch = params.get('use_mini_batch'),
# mini_batch_steps_per_iteration=params.get(
# 'mini_batch_steps_per_iteration'),
random_seed=params.get('random_seed'),
kmeans_plus_plus_num_retries=params.get(
'kmeans_plus_plus_num_retries')).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name='kmeans_loss')
# Outputs a Summary protocol buffer containing a single scalar value.
# Used for visualizing in TensorBoard
summary.scalar('loss/raw', loss)
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow
# /python/ops/control_flow_ops.py
# with_dependencies(dependencies, output_tensor, name=None):
# Produces the content of `output_tensor` only after `dependencies`.
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
'all_scores': all_scores[0],
'cluster_idx': model_predictions[0],
}
eval_metric_ops = {'scores': loss}
# Hook for monitor
training_hooks = [
_InitializeClustersHook(init_op, is_initialized, config.is_chief)
]
relative_tolerance = params.get('relative_tolerance')
if relative_tolerance is not None:
training_hooks.append(_LossRelativeChangeHook(relative_tolerance))
return ModelFnOps(mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op,
training_hooks=training_hooks)
def linear_model_params_fn(features, labels, mode, params):
features = extract(features, 'input')
labels = extract(labels, 'labels')
assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL,
model_fn.ModeKeys.INFER)
prediction, loss = (models.linear_regression_zero_init(features, labels))
train_op = optimizers.optimize_loss(loss,
variables.get_global_step(),
optimizer='Adagrad',
learning_rate=params['learning_rate'])
return prediction, loss, train_op
def _get_train_ops(self, features, targets):
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._model.build_model(
features, self._feature_columns, is_training=True)
loss = self._target_column.loss(logits, targets, features)
train_step = self._model.get_train_step(loss)
with ops.control_dependencies(train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def _export_graph(graph, saver, checkpoint_path, export_dir,
default_graph_signature, named_graph_signatures,
exports_to_keep):
"""Exports graph via session_bundle, by creating a Session."""
with graph.as_default():
with tf_session.Session('') as session:
session.run(variables.initialize_local_variables())
saver.restore(session, checkpoint_path)
export = exporter.Exporter(saver)
export.init(session.graph.as_graph_def(),
default_graph_signature=default_graph_signature,
named_graph_signatures=named_graph_signatures)
export.export(export_dir, contrib_variables.get_global_step(), session,
exports_to_keep=exports_to_keep)
def _train(features):
"""Add training ops to the graph."""
with variable_scope.variable_scope("model"):
model_outputs = state_manager.define_loss(
model, features, estimator_lib.ModeKeys.TRAIN)
train_op = optimizers.optimize_loss(
model_outputs.loss,
global_step=variables.get_global_step(),
optimizer=optimizer,
# Learning rate is set in the Optimizer object
learning_rate=None)
return estimator_lib.EstimatorSpec(loss=model_outputs.loss,
mode=estimator_lib.ModeKeys.TRAIN,
train_op=train_op)
def _get_train_ops(self, features, labels):
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._model.build_model(
features, self._feature_columns, is_training=True)
model_fn_ops = self._head.head_ops(features, labels,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_training_fn, logits=logits)
train_step = self._model.get_train_step(model_fn_ops.loss)
with ops.control_dependencies(train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, model_fn_ops.loss
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
return optimizers.optimize_loss(
loss=loss,
global_step=contrib_variables.get_global_step(),
learning_rate=_LEARNING_RATE,
optimizer=_get_optimizer(optimizer),
gradient_multipliers=(
dnn_linear_combined._extract_embedding_lr_multipliers( # pylint: disable=protected-access
embedding_lr_multipliers, parent_scope, input_layer_scope)),
clip_gradients=gradient_clip_norm,
name=parent_scope,
# Empty summaries to prevent optimizers from logging the training_loss.
summaries=[])
def _logistic_regression_model_fn(features, labels, mode):
_ = mode
logits = layers.linear(
features,
1,
weights_initializer=init_ops.zeros_initializer(),
# Intentionally uses really awful initial values so that
# AUC/precision/recall/etc will change meaningfully even on a toy dataset.
biases_initializer=init_ops.constant_initializer(-10.0))
predictions = math_ops.sigmoid(logits)
loss = loss_ops.sigmoid_cross_entropy(logits, labels)
train_op = optimizers.optimize_loss(loss,
variables.get_global_step(),
optimizer='Adagrad',
learning_rate=0.1)
return predictions, loss, train_op
def sdca_classifier_model_fn(features, targets, mode, params):
"""Estimator's linear model_fn."""
feature_columns = params["feature_columns"]
optimizer = params["optimizer"]
weight_column_name = params["weight_column_name"]
loss_type = params["loss_type"]
enable_centered_bias = params.get("enable_centered_bias", True)
if not isinstance(optimizer, sdca_optimizer.SDCAOptimizer):
raise ValueError("Optimizer must be of type SDCAOptimizer")
loss_fn = {
"logistic_loss": _log_loss_with_two_classes,
"hinge_loss": _hinge_loss,
}[loss_type]
logits, columns_to_variables, bias = (
layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=feature_columns,
num_outputs=1))
train_feature_columns = _maybe_add_bias_column(feature_columns, features,
bias, targets,
enable_centered_bias,
columns_to_variables)
loss = None
if mode != estimator.ModeKeys.INFER:
loss = math_ops.reduce_mean(loss_fn(logits, targets), name="loss")
train_op = None
if mode == estimator.ModeKeys.TRAIN:
global_step = contrib_variables.get_global_step()
# TODO(zoy): Combine linear_feature_columns and columns_to_variables.
train_op = optimizer.get_train_step(train_feature_columns,
weight_column_name, loss_type,
features, targets,
columns_to_variables, global_step)
predictions = {}
predictions[_LOGISTIC] = math_ops.sigmoid(logits)
logits = array_ops.concat(1, [array_ops.zeros_like(logits), logits])
predictions[_PROBABILITIES] = nn.softmax(logits)
predictions[_CLASSES] = math_ops.argmax(logits, 1)
return predictions, loss, train_op
