def test_raise_error_with_custom_init_fn_in_eval(self):
def model_fn(features, labels, mode):
_, _ = features, labels
def init_fn(scaffold, session):
_, _ = scaffold, session
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant(3.),
scaffold=training.Scaffold(init_fn=init_fn),
train_op=constant_op.constant(5.),
eval_metric_ops={
'mean_of_features':
metrics_lib.mean(constant_op.constant(2.))
})
estimator = estimator_lib.Estimator(model_fn=model_fn)
def input_fn():
return dataset_ops.Dataset.range(10)
evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn)
with self.assertRaisesRegexp(ValueError,
'does not support custom init_fn'):
evaluator.begin()
def test_raise_error_with_saveables_other_than_global_variables(self):
def model_fn(features, labels, mode):
_, _ = features, labels
w = variables.VariableV1(
initial_value=[0.],
trainable=False,
collections=[ops.GraphKeys.SAVEABLE_OBJECTS])
init_op = control_flow_ops.group(
[w.initializer,
training.get_global_step().initializer])
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant(3.),
scaffold=training.Scaffold(init_op=init_op),
train_op=constant_op.constant(5.),
eval_metric_ops={
'mean_of_features':
metrics_lib.mean(constant_op.constant(2.))
})
estimator = estimator_lib.Estimator(model_fn=model_fn)
def input_fn():
return dataset_ops.Dataset.range(10)
evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn)
with self.assertRaisesRegexp(ValueError, 'does not support saveables'):
estimator.train(input_fn, hooks=[evaluator])
def _build_estimator(self):
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
distribute = tf.distribute.MirroredStrategy(
) if self._flags.distribute else None
config = estimator.RunConfig(
model_dir=self._flags.checkpoint_dir,
save_summary_steps=self._flags.save_summary_steps,
save_checkpoints_steps=self._flags.checkpoints_steps,
keep_checkpoint_max=2,
session_config=self.sess_config,
log_step_count_steps=self._flags.log_step_count_steps,
train_distribute=distribute,
eval_distribute=distribute)
warm_start_setting = None
if os.path.exists(self._flags.warm_start_dir):
vars_to_warm_start = '.*/embeddings[^/]' if self._flags.warm_start_mode == 'emb' else '.*'
warm_start_setting = estimator.WarmStartSettings(
ckpt_to_initialize_from=self._flags.warm_start_dir,
vars_to_warm_start=vars_to_warm_start)
self.estimator = estimator.Estimator(
self._build_model_fn(),
config=config,
warm_start_from=warm_start_setting)
def test_uses_latest_variable_value(self):
def model_fn(features, labels, mode):
_ = labels
step = tf.compat.v1.train.get_global_step()
w = tf.compat.v1.get_variable(
'w',
shape=[],
initializer=tf.compat.v1.initializers.zeros(),
dtype=tf.dtypes.int64)
if estimator_lib.ModeKeys.TRAIN == mode:
# to consume features, we have control dependency
with tf.control_dependencies([features]):
step_inc = tf.compat.v1.assign_add(
tf.compat.v1.train.get_global_step(), 1)
with tf.control_dependencies([step_inc]):
assign_w_to_step_plus_2 = w.assign(step + 2)
return estimator_lib.EstimatorSpec(
mode,
loss=tf.constant(3.),
train_op=assign_w_to_step_plus_2)
if estimator_lib.ModeKeys.EVAL == mode:
# to consume features, we have control dependency
with tf.control_dependencies([features]):
loss = tf.constant(5.)
mean = metrics_module.Mean()
mean.update_state(w)
return estimator_lib.EstimatorSpec(
mode,
loss=loss,
# w is constant in each step, so the mean.
# w = 0 if step==0 else step+2
eval_metric_ops={'mean_of_const': mean})
estimator = estimator_lib.Estimator(model_fn=model_fn)
def input_fn():
return tf.compat.v1.data.Dataset.range(10)
evaluator = hooks_lib.InMemoryEvaluatorHook(estimator,
input_fn,
every_n_iter=4)
estimator.train(input_fn, hooks=[evaluator])
self.assertTrue(os.path.isdir(estimator.eval_dir()))
step_keyword_to_value = summary_step_keyword_to_value_mapping(
estimator.eval_dir())
# w = 0 if step==0 else step+2
self.assertEqual(0, step_keyword_to_value[0]['mean_of_const'])
self.assertEqual(6, step_keyword_to_value[4]['mean_of_const'])
self.assertEqual(12, step_keyword_to_value[10]['mean_of_const'])
def test_should_not_conflict_with_existing_predictions(self):
def input_fn():
return {'x': [[3.], [5.]], 'id': [[101], [102]]}
def model_fn(features, mode):
del features
global_step = training.get_global_step()
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant([5.]),
predictions={'x': constant_op.constant([5.])},
train_op=global_step.assign_add(1))
estimator = estimator_lib.Estimator(model_fn=model_fn)
estimator.train(input_fn=input_fn, steps=1)
estimator = extenders.forward_features(estimator)
with self.assertRaisesRegexp(ValueError, 'Cannot forward feature key'):
next(estimator.predict(input_fn=input_fn))
def test_runs_eval_metrics(self):
def model_fn(features, labels, mode):
_ = labels
if estimator_lib.ModeKeys.TRAIN == mode:
with tf.control_dependencies([features]):
train_op = tf.compat.v1.assign_add(
tf.compat.v1.train.get_global_step(), 1)
return estimator_lib.EstimatorSpec(mode,
loss=tf.constant(3.),
train_op=train_op)
if estimator_lib.ModeKeys.EVAL == mode:
mean = metrics_module.Mean()
mean.update_state(features)
return estimator_lib.EstimatorSpec(mode,
loss=tf.constant(5.),
eval_metric_ops={
'mean_of_features':
mean,
})
estimator = estimator_lib.Estimator(model_fn=model_fn)
def input_fn():
return tf.compat.v1.data.Dataset.range(10)
evaluator = hooks_lib.InMemoryEvaluatorHook(estimator,
input_fn,
every_n_iter=4)
estimator.train(input_fn, hooks=[evaluator])
self.assertTrue(os.path.isdir(estimator.eval_dir()))
step_keyword_to_value = summary_step_keyword_to_value_mapping(
estimator.eval_dir())
# 4.5 = sum(range(10))/10
# before training
self.assertEqual(4.5, step_keyword_to_value[0]['mean_of_features'])
# intervals (every_n_iter=4)
self.assertEqual(4.5, step_keyword_to_value[4]['mean_of_features'])
self.assertEqual(4.5, step_keyword_to_value[8]['mean_of_features'])
# end
self.assertEqual(4.5, step_keyword_to_value[10]['mean_of_features'])
self.assertEqual(set([0, 4, 8, 10]), set(step_keyword_to_value.keys()))
def test_raise_error_with_custom_saver_in_eval(self):
def model_fn(features, labels, mode):
_, _ = features, labels
mean = metrics_module.Mean()
mean.update_state(constant_op.constant(2.))
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant(3.),
scaffold=training.Scaffold(saver=training.Saver()),
train_op=constant_op.constant(5.),
eval_metric_ops={
'mean_of_features': mean,
})
estimator = estimator_lib.Estimator(model_fn=model_fn)
def input_fn():
return dataset_ops.Dataset.range(10)
evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn)
with self.assertRaisesRegexp(ValueError,
'does not support custom saver'):
evaluator.begin()
