def test_export_strategies_reset(self):
for est in self._estimators_for_tests():
export_strategy_1 = make_export_strategy(est, None, exports_to_keep=None)
ex = Experiment(
est,
train_input_fn='train_input',
eval_input_fn='eval_input',
train_steps=100,
eval_steps=100,
export_strategies=(export_strategy_1,))
ex.train_and_evaluate()
self.assertEqual(1, est.export_count)
# After reset with empty list (None), the count does not change and the
# user provided export strategy list should remain intact.
old_es = ex.reset_export_strategies()
ex.train_and_evaluate()
self.assertAllEqual([export_strategy_1], old_es)
self.assertEqual(1, est.export_count)
# After reset with list, the count should increase with the number of
# items.
export_strategy_2 = make_export_strategy(est, None, exports_to_keep=None)
export_strategy_3 = make_export_strategy(est, None, exports_to_keep=None)
old_es = ex.reset_export_strategies([export_strategy_2, export_strategy_3])
ex.train_and_evaluate()
self.assertAllEqual([], old_es)
self.assertEqual(3, est.export_count)
def test_test(self):
for est in self._estimators_for_tests():
export_strategy = make_export_strategy(est, None, exports_to_keep=None)
ex = Experiment(est, train_input_fn='train_input', eval_input_fn='eval_input',
export_strategies=export_strategy)
ex.test()
self.assertEqual(1, est.fit_count)
self.assertEqual(1, est.eval_count)
self.assertEqual(1, est.export_count)
def test_continuous_train_and_eval(self):
for est in self._estimators_for_tests(eval_dict={'global_step': 100}):
noop_hook = _NoopHook()
export_strategy = make_export_strategy(est, None, exports_to_keep=None)
ex = Experiment(est, train_input_fn='train_input', eval_input_fn='eval_input',
eval_hooks=[noop_hook], train_steps=100, eval_steps=100,
export_strategies=export_strategy)
ex.continuous_train_and_evaluate()
self.assertEqual(1, est.fit_count)
self.assertEqual(1, est.eval_count)
self.assertEqual(1, est.export_count)
self.assertEqual([noop_hook], est.eval_hooks)
def test_test(self):
for est in self._estimators_for_tests():
export_strategy = make_export_strategy(est,
None,
exports_to_keep=None)
ex = Experiment(est,
train_input_fn='train_input',
eval_input_fn='eval_input',
export_strategies=export_strategy)
ex.test()
self.assertEqual(1, est.fit_count)
self.assertEqual(1, est.eval_count)
self.assertEqual(1, est.export_count)
def test_export_strategies_reset(self):
for est in self._estimators_for_tests():
export_strategy_1 = make_export_strategy(est,
None,
exports_to_keep=None)
ex = Experiment(est,
train_input_fn='train_input',
eval_input_fn='eval_input',
train_steps=100,
eval_steps=100,
export_strategies=(export_strategy_1, ))
ex.train_and_evaluate()
self.assertEqual(1, est.export_count)
# After reset with empty list (None), the count does not change and the
# user provided export strategy list should remain intact.
old_es = ex.reset_export_strategies()
ex.train_and_evaluate()
self.assertAllEqual([export_strategy_1], old_es)
self.assertEqual(1, est.export_count)
# After reset with list, the count should increase with the number of
# items.
export_strategy_2 = make_export_strategy(est,
None,
exports_to_keep=None)
export_strategy_3 = make_export_strategy(est,
None,
exports_to_keep=None)
old_es = ex.reset_export_strategies(
[export_strategy_2, export_strategy_3])
ex.train_and_evaluate()
self.assertAllEqual([], old_es)
self.assertEqual(3, est.export_count)
def test_continuous_train_and_eval(self):
for est in self._estimators_for_tests(eval_dict={'global_step': 100}):
noop_hook = _NoopHook()
export_strategy = make_export_strategy(est,
None,
exports_to_keep=None)
ex = Experiment(est,
train_input_fn='train_input',
eval_input_fn='eval_input',
eval_hooks=[noop_hook],
train_steps=100,
eval_steps=100,
export_strategies=export_strategy)
ex.continuous_train_and_evaluate()
self.assertEqual(1, est.fit_count)
self.assertEqual(1, est.eval_count)
self.assertEqual(1, est.export_count)
self.assertEqual([noop_hook], est.eval_hooks)
def test_continuous_train_and_eval_with_predicate_fn(self):
for est in self._estimators_for_tests(eval_dict={'global_step': 100}):
export_strategy = make_export_strategy(est, None, exports_to_keep=None)
ex = Experiment(
est,
train_input_fn='train_input',
eval_input_fn='eval_input',
train_steps=100000000000, # a value will make `ex` never stops.
eval_steps=100,
export_strategies=export_strategy)
def predicate_fn(eval_result):
del eval_result # unused. for fn signature.
return False
ex.continuous_train_and_evaluate(continuous_eval_predicate_fn=predicate_fn)
self.assertEqual(0, est.fit_count)
self.assertEqual(0, est.eval_count)
self.assertEqual(1, est.export_count)
def test_continuous_train_and_eval_with_predicate_fn(self):
for est in self._estimators_for_tests(eval_dict={'global_step': 100}):
export_strategy = make_export_strategy(est,
None,
exports_to_keep=None)
ex = Experiment(
est,
train_input_fn='train_input',
eval_input_fn='eval_input',
train_steps=100000000000, # a value will make `ex` never stops.
eval_steps=100,
export_strategies=export_strategy)
def predicate_fn(eval_result):
del eval_result # unused. for fn signature.
return False
ex.continuous_train_and_evaluate(
continuous_eval_predicate_fn=predicate_fn)
self.assertEqual(0, est.fit_count)
self.assertEqual(0, est.eval_count)
self.assertEqual(1, est.export_count)
