def test_raise_error_with_saveables_other_than_global_variables(self): def model_fn(features, labels, mode): _, _ = features, labels w = variables.VariableV1( initial_value=[0.], trainable=False, collections=[ops.GraphKeys.SAVEABLE_OBJECTS]) init_op = control_flow_ops.group( [w.initializer, training.get_global_step().initializer]) return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), scaffold=training.Scaffold(init_op=init_op), train_op=constant_op.constant(5.), eval_metric_ops={ 'mean_of_features': metrics_lib.mean(constant_op.constant(2.)) }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn) with self.assertRaisesRegexp(ValueError, 'does not support saveables'): estimator.train(input_fn, hooks=[evaluator])
def test_raise_error_with_custom_init_fn_in_eval(self): def model_fn(features, labels, mode): _, _ = features, labels def init_fn(scaffold, session): _, _ = scaffold, session return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), scaffold=training.Scaffold(init_fn=init_fn), train_op=constant_op.constant(5.), eval_metric_ops={ 'mean_of_features': metrics_lib.mean(constant_op.constant(2.)) }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn) with self.assertRaisesRegexp(ValueError, 'does not support custom init_fn'): evaluator.begin()
def test_dnn_classifier(self): embedding = feature_column_lib.embedding_column( feature_column_lib.categorical_column_with_vocabulary_list( 'wire_cast', ['kima', 'omar', 'stringer']), 8) dnn = estimator_lib.DNNClassifier(feature_columns=[embedding], hidden_units=[3, 1]) def train_input_fn(): return dataset_ops.Dataset.from_tensors(({ 'wire_cast': [['omar'], ['kima']] }, [[0], [1]])).repeat(3) def eval_input_fn(): return dataset_ops.Dataset.from_tensors(({ 'wire_cast': [['stringer'], ['kima']] }, [[0], [1]])).repeat(2) evaluator = hooks_lib.InMemoryEvaluatorHook(dnn, eval_input_fn, name='in-memory') dnn.train(train_input_fn, hooks=[evaluator]) self.assertTrue(os.path.isdir(dnn.eval_dir('in-memory'))) step_keyword_to_value = summary_step_keyword_to_value_mapping( dnn.eval_dir('in-memory')) final_metrics = dnn.evaluate(eval_input_fn) step = final_metrics[ops.GraphKeys.GLOBAL_STEP] for summary_tag in final_metrics: if summary_tag == ops.GraphKeys.GLOBAL_STEP: continue self.assertEqual(final_metrics[summary_tag], step_keyword_to_value[step][summary_tag])
def test_uses_latest_variable_value(self): def model_fn(features, labels, mode): _ = labels step = training.get_global_step() w = variable_scope.get_variable( 'w', shape=[], initializer=init_ops.zeros_initializer(), dtype=dtypes.int64) if estimator_lib.ModeKeys.TRAIN == mode: # to consume features, we have control dependency with ops.control_dependencies([features]): step_inc = state_ops.assign_add(training.get_global_step(), 1) with ops.control_dependencies([step_inc]): assign_w_to_step_plus_2 = w.assign(step + 2) return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), train_op=assign_w_to_step_plus_2) if estimator_lib.ModeKeys.EVAL == mode: # to consume features, we have control dependency with ops.control_dependencies([features]): loss = constant_op.constant(5.) return estimator_lib.EstimatorSpec( mode, loss=loss, # w is constant in each step, so the mean. # w = 0 if step==0 else step+2 eval_metric_ops={'mean_of_const': metrics_lib.mean(w)}) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn, every_n_iter=4) estimator.train(input_fn, hooks=[evaluator]) self.assertTrue(os.path.isdir(estimator.eval_dir())) step_keyword_to_value = summary_step_keyword_to_value_mapping( estimator.eval_dir()) # w = 0 if step==0 else step+2 self.assertEqual(0, step_keyword_to_value[0]['mean_of_const']) self.assertEqual(6, step_keyword_to_value[4]['mean_of_const']) self.assertEqual(12, step_keyword_to_value[10]['mean_of_const'])
def test_runs_eval_metrics(self): def model_fn(features, labels, mode): _ = labels if estimator_lib.ModeKeys.TRAIN == mode: with ops.control_dependencies([features]): train_op = state_ops.assign_add(training.get_global_step(), 1) return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), train_op=train_op) if estimator_lib.ModeKeys.EVAL == mode: return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(5.), eval_metric_ops={ 'mean_of_features': metrics_lib.mean(features) }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn, every_n_iter=4) estimator.train(input_fn, hooks=[evaluator]) self.assertTrue(os.path.isdir(estimator.eval_dir())) step_keyword_to_value = summary_step_keyword_to_value_mapping( estimator.eval_dir()) # 4.5 = sum(range(10))/10 # before training self.assertEqual(4.5, step_keyword_to_value[0]['mean_of_features']) # intervals (every_n_iter=4) self.assertEqual(4.5, step_keyword_to_value[4]['mean_of_features']) self.assertEqual(4.5, step_keyword_to_value[8]['mean_of_features']) # end self.assertEqual(4.5, step_keyword_to_value[10]['mean_of_features']) self.assertEqual(set([0, 4, 8, 10]), set(step_keyword_to_value.keys()))
def test_raise_error_with_ps(self): tf_config = { 'cluster': { run_config_lib.TaskType.CHIEF: ['host0:0'], run_config_lib.TaskType.PS: ['host1:1'], }, 'task': { 'type': run_config_lib.TaskType.CHIEF, 'index': 0 } } with test.mock.patch.dict('os.environ', {'TF_CONFIG': json.dumps(tf_config)}): dnn = estimator_lib.DNNClassifier( feature_columns=[feature_column_lib.numeric_column('x')], hidden_units=[3, 1]) def eval_input_fn(): pass with self.assertRaisesRegexp(ValueError, 'supports only single machine'): hooks_lib.InMemoryEvaluatorHook(dnn, eval_input_fn)