def test_raise_error_with_custom_init_fn_in_eval(self): def model_fn(features, labels, mode): _, _ = features, labels def init_fn(scaffold, session): _, _ = scaffold, session return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), scaffold=training.Scaffold(init_fn=init_fn), train_op=constant_op.constant(5.), eval_metric_ops={ 'mean_of_features': metrics_lib.mean(constant_op.constant(2.)) }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn) with self.assertRaisesRegexp(ValueError, 'does not support custom init_fn'): evaluator.begin()
def test_raise_error_with_saveables_other_than_global_variables(self): def model_fn(features, labels, mode): _, _ = features, labels w = variables.VariableV1( initial_value=[0.], trainable=False, collections=[ops.GraphKeys.SAVEABLE_OBJECTS]) init_op = control_flow_ops.group( [w.initializer, training.get_global_step().initializer]) return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), scaffold=training.Scaffold(init_op=init_op), train_op=constant_op.constant(5.), eval_metric_ops={ 'mean_of_features': metrics_lib.mean(constant_op.constant(2.)) }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn) with self.assertRaisesRegexp(ValueError, 'does not support saveables'): estimator.train(input_fn, hooks=[evaluator])
def test_dnn_classifier(self): embedding = feature_column_lib.embedding_column( feature_column_lib.categorical_column_with_vocabulary_list( 'wire_cast', ['kima', 'omar', 'stringer']), 8) dnn = estimator_lib.DNNClassifier(feature_columns=[embedding], hidden_units=[3, 1]) def train_input_fn(): return dataset_ops.Dataset.from_tensors(({ 'wire_cast': [['omar'], ['kima']] }, [[0], [1]])).repeat(3) def eval_input_fn(): return dataset_ops.Dataset.from_tensors(({ 'wire_cast': [['stringer'], ['kima']] }, [[0], [1]])).repeat(2) evaluator = hooks_lib.InMemoryEvaluatorHook(dnn, eval_input_fn, name='in-memory') dnn.train(train_input_fn, hooks=[evaluator]) self.assertTrue(os.path.isdir(dnn.eval_dir('in-memory'))) step_keyword_to_value = summary_step_keyword_to_value_mapping( dnn.eval_dir('in-memory')) final_metrics = dnn.evaluate(eval_input_fn) step = final_metrics[ops.GraphKeys.GLOBAL_STEP] for summary_tag in final_metrics: if summary_tag == ops.GraphKeys.GLOBAL_STEP: continue self.assertEqual(final_metrics[summary_tag], step_keyword_to_value[step][summary_tag])
def test_uses_latest_variable_value(self): def model_fn(features, labels, mode): _ = labels step = tf.compat.v1.train.get_global_step() w = tf.compat.v1.get_variable( 'w', shape=[], initializer=tf.compat.v1.initializers.zeros(), dtype=tf.dtypes.int64) if estimator_lib.ModeKeys.TRAIN == mode: # to consume features, we have control dependency with tf.control_dependencies([features]): step_inc = tf.compat.v1.assign_add( tf.compat.v1.train.get_global_step(), 1) with tf.control_dependencies([step_inc]): assign_w_to_step_plus_2 = w.assign(step + 2) return estimator_lib.EstimatorSpec( mode, loss=tf.constant(3.), train_op=assign_w_to_step_plus_2) if estimator_lib.ModeKeys.EVAL == mode: # to consume features, we have control dependency with tf.control_dependencies([features]): loss = tf.constant(5.) mean = metrics_module.Mean() mean.update_state(w) return estimator_lib.EstimatorSpec( mode, loss=loss, # w is constant in each step, so the mean. # w = 0 if step==0 else step+2 eval_metric_ops={'mean_of_const': mean}) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return tf.compat.v1.data.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn, every_n_iter=4) estimator.train(input_fn, hooks=[evaluator]) self.assertTrue(os.path.isdir(estimator.eval_dir())) step_keyword_to_value = summary_step_keyword_to_value_mapping( estimator.eval_dir()) # w = 0 if step==0 else step+2 self.assertEqual(0, step_keyword_to_value[0]['mean_of_const']) self.assertEqual(6, step_keyword_to_value[4]['mean_of_const']) self.assertEqual(12, step_keyword_to_value[10]['mean_of_const'])
def test_runs_eval_metrics(self): def model_fn(features, labels, mode): _ = labels if estimator_lib.ModeKeys.TRAIN == mode: with tf.control_dependencies([features]): train_op = tf.compat.v1.assign_add( tf.compat.v1.train.get_global_step(), 1) return estimator_lib.EstimatorSpec(mode, loss=tf.constant(3.), train_op=train_op) if estimator_lib.ModeKeys.EVAL == mode: mean = metrics_module.Mean() mean.update_state(features) return estimator_lib.EstimatorSpec(mode, loss=tf.constant(5.), eval_metric_ops={ 'mean_of_features': mean, }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return tf.compat.v1.data.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn, every_n_iter=4) estimator.train(input_fn, hooks=[evaluator]) self.assertTrue(os.path.isdir(estimator.eval_dir())) step_keyword_to_value = summary_step_keyword_to_value_mapping( estimator.eval_dir()) # 4.5 = sum(range(10))/10 # before training self.assertEqual(4.5, step_keyword_to_value[0]['mean_of_features']) # intervals (every_n_iter=4) self.assertEqual(4.5, step_keyword_to_value[4]['mean_of_features']) self.assertEqual(4.5, step_keyword_to_value[8]['mean_of_features']) # end self.assertEqual(4.5, step_keyword_to_value[10]['mean_of_features']) self.assertEqual(set([0, 4, 8, 10]), set(step_keyword_to_value.keys()))
def test_raise_error_with_ps(self): tf_config = { 'cluster': { run_config_lib.TaskType.CHIEF: ['host0:0'], run_config_lib.TaskType.PS: ['host1:1'], }, 'task': { 'type': run_config_lib.TaskType.CHIEF, 'index': 0 } } with test.mock.patch.dict('os.environ', {'TF_CONFIG': json.dumps(tf_config)}): dnn = estimator_lib.DNNClassifier( feature_columns=[feature_column_lib.numeric_column('x')], hidden_units=[3, 1]) def eval_input_fn(): pass with self.assertRaisesRegexp(ValueError, 'supports only single machine'): hooks_lib.InMemoryEvaluatorHook(dnn, eval_input_fn)
def test_raise_error_with_custom_saver_in_eval(self): def model_fn(features, labels, mode): _, _ = features, labels mean = metrics_module.Mean() mean.update_state(constant_op.constant(2.)) return estimator_lib.EstimatorSpec( mode, loss=constant_op.constant(3.), scaffold=training.Scaffold(saver=training.Saver()), train_op=constant_op.constant(5.), eval_metric_ops={ 'mean_of_features': mean, }) estimator = estimator_lib.Estimator(model_fn=model_fn) def input_fn(): return dataset_ops.Dataset.range(10) evaluator = hooks_lib.InMemoryEvaluatorHook(estimator, input_fn) with self.assertRaisesRegexp(ValueError, 'does not support custom saver'): evaluator.begin()