def test_train_max_steps_is_not_incremental(self): with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) learn.graph_actions.train( g, output_dir=self._output_dir, train_op=train_op, loss_op=constant_op.constant(2.0), max_steps=10) step = checkpoint_utils.load_variable( self._output_dir, variables_lib.get_global_step().name) self.assertEqual(10, step) with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) learn.graph_actions.train( g, output_dir=self._output_dir, train_op=train_op, loss_op=constant_op.constant(2.0), max_steps=15) step = checkpoint_utils.load_variable( self._output_dir, variables_lib.get_global_step().name) self.assertEqual(15, step)
def test_train_skip_train_if_max_step_already_saved(self): with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) learn.graph_actions._monitored_train( # pylint: disable=protected-access g, output_dir=self._output_dir, train_op=train_op, loss_op=constant_op.constant(2.0), max_steps=10) step = checkpoint_utils.load_variable( self._output_dir, variables_lib.get_global_step().name) self.assertEqual(10, step) with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) learn.graph_actions._monitored_train( # pylint: disable=protected-access g, output_dir=self._output_dir, train_op=train_op, loss_op=constant_op.constant(2.0), max_steps=10) step = checkpoint_utils.load_variable( self._output_dir, variables_lib.get_global_step().name) self.assertEqual(10, step)
def _make_training_op(training_loss): """Training op for the DNN linear combined model.""" train_ops = [] if dnn_logits is not None: train_ops.append( optimizers.optimize_loss( loss=training_loss, global_step=contrib_variables.get_global_step(), learning_rate=_DNN_LEARNING_RATE, optimizer=_get_optimizer(dnn_optimizer), gradient_multipliers=_extract_embedding_lr_multipliers( # pylint: disable=protected-access embedding_lr_multipliers, dnn_parent_scope, dnn_input_scope.name), clip_gradients=gradient_clip_norm, variables=ops.get_collection(dnn_parent_scope), name=dnn_parent_scope, # Empty summaries, because head already logs "loss" summary. summaries=[])) if linear_logits is not None: train_ops.append( optimizers.optimize_loss( loss=training_loss, global_step=contrib_variables.get_global_step(), learning_rate=_linear_learning_rate(len(linear_feature_columns)), optimizer=_get_optimizer(linear_optimizer), clip_gradients=gradient_clip_norm, variables=ops.get_collection(linear_parent_scope), name=linear_parent_scope, # Empty summaries, because head already logs "loss" summary. summaries=[])) return control_flow_ops.group(*train_ops)
def test_get_global_step(self): with ops.Graph().as_default() as g: self.assertEquals(None, variables_lib2.get_global_step()) variables_lib.Variable( 0, trainable=False, dtype=dtypes.int32, name=ops.GraphKeys.GLOBAL_STEP) self._assert_global_step( variables_lib2.get_global_step(), expected_dtype=dtypes.int32) self._assert_global_step( variables_lib2.get_global_step(g), expected_dtype=dtypes.int32)
def _get_train_ops(self, features, targets): """See base class.""" if self._linear_feature_columns is None: self._linear_feature_columns = layers.infer_real_valued_columns( features) if not isinstance(self._linear_optimizer, sdca_optimizer.SDCAOptimizer): return super(LinearClassifier, self)._get_train_ops(features, targets) # SDCA currently supports binary classification only. if self._n_classes > 2: raise ValueError( "SDCA does not currently support multi-class classification.") global_step = contrib_variables.get_global_step() assert global_step logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=self._linear_feature_columns, num_outputs=self._num_label_columns(), weight_collections=[self._linear_weight_collection], name="linear") with ops.control_dependencies([self._centered_bias()]): loss = self._loss(logits, targets, self._get_weight_tensor(features)) logging_ops.scalar_summary("loss", loss) train_ops = self._linear_optimizer.get_train_step( self._linear_feature_columns, self._weight_column_name, "logistic_loss", features, targets, columns_to_variables, global_step) return train_ops, loss
def gmm_cluster_model_fn(features, labels, mode, params, config=None): """Model function.""" assert labels is None, labels update_params = '' for i in ["w", "m", 'c']: if i in params["update_params"]: update_params += i (all_scores, model_predictions, losses, training_op) = gmm_ops.gmm(parse_tensor_or_dict(features), "random", params["num_clusters"], params["random_seed"], params["covariance_type"], update_params, ) incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses) training_op = with_dependencies([training_op, incr_step], loss) predictions = { 'all_scores': all_scores[0], 'assignments': model_predictions[0][0], } eval_metric_ops = { 'scores': streaming_sum(loss), } return model_fn_lib.ModelFnOps(mode=mode, predictions=predictions, eval_metric_ops=eval_metric_ops, loss=loss, train_op=training_op)
def begin(self): self._last_saved_step = None self._request_summary = True self._global_step_tensor = contrib_variables.get_global_step() if self._global_step_tensor is None: raise RuntimeError( "Global step should be created to use SummarySaverHook.")
def begin(self): self._last_reported_time = None self._last_reported_step = None self._global_step_tensor = contrib_variables.get_global_step() if self._global_step_tensor is None: raise RuntimeError( "Global step should be created to use StepCounterHook.")
def begin(self): self._last_saved_time = None self._last_saved_step = None self._global_step_tensor = contrib_variables.get_global_step() if self._global_step_tensor is None: raise RuntimeError( "Global step should be created to use CheckpointSaverHook.")
def _model_fn(features, labels, mode): """Model function.""" assert labels is None, labels (all_scores, model_predictions, losses, training_op) = clustering_ops.KMeans( self._parse_tensor_or_dict(features), self._num_clusters, self._training_initial_clusters, self._distance_metric, self._use_mini_batch, random_seed=self._random_seed, kmeans_plus_plus_num_retries=self. _kmeans_plus_plus_num_retries).training_graph() incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME) logging_ops.scalar_summary('loss/raw', loss) training_op = with_dependencies([training_op, incr_step], loss) predictions = { KMeansClustering.ALL_SCORES: all_scores[0], KMeansClustering.CLUSTER_IDX: model_predictions[0], } eval_metric_ops = {KMeansClustering.SCORES: loss,} return ModelFnOps( mode=mode, predictions=predictions, eval_metric_ops=eval_metric_ops, loss=loss, train_op=training_op)
def _get_train_ops(self, features, targets): """See base class.""" global_step = contrib_variables.get_global_step() assert global_step logits = self._logits(features, is_training=True) with ops.control_dependencies([ self._centered_bias_step(targets, self._get_weight_tensor(features)) ]): loss = self._loss(logits, targets, self._get_weight_tensor(features)) logging_ops.scalar_summary("loss", loss) linear_vars = self._get_linear_vars() dnn_vars = self._get_dnn_vars() grads = gradients.gradients(loss, dnn_vars + linear_vars) dnn_grads = grads[0:len(dnn_vars)] linear_grads = grads[len(dnn_vars):] train_ops = self._get_linear_training_ops( linear_grads, linear_vars) + self._get_dnn_training_ops( dnn_grads, dnn_vars) train_step = control_flow_ops.group(*train_ops, name="combined_training_op") with ops.control_dependencies([train_step]): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, loss
def begin(self): self._last_reported_time = None self._last_reported_step = None self._global_step_tensor = contrib_variables.get_global_step() if self._global_step_tensor is None: raise RuntimeError( "Global step should be created to use StepCounterHook.")
def _get_train_ops(self, features, targets): """See base class.""" self._validate_linear_feature_columns(features) if not isinstance(self._linear_optimizer, sdca_optimizer.SDCAOptimizer): return super(LinearClassifier, self)._get_train_ops(features, targets) # SDCA currently supports binary classification only. if self._target_column.num_label_columns > 2: raise ValueError( "SDCA does not currently support multi-class classification.") global_step = contrib_variables.get_global_step() assert global_step logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=self._linear_feature_columns, num_outputs=self._target_column.num_label_columns, weight_collections=[self._linear_weight_collection], name="linear") with ops.control_dependencies([self._centered_bias()]): loss = self._loss(logits, targets, features) logging_ops.scalar_summary("loss", loss) train_ops = self._linear_optimizer.get_train_step( self._linear_feature_columns, self._target_column.weight_column_name, "logistic_loss", features, targets, columns_to_variables, global_step) return train_ops, loss
def begin(self): self._last_saved_step = None self._request_summary = True self._global_step_tensor = contrib_variables.get_global_step() if self._global_step_tensor is None: raise RuntimeError( "Global step should be created to use SummarySaverHook.")
def _get_train_ops(self, features, targets): """See base class.""" global_step = contrib_variables.get_global_step() assert global_step features = self._get_feature_dict(features) logits = self._logits(features, is_training=True) if self._enable_centered_bias: centered_bias_step = [self._centered_bias_step(targets, features)] else: centered_bias_step = [] with ops.control_dependencies(centered_bias_step): training_loss = self._target_column.training_loss(logits, targets, features) weighted_average_loss = self._target_column.loss(logits, targets, features) logging_ops.scalar_summary("loss", weighted_average_loss) linear_train_step = self._linear_model.get_train_step(training_loss) dnn_train_step = (self._dnn_model.get_train_step(training_loss) if self._dnn_model else []) with ops.control_dependencies(linear_train_step + dnn_train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, weighted_average_loss
def begin(self): self._last_saved_time = None self._last_saved_step = None self._global_step_tensor = contrib_variables.get_global_step() if self._global_step_tensor is None: raise RuntimeError( "Global step should be created to use CheckpointSaverHook.")
def test_train_chief_monitor(self): with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add( variables_lib.get_global_step(), 1) loss_op = constant_op.constant(2.0) summary.scalar('loss', loss_op) chief_exclusive_monitor = _BaseMonitorWrapper(False) all_workers_monitor = _BaseMonitorWrapper(True) loss = learn.graph_actions.train( g, output_dir=self._output_dir, train_op=train_op, loss_op=loss_op, supervisor_is_chief=True, steps=1, monitors=[chief_exclusive_monitor, all_workers_monitor]) self.assertEqual(2.0, loss) self.assertTrue( chief_exclusive_monitor.is_active and all_workers_monitor.is_active, 'All monitors must have been active.') self.assertTrue( chief_exclusive_monitor.has_step and all_workers_monitor.has_step, 'All monitors must have a step.')
def _model_fn(features, labels, mode, config): """Model function.""" assert labels is None, labels (all_scores, model_predictions, losses, training_op, init_op, is_initialized) = gmm_ops.gmm(self._parse_tensor_or_dict(features), self._training_initial_clusters, self._num_clusters, self._random_seed, self._covariance_type, self._params) incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses) training_op = with_dependencies([training_op, incr_step], loss) training_hooks = [_InitializeClustersHook( init_op, is_initialized, config.is_chief)] predictions = { GMM.ALL_SCORES: all_scores[0], GMM.ASSIGNMENTS: model_predictions[0][0], } eval_metric_ops = { GMM.SCORES: _streaming_sum(loss), } return model_fn_lib.ModelFnOps(mode=mode, predictions=predictions, eval_metric_ops=eval_metric_ops, loss=loss, train_op=training_op, training_hooks=training_hooks)
def _get_train_ops(self, features, targets): """See base class.""" global_step = contrib_variables.get_global_step() assert global_step logits = self._logits(features, is_training=True) if self._enable_centered_bias: centered_bias_step = [self._centered_bias_step(targets, features)] else: centered_bias_step = [] with ops.control_dependencies(centered_bias_step): loss = self._loss(logits, targets, features) logging_ops.scalar_summary("loss", loss) linear_vars = self._get_linear_vars() dnn_vars = self._get_dnn_vars() grads = gradients.gradients(loss, dnn_vars + linear_vars) if self._gradient_clip_norm: grads, _ = clip_ops.clip_by_global_norm(grads, self._gradient_clip_norm) dnn_grads = grads[0 : len(dnn_vars)] linear_grads = grads[len(dnn_vars) :] train_ops = self._get_linear_training_ops(linear_grads, linear_vars) + self._get_dnn_training_ops( dnn_grads, dnn_vars ) train_step = control_flow_ops.group(*train_ops, name="combined_training_op") with ops.control_dependencies([train_step]): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, loss
def _invalid_model_fn(features, labels): # pylint: disable=unused-argument w = variables_lib.Variable(42.0, 'weight') update_global_step = variables.get_global_step().assign_add(1) with control_flow_ops.control_dependencies([update_global_step]): loss = 100.0 - w return None, loss, None
def test_train_summaries(self): with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add( variables_lib.get_global_step(), 1) loss_op = constant_op.constant(2.0) summary.scalar('loss', loss_op) writer = learn.graph_actions.get_summary_writer(self._output_dir) self._assert_summaries(self._output_dir, writer) self._assert_ckpt(self._output_dir, False) loss = learn.graph_actions._monitored_train( # pylint: disable=protected-access g, output_dir=self._output_dir, train_op=train_op, loss_op=loss_op, steps=1) meta_graph_def = meta_graph.create_meta_graph_def( graph_def=g.as_graph_def(add_shapes=True), saver_def=monitored_session.Scaffold().finalize( ).saver.saver_def) self.assertEqual(2.0, loss) self._assert_summaries(self._output_dir, writer, expected_graphs=[g], expected_meta_graphs=[meta_graph_def], expected_summaries={1: { 'loss': 2.0 }}) self._assert_ckpt(self._output_dir, True)
def test_train_summaries(self): with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) loss_op = constant_op.constant(2.0) summary.scalar('loss', loss_op) self._assert_summaries(self._output_dir) self._assert_ckpt(self._output_dir, False) loss = learn.graph_actions.train( g, output_dir=self._output_dir, train_op=train_op, loss_op=loss_op, steps=1) # TODO(ebrevdo,ptucker,ispir): this meta_graph_def lacks the # SaverDef, so we can't add it to the summary assertion test below. # meta_graph_def = meta_graph.create_meta_graph_def() self.assertEqual(2.0, loss) self._assert_summaries( self._output_dir, expected_graphs=[g], expected_summaries={1: { 'loss': 2.0 }}) self._assert_ckpt(self._output_dir, True)
def _model_fn(features, labels, mode): """Model function.""" assert labels is None, labels (all_scores, model_predictions, losses, training_op) = clustering_ops.KMeans( self._parse_tensor_or_dict(features), self._num_clusters, self._training_initial_clusters, self._distance_metric, self._use_mini_batch, random_seed=self._random_seed, kmeans_plus_plus_num_retries=self. _kmeans_plus_plus_num_retries).training_graph() incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME) logging_ops.scalar_summary('loss/raw', loss) training_op = with_dependencies([training_op, incr_step], loss) predictions = { KMeansClustering.ALL_SCORES: all_scores[0], KMeansClustering.CLUSTER_IDX: model_predictions[0], } eval_metric_ops = {KMeansClustering.SCORES: loss,} if self._relative_tolerance is not None: training_hooks = [self.LossRelativeChangeHook(self._relative_tolerance)] else: training_hooks = None return ModelFnOps( mode=mode, predictions=predictions, eval_metric_ops=eval_metric_ops, loss=loss, train_op=training_op, training_hooks=training_hooks)
def test_train_summaries(self): with ops.Graph().as_default() as g, self.test_session(g): with ops.control_dependencies(self._build_inference_graph()): train_op = state_ops.assign_add(variables_lib.get_global_step(), 1) loss_op = constant_op.constant(2.0) summary.scalar('loss', loss_op) writer = learn.graph_actions.get_summary_writer(self._output_dir) self._assert_summaries(self._output_dir, writer) self._assert_ckpt(self._output_dir, False) loss = learn.graph_actions._monitored_train( # pylint: disable=protected-access g, output_dir=self._output_dir, train_op=train_op, loss_op=loss_op, steps=1) meta_graph_def = meta_graph.create_meta_graph_def( graph_def=g.as_graph_def(add_shapes=True), saver_def=monitored_session.Scaffold().finalize().saver.saver_def) self.assertEqual(2.0, loss) self._assert_summaries( self._output_dir, writer, expected_graphs=[g], expected_meta_graphs=[meta_graph_def], expected_summaries={1: { 'loss': 2.0 }}) self._assert_ckpt(self._output_dir, True)
def _get_train_ops(self, features, targets): """See base class.""" global_step = contrib_variables.get_global_step() assert global_step features = self._get_feature_dict(features) logits = self._logits(features, is_training=True) if self._enable_centered_bias: centered_bias_step = [self._centered_bias_step(targets, features)] else: centered_bias_step = [] with ops.control_dependencies(centered_bias_step): training_loss = self._target_column.training_loss(logits, targets, features) weighted_average_loss = self._target_column.loss(logits, targets, features) logging_ops.scalar_summary("loss", weighted_average_loss) linear_train_step = self._linear_model.get_train_step(training_loss) dnn_train_step = (self._dnn_model.get_train_step(training_loss) if self._dnn_model else []) with ops.control_dependencies(linear_train_step + dnn_train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, weighted_average_loss
def _argument_checker(features, labels, mode, params, config=None, model_dir=None): _, _, _ = features, labels, config self.assertEqual(model_fn.ModeKeys.TRAIN, mode) self.assertEqual(expected_param, params) self.assertEqual(model_dir, expected_model_dir) return (constant_op.constant(0.), constant_op.constant(0.), variables.get_global_step().assign_add(1))
def _train_op_fn(loss): global_step = contrib_variables.get_global_step() my_vars = ops.get_collection(parent_scope) grads = gradients.gradients(loss, my_vars) if gradient_clip_norm: grads, _ = clip_ops.clip_by_global_norm(grads, gradient_clip_norm) return (_get_optimizer(optimizer).apply_gradients( zip(grads, my_vars), global_step=global_step))
def model_fn(features, labels): # dummy variable: _ = variables_lib.Variable([0.]) _ = labels predictions = features["x"] loss = constant_op.constant([2.]) update_global_step = variables.get_global_step().assign_add(1) return predictions, loss, update_global_step
def _train_op_fn(loss): global_step = contrib_variables.get_global_step() assert global_step train_step = model.get_train_step(loss) with ops.control_dependencies(train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op
def _train_op_fn(loss): global_step = contrib_variables.get_global_step() my_vars = ops.get_collection("linear") grads = gradients.gradients(loss, my_vars) if gradient_clip_norm: grads, _ = clip_ops.clip_by_global_norm(grads, gradient_clip_norm) return (_get_optimizer(optimizer).apply_gradients( zip(grads, my_vars), global_step=global_step))
def _train_op_fn(loss): global_step = contrib_variables.get_global_step() assert global_step train_step = model.get_train_step(loss) with ops.control_dependencies(train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op
def _train_op_fn(unused_loss): global_step = contrib_variables.get_global_step() sdca_model, train_op = optimizer.get_train_step( columns_to_variables, weight_column_name, loss_type, features, targets, global_step) if update_weights_hook is not None: update_weights_hook.set_parameters(sdca_model, train_op) return train_op
def _model_fn_scaffold(features, labels, mode): _, _ = features, labels return model_fn.ModelFnOps( mode=mode, predictions=constant_op.constant(0.), loss=constant_op.constant(0.), train_op=variables.get_global_step().assign_add(1), scaffold=monitored_session.Scaffold(init_fn=_init_fn))
def _train_op_fn(unused_loss): global_step = contrib_variables.get_global_step() sdca_model, train_op = optimizer.get_train_step( columns_to_variables, weight_column_name, loss_type, features, labels, global_step) if update_weights_hook is not None: update_weights_hook.set_parameters(sdca_model, train_op) return train_op
def model_fn(features, labels): # dummy variable: _ = variables_lib.Variable([0.]) _ = labels predictions = features["x"] loss = constant_op.constant([2.]) update_global_step = variables.get_global_step().assign_add(1) return predictions, loss, update_global_step
def dynamic_model_fn(features, targets, mode, params): VOCAB_SIZE = len(params['token_map']) embeddings = embedding_matrix(VOCAB_SIZE, params['embedding_dim']) image = features['image'] visual_features = convolutional_features(tf.expand_dims(image, -1), params['vdim']) lstm = LSTMSingleton.get_instance(params['hdim']) if mode is tf.contrib.learn.ModeKeys.TRAIN: target_tokens = targets['target'] # Decoder inputs should be in time-major order. decoder_inputs = tf.to_int64(array_ops.transpose(target_tokens, [1, 0])) sequence_length = targets['sequence_length'] feed_prev_output = params['feed_prev_output'] else: # Doesn't matter how long the sequence is since we will be feeding in # tokens predicted from the previous iteration. decoder_inputs = tf.zeros((1, params['batch_size']), dtype=tf.int64) sequence_length = tf.constant(np.full((params['batch_size'],), MAX_LENGTH), dtype=tf.int32) feed_prev_output = True # Builds the actual model. initial_state = get_initial_lstm_state( tf.reduce_mean(visual_features, (1,2)), params) init_token = start_sequence_token(params['batch_size']) fn = get_loop_fn(decoder_inputs, sequence_length, visual_features, lstm, initial_state, attention, init_token, embeddings, feed_prev_output=feed_prev_output, hparams=params) token_logits, final_state, _ = tf.nn.raw_rnn(lstm, fn) # 'token_logits' holds the log-probability of each token for each # iteration of the decoding sequence. # Get logits as packed tensor in batch-major order token_logits = array_ops.transpose(token_logits.pack(), [1, 0, 2]) if mode is tf.contrib.learn.ModeKeys.TRAIN: loss = sequence_loss(token_logits, targets['target'], targets['weights']) global_step = variables.get_global_step() learning_rate = tf.train.exponential_decay( params['starter_learning_rate'], global_step, 20000, 0.96, staircase=True) optimizer = tf.train.AdamOptimizer(params['starter_learning_rate'], epsilon=params['epsilon']) train_op = optimizer.minimize(loss, global_step=global_step) return (None, loss, train_op) else: predicted_tokens = tf.argmax(token_logits, 2) print "pred tokens shape:", predicted_tokens.get_shape() return (predicted_tokens, None, None)
def linear_model_fn_with_model_fn_ops(features, labels, mode): """Same as linear_model_fn, but returns `ModelFnOps`.""" assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, model_fn.ModeKeys.INFER) prediction, loss = (models.linear_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss( loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return model_fn.ModelFnOps( mode=mode, predictions=prediction, loss=loss, train_op=train_op)
def linear_model_fn_with_model_fn_ops(features, labels, mode): """Same as linear_model_fn, but returns `ModelFnOps`.""" assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, model_fn.ModeKeys.INFER) prediction, loss = (models.linear_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss( loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return model_fn.ModelFnOps( mode=mode, predictions=prediction, loss=loss, train_op=train_op)
def _get_train_ops(self, features, _): (_, _, losses, training_op) = gmm_ops.gmm( self._parse_tensor_or_dict(features), self._training_initial_clusters, self._num_clusters, self._random_seed, self._covariance_type, self._params) incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses) training_op = with_dependencies([training_op, incr_step], loss) return training_op, loss
def _get_train_ops(self, features, _): (_, _, losses, training_op) = gmm_ops.gmm(self._parse_tensor_or_dict(features), self._training_initial_clusters, self._num_clusters, self._random_seed, self._covariance_type, self._params) incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses) training_op = with_dependencies([training_op, incr_step], loss) return training_op, loss
def test_create_global_step(self): self.assertEquals(None, variables_lib2.get_global_step()) with ops.Graph().as_default() as g: global_step = variables_lib2.create_global_step() self._assert_global_step(global_step) self.assertRaisesRegexp(ValueError, 'already exists', variables_lib2.create_global_step) self.assertRaisesRegexp(ValueError, 'already exists', variables_lib2.create_global_step, g) self._assert_global_step(variables_lib2.create_global_step(ops.Graph()))
def _make_training_op(training_loss): global_step = contrib_variables.get_global_step() assert global_step linear_train_step = self._linear_model.get_train_step(training_loss) dnn_train_step = (self._dnn_model.get_train_step(training_loss) if self._dnn_model else []) with ops.control_dependencies(linear_train_step + dnn_train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op
def _make_training_op(training_loss): global_step = contrib_variables.get_global_step() assert global_step linear_train_step = self._linear_model.get_train_step(training_loss) dnn_train_step = (self._dnn_model.get_train_step(training_loss) if self._dnn_model else []) with ops.control_dependencies(linear_train_step + dnn_train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op
def logistic_model_no_mode_fn(features, labels): features = extract(features, 'input') labels = extract(labels, 'labels') labels = array_ops.one_hot(labels, 3, 1, 0) prediction, loss = (models.logistic_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss( loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return { 'class': math_ops.argmax(prediction, 1), 'prob': prediction }, loss, train_op
def linear_model_fn(features, labels, mode): features = extract(features, 'input') labels = extract(labels, 'labels') assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, model_fn.ModeKeys.INFER) if isinstance(features, dict): (_, features), = features.items() prediction, loss = (models.linear_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss( loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return prediction, loss, train_op
def logistic_model_no_mode_fn(features, labels): features = extract(features, 'input') labels = extract(labels, 'labels') labels = array_ops.one_hot(labels, 3, 1, 0) prediction, loss = (models.logistic_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss( loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return { 'class': math_ops.argmax(prediction, 1), 'prob': prediction }, loss, train_op
def linear_model_fn(features, labels, mode): features = extract(features, 'input') labels = extract(labels, 'labels') assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, model_fn.ModeKeys.INFER) if isinstance(features, dict): (_, features), = features.items() prediction, loss = (models.linear_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss( loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return prediction, loss, train_op
def _train_op_fn(loss): """Returns the op to optimize the loss.""" return optimizers.optimize_loss( loss=loss, global_step=contrib_variables.get_global_step(), learning_rate=_LEARNING_RATE, optimizer=_get_optimizer(optimizer), clip_gradients=gradient_clip_norm, name=parent_scope, # Empty summaries to prevent optimizers from logging the training_loss. summaries=[])
def _get_train_ops(self, features, targets): global_step = contrib_variables.get_global_step() assert global_step logits = self._model.build_model(features, self._feature_columns, is_training=True) loss = self._target_column.loss(logits, targets, features) train_step = self._model.get_train_step(loss) with ops.control_dependencies(train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, loss
def _invalid_model_fn(features, labels, mode): # pylint: disable=unused-argument w = variables_lib.Variable(42.0, 'weight') loss = 100.0 - w update_global_step = variables.get_global_step().assign_add(1) with control_flow_ops.control_dependencies([update_global_step]): train_op = w.assign_add(loss / 100.0) predictions = loss if mode == model_fn.ModeKeys.EVAL: loss = None return predictions, loss, train_op
def _train_op_fn(loss): """Returns the op to optimize the loss.""" return optimizers.optimize_loss( loss=loss, global_step=contrib_variables.get_global_step(), learning_rate=_LEARNING_RATE, optimizer=_get_optimizer(optimizer), clip_gradients=gradient_clip_norm, name=parent_scope, # Empty summaries to prevent optimizers from logging the training_loss. summaries=[])
def kmeans_cluster_model_fn(features, labels, mode, params, config): """Model function for KMeansClustering estimator.""" # https://wiki.python.org/moin/UsingAssertionsEffectively assert labels is None, "labels are not needed: " + labels # clustering_ops.KMeans 是重要的算法实现过程 # https://github.com/tensorflow/tensorflow/blob/master/tensorflow # /contrib/factorization/python/ops/clustering_ops.py (all_scores, model_predictions, losses, is_initialized, cluster_centers_var, init_op, training_op) = \ clustering_ops.KMeans( _parse_tensor_or_dict(features), params.get('num_clusters'), initial_clusters=clustering_ops.RANDOM_INIT, distance_metric=clustering_ops.SQUARED_EUCLIDEAN_DISTANCE, use_mini_batch=False, mini_batch_steps_per_iteration=1, # use_mini_batch = params.get('use_mini_batch'), # mini_batch_steps_per_iteration=params.get( # 'mini_batch_steps_per_iteration'), random_seed=params.get('random_seed'), kmeans_plus_plus_num_retries=params.get( 'kmeans_plus_plus_num_retries')).training_graph() incr_step = state_ops.assign_add(variables.get_global_step(), 1) loss = math_ops.reduce_sum(losses, name='kmeans_loss') # Outputs a Summary protocol buffer containing a single scalar value. # Used for visualizing in TensorBoard summary.scalar('loss/raw', loss) # https://github.com/tensorflow/tensorflow/blob/master/tensorflow # /python/ops/control_flow_ops.py # with_dependencies(dependencies, output_tensor, name=None): # Produces the content of `output_tensor` only after `dependencies`. training_op = with_dependencies([training_op, incr_step], loss) predictions = { 'all_scores': all_scores[0], 'cluster_idx': model_predictions[0], } eval_metric_ops = {'scores': loss} # Hook for monitor training_hooks = [ _InitializeClustersHook(init_op, is_initialized, config.is_chief) ] relative_tolerance = params.get('relative_tolerance') if relative_tolerance is not None: training_hooks.append(_LossRelativeChangeHook(relative_tolerance)) return ModelFnOps(mode=mode, predictions=predictions, eval_metric_ops=eval_metric_ops, loss=loss, train_op=training_op, training_hooks=training_hooks)
def linear_model_params_fn(features, labels, mode, params): features = extract(features, 'input') labels = extract(labels, 'labels') assert mode in (model_fn.ModeKeys.TRAIN, model_fn.ModeKeys.EVAL, model_fn.ModeKeys.INFER) prediction, loss = (models.linear_regression_zero_init(features, labels)) train_op = optimizers.optimize_loss(loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=params['learning_rate']) return prediction, loss, train_op
def _get_train_ops(self, features, targets): global_step = contrib_variables.get_global_step() assert global_step logits = self._model.build_model( features, self._feature_columns, is_training=True) loss = self._target_column.loss(logits, targets, features) train_step = self._model.get_train_step(loss) with ops.control_dependencies(train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, loss
def _export_graph(graph, saver, checkpoint_path, export_dir, default_graph_signature, named_graph_signatures, exports_to_keep): """Exports graph via session_bundle, by creating a Session.""" with graph.as_default(): with tf_session.Session('') as session: session.run(variables.initialize_local_variables()) saver.restore(session, checkpoint_path) export = exporter.Exporter(saver) export.init(session.graph.as_graph_def(), default_graph_signature=default_graph_signature, named_graph_signatures=named_graph_signatures) export.export(export_dir, contrib_variables.get_global_step(), session, exports_to_keep=exports_to_keep)
def _train(features): """Add training ops to the graph.""" with variable_scope.variable_scope("model"): model_outputs = state_manager.define_loss( model, features, estimator_lib.ModeKeys.TRAIN) train_op = optimizers.optimize_loss( model_outputs.loss, global_step=variables.get_global_step(), optimizer=optimizer, # Learning rate is set in the Optimizer object learning_rate=None) return estimator_lib.EstimatorSpec(loss=model_outputs.loss, mode=estimator_lib.ModeKeys.TRAIN, train_op=train_op)
def _get_train_ops(self, features, labels): global_step = contrib_variables.get_global_step() assert global_step logits = self._model.build_model( features, self._feature_columns, is_training=True) model_fn_ops = self._head.head_ops(features, labels, tf.contrib.learn.ModeKeys.TRAIN, _noop_training_fn, logits=logits) train_step = self._model.get_train_step(model_fn_ops.loss) with ops.control_dependencies(train_step): with ops.get_default_graph().colocate_with(global_step): return state_ops.assign_add(global_step, 1).op, model_fn_ops.loss
def _train_op_fn(loss): """Returns the op to optimize the loss.""" return optimizers.optimize_loss( loss=loss, global_step=contrib_variables.get_global_step(), learning_rate=_LEARNING_RATE, optimizer=_get_optimizer(optimizer), gradient_multipliers=( dnn_linear_combined._extract_embedding_lr_multipliers( # pylint: disable=protected-access embedding_lr_multipliers, parent_scope, input_layer_scope)), clip_gradients=gradient_clip_norm, name=parent_scope, # Empty summaries to prevent optimizers from logging the training_loss. summaries=[])
def _logistic_regression_model_fn(features, labels, mode): _ = mode logits = layers.linear( features, 1, weights_initializer=init_ops.zeros_initializer(), # Intentionally uses really awful initial values so that # AUC/precision/recall/etc will change meaningfully even on a toy dataset. biases_initializer=init_ops.constant_initializer(-10.0)) predictions = math_ops.sigmoid(logits) loss = loss_ops.sigmoid_cross_entropy(logits, labels) train_op = optimizers.optimize_loss(loss, variables.get_global_step(), optimizer='Adagrad', learning_rate=0.1) return predictions, loss, train_op
def sdca_classifier_model_fn(features, targets, mode, params): """Estimator's linear model_fn.""" feature_columns = params["feature_columns"] optimizer = params["optimizer"] weight_column_name = params["weight_column_name"] loss_type = params["loss_type"] enable_centered_bias = params.get("enable_centered_bias", True) if not isinstance(optimizer, sdca_optimizer.SDCAOptimizer): raise ValueError("Optimizer must be of type SDCAOptimizer") loss_fn = { "logistic_loss": _log_loss_with_two_classes, "hinge_loss": _hinge_loss, }[loss_type] logits, columns_to_variables, bias = ( layers.weighted_sum_from_feature_columns( columns_to_tensors=features, feature_columns=feature_columns, num_outputs=1)) train_feature_columns = _maybe_add_bias_column(feature_columns, features, bias, targets, enable_centered_bias, columns_to_variables) loss = None if mode != estimator.ModeKeys.INFER: loss = math_ops.reduce_mean(loss_fn(logits, targets), name="loss") train_op = None if mode == estimator.ModeKeys.TRAIN: global_step = contrib_variables.get_global_step() # TODO(zoy): Combine linear_feature_columns and columns_to_variables. train_op = optimizer.get_train_step(train_feature_columns, weight_column_name, loss_type, features, targets, columns_to_variables, global_step) predictions = {} predictions[_LOGISTIC] = math_ops.sigmoid(logits) logits = array_ops.concat(1, [array_ops.zeros_like(logits), logits]) predictions[_PROBABILITIES] = nn.softmax(logits) predictions[_CLASSES] = math_ops.argmax(logits, 1) return predictions, loss, train_op