def testDenseFeatures_shareAcrossApplication(self): features = { "image": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]], [[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]], } feature_columns = [ hub.image_embedding_column("image", self.randomly_initialized_spec), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest( "Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): # We want to test with dense_features_v2.DenseFeatures. This symbol was # added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a. feature_layer = tf.compat.v2.keras.layers.DenseFeatures( feature_columns) feature_layer_out_1 = feature_layer(features) feature_layer_out_2 = feature_layer(features) with tf_v1.train.MonitoredSession() as sess: output_1 = sess.run(feature_layer_out_1) output_2 = sess.run(feature_layer_out_2) self.assertAllClose(output_1, output_2)
def testDenseFeatures_shareAcrossApplication(self): features = { "image": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]], [[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]], } feature_columns = [ hub.image_embedding_column("image", self.spec), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest( "Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): feature_layer = feature_column_v2.DenseFeatures(feature_columns) global_vars_before = tf.global_variables() feature_layer_out_1 = feature_layer(features) global_vars_middle = tf.global_variables() feature_layer_out_2 = feature_layer(features) global_vars_after = tf.global_variables() self.assertLen(global_vars_before, 0) self.assertLen(global_vars_middle, 1) self.assertLen(global_vars_after, 1) with tf_v1.train.MonitoredSession() as sess: output_1 = sess.run(feature_layer_out_1) output_2 = sess.run(feature_layer_out_2) self.assertAllClose(output_1, [[0.5, 0.7, 0.9], [0.8, 0.9, 1.0]]) self.assertAllEqual(output_1, output_2)
def testDenseFeatures_shareAcrossApplication(self): features = { "text": ["hello world", "pair-programming"], } feature_columns = [ hub.text_embedding_column("text", self.spec, trainable=True), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest("Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): # We want to test with dense_features_v2.DenseFeatures. This symbol was # added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a. feature_layer = tf.compat.v2.keras.layers.DenseFeatures(feature_columns) feature_layer_out_1 = feature_layer(features) feature_layer_out_2 = feature_layer(features) # We define loss only on the first layer. Since layers should have shared # weights, we expect the second layer will change too. loss = feature_layer_out_1 - tf.constant(0.005) optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.7) train_op = optimizer.minimize(loss) with tf.compat.v1.train.MonitoredSession() as sess: before_update_1 = sess.run(feature_layer_out_1) sess.run(train_op) after_update_1 = sess.run(feature_layer_out_1) after_update_2 = sess.run(feature_layer_out_2) self.assertAllEqual(before_update_1, [[1, 2, 3, 4], [5, 5, 5, 5]]) self.assertAllEqual(after_update_1, after_update_2)
def testDenseFeatures_shareAcrossApplication(self): features = { "text": ["hello world", "pair-programming"], } feature_columns = [ hub.text_embedding_column("text", self.spec, trainable=True), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest("Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): feature_layer = _dense_features_module.DenseFeatures(feature_columns) feature_layer_out_1 = feature_layer(features) feature_layer_out_2 = feature_layer(features) # We define loss only on the first layer. Since layers should have shared # weights, we expect the second layer will change too. loss = feature_layer_out_1 - tf.constant(0.005) optimizer = tf_v1.train.GradientDescentOptimizer(learning_rate=0.7) train_op = optimizer.minimize(loss) with tf_v1.train.MonitoredSession() as sess: before_update_1 = sess.run(feature_layer_out_1) sess.run(train_op) after_update_1 = sess.run(feature_layer_out_1) after_update_2 = sess.run(feature_layer_out_2) self.assertAllEqual(before_update_1, [[1, 2, 3, 4], [5, 5, 5, 5]]) self.assertAllEqual(after_update_1, after_update_2)
def testDenseFeatures(self): features = { "image_a": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]], [[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]], "image_b": [[[[0.1, 0.2, 0.1], [0.2, 0.1, 0.2]]], [[[0.1, 0.2, 0.3], [0.3, 0.2, 0.1]]]], } feature_columns = [ hub.image_embedding_column("image_a", self.spec), hub.image_embedding_column("image_b", self.spec), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest( "Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): # We want to test with dense_features_v2.DenseFeatures. This symbol was # added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a. feature_layer = tf.compat.v2.keras.layers.DenseFeatures( feature_columns) feature_layer_out = feature_layer(features) with tf.compat.v1.train.MonitoredSession() as sess: output = sess.run(feature_layer_out) self.assertAllClose(output, [[0.5, 0.7, 0.9, 0.3, 0.3, 0.3], [0.8, 0.9, 1.0, 0.4, 0.4, 0.4]])
def testFeatureColumnsWithResources(self, mock_add_resource): feature_column = hub.text_embedding_column("text_a", self.spec) if not isinstance(feature_column, feature_column_v2.FeatureColumn): self.skipTest( "Resources not implemented in the state manager of feature " "column v2.") self.assertTrue( feature_column_v2.is_feature_column_v2([feature_column]))
def linear_logit_fn(features): """Linear model logit_fn. Args: features: This is the first item returned from the `input_fn` passed to `train`, `evaluate`, and `predict`. This should be a single `Tensor` or `dict` of same. Returns: A `Tensor` representing the logits. """ if feature_column_v2.is_feature_column_v2(feature_columns): shared_state_manager = feature_column_v2.SharedEmbeddingStateManager( ) linear_model = feature_column_v2.LinearModel( feature_columns=feature_columns, units=units, sparse_combiner=sparse_combiner, shared_state_manager=shared_state_manager) logits = linear_model(features) bias = linear_model.bias_variable # We'd like to get all the non-bias variables associated with this # LinearModel. This includes the shared embedding variables as well. variables = linear_model.variables variables.remove(bias) variables.extend(shared_state_manager.variables) # Expand (potential) Partitioned variables bias = _get_expanded_variable_list([bias]) variables = _get_expanded_variable_list(variables) else: linear_model = feature_column._LinearModel( # pylint: disable=protected-access feature_columns=feature_columns, units=units, sparse_combiner=sparse_combiner, name='linear_model') logits = linear_model(features) cols_to_vars = linear_model.cols_to_vars() bias = cols_to_vars.pop('bias') variables = cols_to_vars.values() if units > 1: summary.histogram('bias', bias) else: # If units == 1, the bias value is a length-1 list of a scalar Tensor, # so we should provide a scalar summary. summary.scalar('bias', bias[0][0]) summary.scalar('fraction_of_zero_weights', _compute_fraction_of_zero(variables)) return logits
def linear_logit_fn(features): """Linear model logit_fn. Args: features: This is the first item returned from the `input_fn` passed to `train`, `evaluate`, and `predict`. This should be a single `Tensor` or `dict` of same. Returns: A `Tensor` representing the logits. """ if feature_column_v2.is_feature_column_v2(feature_columns): shared_state_manager = feature_column_v2.SharedEmbeddingStateManager() linear_model = feature_column_v2.LinearModel( feature_columns=feature_columns, units=units, sparse_combiner=sparse_combiner, shared_state_manager=shared_state_manager) logits = linear_model(features) bias = linear_model.bias_variable # We'd like to get all the non-bias variables associated with this # LinearModel. This includes the shared embedding variables as well. variables = linear_model.variables variables.remove(bias) variables.extend(shared_state_manager.variables) # Expand (potential) Partitioned variables bias = _get_expanded_variable_list([bias]) variables = _get_expanded_variable_list(variables) else: linear_model = feature_column._LinearModel( # pylint: disable=protected-access feature_columns=feature_columns, units=units, sparse_combiner=sparse_combiner, name='linear_model') logits = linear_model(features) cols_to_vars = linear_model.cols_to_vars() bias = cols_to_vars.pop('bias') variables = cols_to_vars.values() if units > 1: summary.histogram('bias', bias) else: # If units == 1, the bias value is a length-1 list of a scalar Tensor, # so we should provide a scalar summary. summary.scalar('bias', bias[0][0]) summary.scalar('fraction_of_zero_weights', _compute_fraction_of_zero(variables)) return logits
def testDenseFeatures(self): features = { "text_a": ["hello world", "pair-programming"], "text_b": ["hello world", "oov token"], } feature_columns = [ hub.text_embedding_column("text_a", self.spec, trainable=False), hub.text_embedding_column("text_b", self.spec, trainable=False), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest("Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): feature_layer = _dense_features_module.DenseFeatures(feature_columns) feature_layer_out = feature_layer(features) with tf_v1.train.MonitoredSession() as sess: output = sess.run(feature_layer_out) self.assertAllEqual( output, [[1, 2, 3, 4, 1, 2, 3, 4], [5, 5, 5, 5, 0, 0, 0, 0]])
def testDenseFeatures_shareAcrossApplication(self): features = { "image": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]], [[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]], } feature_columns = [ hub.image_embedding_column("image", self.randomly_initialized_spec), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest("Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): feature_layer = _dense_features_module.DenseFeatures(feature_columns) feature_layer_out_1 = feature_layer(features) feature_layer_out_2 = feature_layer(features) with tf_v1.train.MonitoredSession() as sess: output_1 = sess.run(feature_layer_out_1) output_2 = sess.run(feature_layer_out_2) self.assertAllClose(output_1, output_2)
def testDenseFeatures(self): features = { "text_a": ["hello world", "pair-programming"], "text_b": ["hello world", "oov token"], } feature_columns = [ hub.text_embedding_column("text_a", self.spec, trainable=False), hub.text_embedding_column("text_b", self.spec, trainable=False), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest("Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): # We want to test with dense_features_v2.DenseFeatures. This symbol was # added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a. feature_layer = tf.compat.v2.keras.layers.DenseFeatures(feature_columns) feature_layer_out = feature_layer(features) with tf.compat.v1.train.MonitoredSession() as sess: output = sess.run(feature_layer_out) self.assertAllEqual( output, [[1, 2, 3, 4, 1, 2, 3, 4], [5, 5, 5, 5, 0, 0, 0, 0]])
def testDenseFeatures(self): features = { "image_a": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]], [[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]], "image_b": [[[[0.1, 0.2, 0.1], [0.2, 0.1, 0.2]]], [[[0.1, 0.2, 0.3], [0.3, 0.2, 0.1]]]], } feature_columns = [ hub.image_embedding_column("image_a", self.spec), hub.image_embedding_column("image_b", self.spec), ] if not feature_column_v2.is_feature_column_v2(feature_columns): self.skipTest("Resources not implemented in the state manager of feature " "column v2.") with tf.Graph().as_default(): feature_layer = _dense_features_module.DenseFeatures(feature_columns) feature_layer_out = feature_layer(features) with tf_v1.train.MonitoredSession() as sess: output = sess.run(feature_layer_out) self.assertAllClose( output, [[0.5, 0.7, 0.9, 0.3, 0.3, 0.3], [0.8, 0.9, 1.0, 0.4, 0.4, 0.4]])
def __init__(self, feature_columns, model_dir=None, n_classes=2, weight_column=None, label_vocabulary=None, optimizer='Ftrl', config=None, partitioner=None, warm_start_from=None, loss_reduction=losses.Reduction.SUM, sparse_combiner='sum'): """Construct a `LinearClassifier` estimator object. Args: feature_columns: An iterable containing all the feature columns used by the model. All items in the set should be instances of classes derived from `FeatureColumn`. model_dir: Directory to save model parameters, graph and etc. This can also be used to load checkpoints from the directory into a estimator to continue training a previously saved model. n_classes: number of label classes. Default is binary classification. Note that class labels are integers representing the class index (i.e. values from 0 to n_classes-1). For arbitrary label values (e.g. string labels), convert to class indices first. weight_column: A string or a `_NumericColumn` created by `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. If it is a string, it is used as a key to fetch weight tensor from the `features`. If it is a `_NumericColumn`, raw tensor is fetched by key `weight_column.key`, then weight_column.normalizer_fn is applied on it to get weight tensor. label_vocabulary: A list of strings represents possible label values. If given, labels must be string type and have any value in `label_vocabulary`. If it is not given, that means labels are already encoded as integer or float within [0, 1] for `n_classes=2` and encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 . Also there will be errors if vocabulary is not provided and labels are string. optimizer: An instance of `tf.Optimizer` or `tf.estimator.experimental.LinearSDCA` used to train the model. Can also be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to FTRL optimizer. config: `RunConfig` object to configure the runtime settings. partitioner: Optional. Partitioner for input layer. warm_start_from: A string filepath to a checkpoint to warm-start from, or a `WarmStartSettings` object to fully configure warm-starting. If the string filepath is provided instead of a `WarmStartSettings`, then all weights and biases are warm-started, and it is assumed that vocabularies and Tensor names are unchanged. loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to reduce training loss over batch. Defaults to `SUM`. sparse_combiner: A string specifying how to reduce if a categorical column is multivalent. One of "mean", "sqrtn", and "sum" -- these are effectively different ways to do example-level normalization, which can be useful for bag-of-words features. for more details, see `tf.feature_column.linear_model`. Returns: A `LinearClassifier` estimator. Raises: ValueError: if n_classes < 2. """ if isinstance(optimizer, LinearSDCA): if sparse_combiner != 'sum': raise ValueError( 'sparse_combiner must be "sum" when optimizer ' 'is a LinearSDCA object.') if not feature_column_v2.is_feature_column_v2(feature_columns): raise ValueError('V2 feature columns required when optimizer ' 'is a LinearSDCA object.') if n_classes > 2: raise ValueError( 'LinearSDCA cannot be used in a multi-class setting.') if n_classes == 2: head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( # pylint: disable=protected-access weight_column=weight_column, label_vocabulary=label_vocabulary, loss_reduction=loss_reduction) else: head = head_lib._multi_class_head_with_softmax_cross_entropy_loss( # pylint: disable=protected-access n_classes, weight_column=weight_column, label_vocabulary=label_vocabulary, loss_reduction=loss_reduction) def _model_fn(features, labels, mode, config): """Call the defined shared _linear_model_fn.""" return _linear_model_fn(features=features, labels=labels, mode=mode, head=head, feature_columns=tuple(feature_columns or []), optimizer=optimizer, partitioner=partitioner, config=config, sparse_combiner=sparse_combiner) super(LinearClassifier, self).__init__(model_fn=_model_fn, model_dir=model_dir, config=config, warm_start_from=warm_start_from)
def __init__( self, hidden_units, feature_columns, model_dir=None, label_dimension=1, weight_column=None, optimizer='Adagrad', activation_fn=nn.relu, dropout=None, input_layer_partitioner=None, config=None, warm_start_from=None, loss_reduction=losses.Reduction.SUM, batch_norm=False, ): """Initializes a `DNNRegressor` instance. Args: hidden_units: Iterable of number hidden units per layer. All layers are fully connected. Ex. `[64, 32]` means first layer has 64 nodes and second one has 32. feature_columns: An iterable containing all the feature columns used by the model. All items in the set should be instances of classes derived from `_FeatureColumn`. model_dir: Directory to save model parameters, graph and etc. This can also be used to load checkpoints from the directory into a estimator to continue training a previously saved model. label_dimension: Number of regression targets per example. This is the size of the last dimension of the labels and logits `Tensor` objects (typically, these have shape `[batch_size, label_dimension]`). weight_column: A string or a `_NumericColumn` created by `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. If it is a string, it is used as a key to fetch weight tensor from the `features`. If it is a `_NumericColumn`, raw tensor is fetched by key `weight_column.key`, then weight_column.normalizer_fn is applied on it to get weight tensor. optimizer: An instance of `tf.Optimizer` used to train the model. Can also be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to Adagrad optimizer. activation_fn: Activation function applied to each layer. If `None`, will use `tf.nn.relu`. dropout: When not `None`, the probability we will drop out a given coordinate. input_layer_partitioner: Optional. Partitioner for input layer. Defaults to `min_max_variable_partitioner` with `min_slice_size` 64 << 20. config: `RunConfig` object to configure the runtime settings. warm_start_from: A string filepath to a checkpoint to warm-start from, or a `WarmStartSettings` object to fully configure warm-starting. If the string filepath is provided instead of a `WarmStartSettings`, then all weights are warm-started, and it is assumed that vocabularies and Tensor names are unchanged. loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to reduce training loss over batch. Defaults to `SUM`. batch_norm: Whether to use batch normalization after each hidden layer. """ shared_state_manager = None if feature_column_v2.is_feature_column_v2(feature_columns): shared_state_manager = feature_column_v2.SharedEmbeddingStateManager( ) def _model_fn(features, labels, mode, config): """Call the defined shared _dnn_model_fn.""" return _dnn_model_fn( features=features, labels=labels, mode=mode, head=head_lib._regression_head( # pylint: disable=protected-access label_dimension=label_dimension, weight_column=weight_column, loss_reduction=loss_reduction), hidden_units=hidden_units, feature_columns=tuple(feature_columns or []), optimizer=optimizer, activation_fn=activation_fn, dropout=dropout, input_layer_partitioner=input_layer_partitioner, config=config, batch_norm=batch_norm, shared_state_manager=shared_state_manager) super(DNNRegressor, self).__init__(model_fn=_model_fn, model_dir=model_dir, config=config, warm_start_from=warm_start_from)
def testFeatureColumnsWithResources(self, mock_add_resource): feature_column = hub.text_embedding_column("text_a", self.spec) self.assertTrue(feature_column_v2.is_feature_column_v2([feature_column]))
def testFeatureColumnsWithNoResources(self, mock_add_resource): mock_add_resource.side_effect = NotImplementedError feature_column = hub.text_embedding_column("text_a", self.spec) self.assertFalse(feature_column_v2.is_feature_column_v2([feature_column]))
def __init__(self, units, hidden_units, feature_columns, activation_fn, dropout, input_layer_partitioner, batch_norm, shared_state_manager, name=None, **kwargs): super(_DNNModel, self).__init__(name=name, **kwargs) if feature_column_v2.is_feature_column_v2(feature_columns): self._input_layer = feature_column_v2.FeatureLayer( feature_columns=feature_columns, name='input_layer', shared_state_manager=shared_state_manager) else: self._input_layer = feature_column.InputLayer( feature_columns=feature_columns, name='input_layer', create_scope_now=False) self._add_layer(self._input_layer, 'input_layer') self._dropout = dropout self._batch_norm = batch_norm self._hidden_layers = [] self._dropout_layers = [] self._batch_norm_layers = [] self._hidden_layer_scope_names = [] for layer_id, num_hidden_units in enumerate(hidden_units): with variable_scope.variable_scope( 'hiddenlayer_%d' % layer_id) as hidden_layer_scope: hidden_layer = core_layers.Dense( units=num_hidden_units, activation=activation_fn, kernel_initializer=init_ops.glorot_uniform_initializer(), name=hidden_layer_scope, _scope=hidden_layer_scope) self._add_layer(hidden_layer, hidden_layer_scope.name) self._hidden_layer_scope_names.append(hidden_layer_scope.name) self._hidden_layers.append(hidden_layer) if self._dropout is not None: dropout_layer = core_layers.Dropout(rate=self._dropout) self._add_layer(dropout_layer, dropout_layer.name) self._dropout_layers.append(dropout_layer) if self._batch_norm: batch_norm_layer = normalization.BatchNormalization( # The default momentum 0.99 actually crashes on certain # problem, so here we use 0.999, which is the default of # tf.contrib.layers.batch_norm. momentum=0.999, trainable=True, name='batchnorm_%d' % layer_id, _scope='batchnorm_%d' % layer_id) self._add_layer(batch_norm_layer, batch_norm_layer.name) self._batch_norm_layers.append(batch_norm_layer) with variable_scope.variable_scope('logits') as logits_scope: self._logits_layer = core_layers.Dense( units=units, activation=None, kernel_initializer=init_ops.glorot_uniform_initializer(), name=logits_scope, _scope=logits_scope) self._add_layer(self._logits_layer, logits_scope.name) self._logits_scope_name = logits_scope.name self._input_layer_partitioner = input_layer_partitioner
def __init__(self, units, hidden_units, feature_columns, activation_fn, dropout, input_layer_partitioner, batch_norm, name=None, **kwargs): super(_DNNModel, self).__init__(name=name, **kwargs) if feature_column_v2.is_feature_column_v2(feature_columns): self._input_layer = feature_column_v2.FeatureLayer( feature_columns=feature_columns, name='input_layer') else: self._input_layer = feature_column.InputLayer( feature_columns=feature_columns, name='input_layer', create_scope_now=False) self._add_layer(self._input_layer, 'input_layer') self._dropout = dropout self._batch_norm = batch_norm self._hidden_layers = [] self._dropout_layers = [] self._batch_norm_layers = [] self._hidden_layer_scope_names = [] for layer_id, num_hidden_units in enumerate(hidden_units): with variable_scope.variable_scope('hiddenlayer_%d' % layer_id) as hidden_layer_scope: hidden_layer = core_layers.Dense( units=num_hidden_units, activation=activation_fn, kernel_initializer=init_ops.glorot_uniform_initializer(), name=hidden_layer_scope, _scope=hidden_layer_scope) self._add_layer(hidden_layer, hidden_layer_scope.name) self._hidden_layer_scope_names.append(hidden_layer_scope.name) self._hidden_layers.append(hidden_layer) if self._dropout is not None: dropout_layer = core_layers.Dropout(rate=self._dropout) self._add_layer(dropout_layer, dropout_layer.name) self._dropout_layers.append(dropout_layer) if self._batch_norm: batch_norm_layer = normalization.BatchNormalization( # The default momentum 0.99 actually crashes on certain # problem, so here we use 0.999, which is the default of # tf.contrib.layers.batch_norm. momentum=0.999, trainable=True, name='batchnorm_%d' % layer_id, _scope='batchnorm_%d' % layer_id) self._add_layer(batch_norm_layer, batch_norm_layer.name) self._batch_norm_layers.append(batch_norm_layer) with variable_scope.variable_scope('logits') as logits_scope: self._logits_layer = core_layers.Dense( units=units, activation=None, kernel_initializer=init_ops.glorot_uniform_initializer(), name=logits_scope, _scope=logits_scope) self._add_layer(self._logits_layer, logits_scope.name) self._logits_scope_name = logits_scope.name self._input_layer_partitioner = input_layer_partitioner
def _sdca_model_fn(features, labels, mode, head, feature_columns, optimizer): """A model_fn for linear models that use the SDCA optimizer. Args: features: dict of `Tensor`. labels: `Tensor` of shape `[batch_size]`. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. head: A `Head` instance. feature_columns: An iterable containing all the feature columns used by the model. optimizer: a `LinearSDCA` instance. Returns: An `EstimatorSpec` instance. Raises: ValueError: mode or params are invalid, or features has the wrong type. """ assert feature_column_v2.is_feature_column_v2(feature_columns) if isinstance(head, head_lib._BinaryLogisticHeadWithSigmoidCrossEntropyLoss): # pylint: disable=protected-access loss_type = 'logistic_loss' elif isinstance(head, head_lib._RegressionHeadWithMeanSquaredErrorLoss): # pylint: disable=protected-access assert head.logits_dimension == 1 loss_type = 'squared_loss' else: raise ValueError('Unsupported head type: {}'.format(head)) linear_model = feature_column_v2.LinearModel( feature_columns=feature_columns, units=1, sparse_combiner='sum') logits = linear_model(features) bias = linear_model.bias_variable # We'd like to get all the non-bias variables associated with this # LinearModel. # TODO(rohanj): Figure out how to get shared embedding weights variable # here. variables = linear_model.variables variables.remove(bias) # Expand (potential) Partitioned variables bias = _get_expanded_variable_list([bias]) variables = _get_expanded_variable_list(variables) summary.scalar('bias', bias[0][0]) summary.scalar('fraction_of_zero_weights', _compute_fraction_of_zero(variables)) if mode == model_fn.ModeKeys.TRAIN: sdca_model, train_op = optimizer.get_train_step( linear_model._state_manager, # pylint: disable=protected-access head._weight_column, # pylint: disable=protected-access loss_type, feature_columns, features, labels, linear_model.bias_variable, training.get_global_step()) update_weights_hook = _SDCAUpdateWeightsHook(sdca_model, train_op) model_fn_ops = head.create_estimator_spec( features=features, mode=mode, labels=labels, train_op_fn=lambda unused_loss_fn: train_op, logits=logits) return model_fn_ops._replace( training_chief_hooks=(model_fn_ops.training_chief_hooks + (update_weights_hook, ))) else: return head.create_estimator_spec(features=features, mode=mode, labels=labels, logits=logits)
def __init__( self, hidden_units, feature_columns, model_dir=None, label_dimension=1, weight_column=None, optimizer='Adagrad', activation_fn=nn.relu, dropout=None, input_layer_partitioner=None, config=None, warm_start_from=None, loss_reduction=losses.Reduction.SUM, batch_norm=False, ): """Initializes a `DNNRegressor` instance. Args: hidden_units: Iterable of number hidden units per layer. All layers are fully connected. Ex. `[64, 32]` means first layer has 64 nodes and second one has 32. feature_columns: An iterable containing all the feature columns used by the model. All items in the set should be instances of classes derived from `_FeatureColumn`. model_dir: Directory to save model parameters, graph and etc. This can also be used to load checkpoints from the directory into a estimator to continue training a previously saved model. label_dimension: Number of regression targets per example. This is the size of the last dimension of the labels and logits `Tensor` objects (typically, these have shape `[batch_size, label_dimension]`). weight_column: A string or a `_NumericColumn` created by `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. If it is a string, it is used as a key to fetch weight tensor from the `features`. If it is a `_NumericColumn`, raw tensor is fetched by key `weight_column.key`, then weight_column.normalizer_fn is applied on it to get weight tensor. optimizer: An instance of `tf.Optimizer` used to train the model. Can also be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to Adagrad optimizer. activation_fn: Activation function applied to each layer. If `None`, will use `tf.nn.relu`. dropout: When not `None`, the probability we will drop out a given coordinate. input_layer_partitioner: Optional. Partitioner for input layer. Defaults to `min_max_variable_partitioner` with `min_slice_size` 64 << 20. config: `RunConfig` object to configure the runtime settings. warm_start_from: A string filepath to a checkpoint to warm-start from, or a `WarmStartSettings` object to fully configure warm-starting. If the string filepath is provided instead of a `WarmStartSettings`, then all weights are warm-started, and it is assumed that vocabularies and Tensor names are unchanged. loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to reduce training loss over batch. Defaults to `SUM`. batch_norm: Whether to use batch normalization after each hidden layer. """ shared_state_manager = None if feature_column_v2.is_feature_column_v2(feature_columns): shared_state_manager = feature_column_v2.SharedEmbeddingStateManager() def _model_fn(features, labels, mode, config): """Call the defined shared _dnn_model_fn.""" return _dnn_model_fn( features=features, labels=labels, mode=mode, head=head_lib._regression_head( # pylint: disable=protected-access label_dimension=label_dimension, weight_column=weight_column, loss_reduction=loss_reduction), hidden_units=hidden_units, feature_columns=tuple(feature_columns or []), optimizer=optimizer, activation_fn=activation_fn, dropout=dropout, input_layer_partitioner=input_layer_partitioner, config=config, batch_norm=batch_norm, shared_state_manager=shared_state_manager) super(DNNRegressor, self).__init__( model_fn=_model_fn, model_dir=model_dir, config=config, warm_start_from=warm_start_from)
def testFeatureColumnsIsV2(self): feature_column = hub.text_embedding_column_v2("text_a", self.model) self.assertTrue(feature_column_v2.is_feature_column_v2([feature_column]))
def __init__(self, feature_columns, model_dir=None, label_dimension=1, weight_column=None, optimizer='Ftrl', config=None, partitioner=None, warm_start_from=None, loss_reduction=losses.Reduction.SUM, sparse_combiner='sum'): """Initializes a `LinearRegressor` instance. Args: feature_columns: An iterable containing all the feature columns used by the model. All items in the set should be instances of classes derived from `FeatureColumn`. model_dir: Directory to save model parameters, graph and etc. This can also be used to load checkpoints from the directory into a estimator to continue training a previously saved model. label_dimension: Number of regression targets per example. This is the size of the last dimension of the labels and logits `Tensor` objects (typically, these have shape `[batch_size, label_dimension]`). weight_column: A string or a `_NumericColumn` created by `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. If it is a string, it is used as a key to fetch weight tensor from the `features`. If it is a `_NumericColumn`, raw tensor is fetched by key `weight_column.key`, then weight_column.normalizer_fn is applied on it to get weight tensor. optimizer: An instance of `tf.Optimizer` or `tf.estimator.experimental.LinearSDCA` used to train the model. Can also be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to FTRL optimizer. config: `RunConfig` object to configure the runtime settings. partitioner: Optional. Partitioner for input layer. warm_start_from: A string filepath to a checkpoint to warm-start from, or a `WarmStartSettings` object to fully configure warm-starting. If the string filepath is provided instead of a `WarmStartSettings`, then all weights and biases are warm-started, and it is assumed that vocabularies and Tensor names are unchanged. loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to reduce training loss over batch. Defaults to `SUM`. sparse_combiner: A string specifying how to reduce if a categorical column is multivalent. One of "mean", "sqrtn", and "sum" -- these are effectively different ways to do example-level normalization, which can be useful for bag-of-words features. for more details, see `tf.feature_column.linear_model`. """ if isinstance(optimizer, LinearSDCA): if sparse_combiner != 'sum': raise ValueError( 'sparse_combiner must be "sum" when optimizer ' 'is a LinearSDCA object.') if not feature_column_v2.is_feature_column_v2(feature_columns): raise ValueError('V2 feature columns required when optimizer ' 'is a LinearSDCA object.') if label_dimension > 1: raise ValueError( 'LinearSDCA can only be used with one-dimensional ' 'label.') head = head_lib._regression_head( # pylint: disable=protected-access label_dimension=label_dimension, weight_column=weight_column, loss_reduction=loss_reduction) def _model_fn(features, labels, mode, config): """Call the defined shared _linear_model_fn.""" return _linear_model_fn(features=features, labels=labels, mode=mode, head=head, feature_columns=tuple(feature_columns or []), optimizer=optimizer, partitioner=partitioner, config=config, sparse_combiner=sparse_combiner) super(LinearRegressor, self).__init__(model_fn=_model_fn, model_dir=model_dir, config=config, warm_start_from=warm_start_from)