def encode_features(features, feature_columns, mode=tf_estimator.ModeKeys.TRAIN, scope=None): """Returns dense tensors from features using feature columns. This function encodes the feature column transformation on the 'raw' `features`. Args: features: (dict) mapping feature names to feature values, possibly obtained from input_fn. feature_columns: (list) list of feature columns. mode: (`estimator.ModeKeys`) Specifies if this is training, evaluation or inference. See `ModeKeys`. scope: (str) variable scope for the per column input layers. Returns: (dict) A mapping from columns to dense tensors. """ # Having scope here for backward compatibility. del scope trainable = (mode == tf_estimator.ModeKeys.TRAIN) cols_to_tensors = {} # TODO: Ensure only v2 Feature Columns are used. if hasattr(feature_column_lib, "is_feature_column_v2" ) and feature_column_lib.is_feature_column_v2(feature_columns): dense_feature_columns = [ col for col in feature_columns if not _is_sequence_column_v2(col) ] sequence_feature_columns = [ col for col in feature_columns if _is_sequence_column_v2(col) ] if dense_feature_columns: dense_layer = tf.compat.v1.keras.layers.DenseFeatures( feature_columns=dense_feature_columns, name="encoding_layer", trainable=trainable) dense_layer(features, cols_to_output_tensors=cols_to_tensors) for col in sequence_feature_columns: sequence_feature_layer = tf.keras.experimental.SequenceFeatures( col) sequence_input, _ = sequence_feature_layer(features) cols_to_tensors[col] = sequence_input else: tf.compat.v1.feature_column.input_layer( features=features, feature_columns=feature_columns, trainable=trainable, cols_to_output_tensors=cols_to_tensors) return cols_to_tensors
def linear_logit_fn(features): """Linear model logit_fn. Args: features: This is the first item returned from the `input_fn` passed to `train`, `evaluate`, and `predict`. This should be a single `Tensor` or `dict` of same. Returns: A `Tensor` representing the logits. """ if feature_column_lib.is_feature_column_v2(feature_columns): linear_model = feature_column_lib.LinearModel( feature_columns=feature_columns, units=units, sparse_combiner=sparse_combiner, name='linear_model') logits = linear_model(features) bias = linear_model.bias # We'd like to get all the non-bias variables associated with this # LinearModel. # TODO(rohanj): Figure out how to get shared embedding weights variable # here. variables = linear_model.variables variables.remove(bias) # Expand (potential) Partitioned variables bias = _get_expanded_variable_list([bias]) else: linear_model = feature_column._LinearModel( # pylint: disable=protected-access feature_columns=feature_columns, units=units, sparse_combiner=sparse_combiner, name='linear_model') logits = linear_model(features) cols_to_vars = linear_model.cols_to_vars() bias = cols_to_vars.pop('bias') variables = cols_to_vars.values() variables = _get_expanded_variable_list(variables) if units > 1: summary.histogram('bias', bias) else: # If units == 1, the bias value is a length-1 list of a scalar Tensor, # so we should provide a scalar summary. summary.scalar('bias', bias[0][0]) summary.scalar('fraction_of_zero_weights', _compute_fraction_of_zero(variables)) return logits
def _validate_linear_sdca_optimizer_for_linear_classifier( feature_columns, n_classes, optimizer, sparse_combiner): """Helper function for the initialization of LinearClassifier.""" if isinstance(optimizer, LinearSDCA): if sparse_combiner != 'sum': raise ValueError('sparse_combiner must be "sum" when optimizer ' 'is a LinearSDCA object.') if not feature_column_lib.is_feature_column_v2(feature_columns): raise ValueError('V2 feature columns required when optimizer ' 'is a LinearSDCA object.') if n_classes > 2: raise ValueError('LinearSDCA cannot be used in a multi-class setting.')
def _validate_linear_sdca_optimizer_for_linear_regressor( feature_columns, label_dimension, optimizer, sparse_combiner): """Helper function for the initialization of LinearRegressor.""" if isinstance(optimizer, LinearSDCA): if sparse_combiner != 'sum': raise ValueError('sparse_combiner must be "sum" when optimizer ' 'is a LinearSDCA object.') if not feature_column_lib.is_feature_column_v2(feature_columns): raise ValueError('V2 feature columns required when optimizer ' 'is a LinearSDCA object.') if label_dimension > 1: raise ValueError('LinearSDCA can only be used with one-dimensional ' 'label.')
def encode_features(features, feature_columns, mode=model_fn.ModeKeys.TRAIN, scope=None): """Returns dense tensors from features using feature columns. This function encodes the feature column transformation on the 'raw' `features`. Args: features: (dict) mapping feature names to feature values, possibly obtained from input_fn. feature_columns: (list) list of feature columns. mode: (`estimator.ModeKeys`) Specifies if this is training, evaluation or inference. See `ModeKeys`. scope: (str) variable scope for the per column input layers. Returns: (dict) A mapping from columns to dense tensors. """ # Having scope here for backward compatibility. del scope trainable = (mode == model_fn.ModeKeys.TRAIN) cols_to_tensors = {} if hasattr(feature_column_lib, "is_feature_column_v2" ) and feature_column_lib.is_feature_column_v2(feature_columns): dense_layer = feature_column_lib.DenseFeatures( feature_columns=feature_columns, name="encoding_layer", trainable=trainable) dense_layer(features, cols_to_output_tensors=cols_to_tensors) else: feature_column.input_layer( features=features, feature_columns=feature_columns, trainable=trainable, cols_to_output_tensors=cols_to_tensors) return cols_to_tensors
def __init__(self, units, hidden_units, feature_columns, activation_fn, dropout, input_layer_partitioner, batch_norm, name=None, **kwargs): super(_DNNModel, self).__init__(name=name, **kwargs) if feature_column_lib.is_feature_column_v2(feature_columns): self._input_layer = feature_column_lib.DenseFeatures( feature_columns=feature_columns, name='input_layer') else: self._input_layer = feature_column.InputLayer( feature_columns=feature_columns, name='input_layer', create_scope_now=False) self._add_layer(self._input_layer, 'input_layer') self._dropout = dropout self._batch_norm = batch_norm self._hidden_layers = [] self._dropout_layers = [] self._batch_norm_layers = [] self._hidden_layer_scope_names = [] for layer_id, num_hidden_units in enumerate(hidden_units): with variable_scope.variable_scope( 'hiddenlayer_%d' % layer_id) as hidden_layer_scope: hidden_layer = core_layers.Dense( units=num_hidden_units, activation=activation_fn, kernel_initializer=init_ops.glorot_uniform_initializer(), name=hidden_layer_scope, _scope=hidden_layer_scope) self._add_layer(hidden_layer, hidden_layer_scope.name) self._hidden_layer_scope_names.append(hidden_layer_scope.name) self._hidden_layers.append(hidden_layer) if self._dropout is not None: dropout_layer = core_layers.Dropout(rate=self._dropout) self._add_layer(dropout_layer, dropout_layer.name) self._dropout_layers.append(dropout_layer) if self._batch_norm: batch_norm_layer = normalization.BatchNormalization( # The default momentum 0.99 actually crashes on certain # problem, so here we use 0.999, which is the default of # tf.contrib.layers.batch_norm. momentum=0.999, trainable=True, name='batchnorm_%d' % layer_id, _scope='batchnorm_%d' % layer_id) self._add_layer(batch_norm_layer, batch_norm_layer.name) self._batch_norm_layers.append(batch_norm_layer) with variable_scope.variable_scope('logits') as logits_scope: self._logits_layer = core_layers.Dense( units=units, activation=None, kernel_initializer=init_ops.glorot_uniform_initializer(), name=logits_scope, _scope=logits_scope) self._add_layer(self._logits_layer, logits_scope.name) self._logits_scope_name = logits_scope.name self._input_layer_partitioner = input_layer_partitioner
def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): with tf.compat.v1.variable_scope(scope_name): if problem_type == phoenix_spec_pb2.PhoenixSpec.CNN: # Sometimes we only get the image feature as a tensor. if not isinstance(features, dict): return features, None return tf.cast(features[feature_columns[0].name], dtype=tf.float32), None # DNN elif problem_type == phoenix_spec_pb2.PhoenixSpec.DNN: # To allow running a custom evaluation where multiple batches are # aggregated in a single metric_fn call, we need to define the # batch_size based on the input_fn, but DenseFeatures does not allow # this. if (len(feature_columns) == 1 and isinstance( feature_columns[0], type(tf.feature_column.numeric_column("x")))): return tf.cast(features[feature_columns[0].name], dtype=tf.float32), None # All are TF1 feature columns elif all([ not feature_column_lib.is_feature_column_v2([fc]) for fc in feature_columns ]): return tf.compat.v1.feature_column.input_layer( features, feature_columns, trainable=is_training), None # Some are TF1 feature columns elif any([ not feature_column_lib.is_feature_column_v2([fc]) for fc in feature_columns ]): fc_v1 = [ fc for fc in feature_columns if not feature_column_lib.is_feature_column_v2([fc]) ] fc_v2 = [ fc for fc in feature_columns if feature_column_lib.is_feature_column_v2([fc]) ] input_1 = tf.compat.v1.feature_column.input_layer( features, fc_v1, trainable=is_training) input_2 = tf.keras.layers.DenseFeatures( fc_v2, name="input_layer_fc_v2", trainable=is_training)(features) return tf.concat([input_1, input_2], axis=1), None # None is TF1 feature columns else: return tf.keras.layers.DenseFeatures( feature_columns, name="input_layer", trainable=is_training)(features), None # RNN elif (problem_type == phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS or problem_type == phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS): if lengths_feature_name: return (tf.cast(features[feature_columns[0].name], dtype=tf.float32), features[lengths_feature_name]) elif (feature_columns[0].name in features and not isinstance( features[feature_columns[0].name], tf.SparseTensor)): return tf.cast(features[feature_columns[0].name], dtype=tf.float32), None else: # IMPORTANT NOTE: # When you use Keras layers with variables, always give them a name! # If not, keras will add "_#" (e.g., dense_1 instead of dense). # It will add the suffix even if the outer-scope is different. # This is a surprising behavior. # TODO(mazzawi): Contact the Keras team about this. return tf.keras.experimental.SequenceFeatures( feature_columns=feature_columns, trainable=is_training, name=scope_name)(features) else: raise ValueError("Unknown problem type")
def __init__(self, units, hidden_units, feature_columns, activation_fn, dropout, batch_norm, name=None, **kwargs): super(_DNNModelV2, self).__init__(name=name, **kwargs) # Add this name_scope for backward compatibility, as previously it's used # in variable_scope with ops.name_scope( 'input_from_feature_columns') as input_feature_column_scope: layer_name = input_feature_column_scope + 'input_layer' if feature_column_lib.is_feature_column_v2(feature_columns): self._input_layer = feature_column_lib.DenseFeatures( feature_columns=feature_columns, name=layer_name) else: self._input_layer = feature_column.InputLayer( feature_columns=feature_columns, name=layer_name, create_scope_now=False) self._add_layer(self._input_layer, self._input_layer.name) self._dropout = dropout self._batch_norm = batch_norm self._hidden_layers = [] self._dropout_layers = [] self._batch_norm_layers = [] self._hidden_layer_scope_names = [] for layer_id, num_hidden_units in enumerate(hidden_units): with ops.name_scope('hiddenlayer_%d' % layer_id) as hidden_layer_scope: # Get scope name without the trailing slash. hidden_shared_name = _name_from_scope_name(hidden_layer_scope) hidden_layer = core_layers.Dense( units=num_hidden_units, activation=activation_fn, kernel_initializer=init_ops.glorot_uniform_initializer(), name=hidden_shared_name) self._add_layer(hidden_layer, hidden_shared_name) self._hidden_layer_scope_names.append(hidden_shared_name) self._hidden_layers.append(hidden_layer) if self._dropout is not None: dropout_layer = core_layers.Dropout(rate=self._dropout) self._add_layer(dropout_layer, dropout_layer.name) self._dropout_layers.append(dropout_layer) if self._batch_norm: batch_norm_name = hidden_shared_name + '/batchnorm_%d' % layer_id batch_norm_layer = normalization.BatchNormalization( # The default momentum 0.99 actually crashes on certain # problem, so here we use 0.999, which is the default of # tf.contrib.layers.batch_norm. momentum=0.999, trainable=True, name=batch_norm_name) self._add_layer(batch_norm_layer, batch_norm_name) self._batch_norm_layers.append(batch_norm_layer) with ops.name_scope('logits') as logits_scope: logits_shared_name = _name_from_scope_name(logits_scope) self._logits_layer = core_layers.Dense( units=units, activation=None, kernel_initializer=init_ops.glorot_uniform_initializer(), name=logits_shared_name) self._add_layer(self._logits_layer, logits_shared_name) self._logits_scope_name = logits_shared_name
def encode_listwise_features(features, context_feature_columns, example_feature_columns, input_size=None, mode=tf_estimator.ModeKeys.TRAIN, scope=None): """Returns dense tensors from features using feature columns. Args: features: (dict) mapping feature names (str) to feature values (`tf.Tensor` or `tf.SparseTensor`), possibly obtained from input_fn. For context features, the tensors are 2-D, while for example features the tensors are 3-D. context_feature_columns: (dict) context feature names to columns. example_feature_columns: (dict) example feature names to columns. input_size: (int) [DEPRECATED: Use without this argument.] number of examples per query. If this is None, input_size is inferred as the size of second dimension of the Tensor corresponding to one of the example feature columns. mode: (`estimator.ModeKeys`) Specifies if this is training, evaluation or inference. See `ModeKeys`. scope: (str) variable scope for the per column input layers. Returns: context_features: (dict) A mapping from context feature names to dense 2-D tensors of shape [batch_size, ...]. example_features: (dict) A mapping from example feature names to dense 3-D tensors of shape [batch_size, input_size, ...]. Raises: ValueError: If `input size` is not equal to 2nd dimension of example tensors. """ context_features = {} if context_feature_columns: context_cols_to_tensors = encode_features( features, context_feature_columns.values(), mode=mode, scope=scope) context_features = { name: context_cols_to_tensors[col] for name, col in six.iteritems(context_feature_columns) } # Compute example_features. Note that the keys in `example_feature_columns` # dict can be different from the keys in the `features` dict. We only need to # reshape the per-example tensors in `features`. To obtain the keys for # per-example features, we use the parsing feature specs. example_features = {} if example_feature_columns: if feature_column_lib.is_feature_column_v2( example_feature_columns.values()): example_specs = tf.compat.v2.feature_column.make_parse_example_spec( example_feature_columns.values()) else: example_specs = tf.compat.v1.feature_column.make_parse_example_spec( example_feature_columns.values()) example_name = next(six.iterkeys(example_specs)) batch_size = tf.shape(input=features[example_name])[0] if input_size is None: input_size = tf.shape(input=features[example_name])[1] # Reshape [batch_size, input_size] to [batch * input_size] so that # features are encoded. reshaped_features = {} for name in example_specs: if name not in features: tf.compat.v1.logging.warn( "Feature {} is not found.".format(name)) continue try: reshaped_features[name] = utils.reshape_first_ndims( features[name], 2, [batch_size * input_size]) except: raise ValueError( "2nd dimension of tensor must be equal to input size: {}, " "but found feature {} with shape {}.".format( input_size, name, features[name].get_shape())) example_cols_to_tensors = encode_features( reshaped_features, example_feature_columns.values(), mode=mode, scope=scope) example_features = { name: utils.reshape_first_ndims(example_cols_to_tensors[col], 1, [batch_size, input_size]) for name, col in six.iteritems(example_feature_columns) } return context_features, example_features
def __init__(self, units, hidden_units, feature_columns, activation_fn, dropout, batch_norm, name=None, **kwargs): super(_DNNModelV2, self).__init__(name=name, **kwargs) with ops.name_scope( 'input_from_feature_columns') as input_feature_column_scope: layer_name = input_feature_column_scope + 'input_layer' if feature_column_lib.is_feature_column_v2(feature_columns): self._input_layer = dense_features_v2.DenseFeatures( feature_columns=feature_columns, name=layer_name) else: raise ValueError( 'Received a feature column from TensorFlow v1, but this is a ' 'TensorFlow v2 Estimator. Please either use v2 feature columns ' '(accessible via tf.feature_column.* in TF 2.x) with this ' 'Estimator, or switch to a v1 Estimator for use with v1 feature ' 'columns (accessible via tf.compat.v1.estimator.* and ' 'tf.compat.v1.feature_column.*, respectively.') self._dropout = dropout self._batch_norm = batch_norm self._hidden_layers = [] self._dropout_layers = [] self._batch_norm_layers = [] self._hidden_layer_scope_names = [] for layer_id, num_hidden_units in enumerate(hidden_units): with ops.name_scope('hiddenlayer_%d' % layer_id) as hidden_layer_scope: # Get scope name without the trailing slash. hidden_shared_name = _name_from_scope_name(hidden_layer_scope) hidden_layer = keras_core.Dense( units=num_hidden_units, activation=activation_fn, kernel_initializer=tf.compat.v1.glorot_uniform_initializer(), name=hidden_shared_name) self._hidden_layer_scope_names.append(hidden_shared_name) self._hidden_layers.append(hidden_layer) if self._dropout is not None: dropout_layer = keras_core.Dropout(rate=self._dropout) self._dropout_layers.append(dropout_layer) if self._batch_norm: batch_norm_name = hidden_shared_name + '/batchnorm_%d' % layer_id # TODO(scottzhu): Change back to use BatchNormalization when the # cleanup is done. batch_norm_layer = keras_norm.BatchNormalizationBase( # The default momentum 0.99 actually crashes on certain # problem, so here we use 0.999, which is the default of # tf.contrib.layers.batch_norm. momentum=0.999, trainable=True, name=batch_norm_name) self._batch_norm_layers.append(batch_norm_layer) with ops.name_scope('logits') as logits_scope: logits_shared_name = _name_from_scope_name(logits_scope) self._logits_layer = keras_core.Dense( units=units, activation=None, kernel_initializer=tf.compat.v1.glorot_uniform_initializer(), name=logits_shared_name) self._logits_scope_name = logits_shared_name
def _sdca_model_fn(features, labels, mode, head, feature_columns, optimizer): """A model_fn for linear models that use the SDCA optimizer. Args: features: dict of `Tensor`. labels: `Tensor` of shape `[batch_size]`. mode: Defines whether this is training, evaluation or prediction. See `ModeKeys`. head: A `Head` instance. feature_columns: An iterable containing all the feature columns used by the model. optimizer: a `LinearSDCA` instance. Returns: An `EstimatorSpec` instance. Raises: ValueError: mode or params are invalid, or features has the wrong type. """ assert feature_column_lib.is_feature_column_v2(feature_columns) # The _sdca_model_fn is used by _linear_model_fn with LinearSDCA optimizer in # both v1 and v2 Linear Estimators, and the only difference is to check head # type. Here we check the instance for both v1 and v2 Head to avoid duplicate # codes. Later v1 and v2 versions of _sdca_model_fn can be created if # necessary. if isinstance(head, (binary_class_head.BinaryClassHead, head_lib._BinaryLogisticHeadWithSigmoidCrossEntropyLoss)): # pylint: disable=protected-access loss_type = 'logistic_loss' elif isinstance(head, (regression_head.RegressionHead, head_lib._RegressionHeadWithMeanSquaredErrorLoss)): # pylint: disable=protected-access assert head.logits_dimension == 1 loss_type = 'squared_loss' else: raise ValueError('Unsupported head type: {}'.format(head)) linear_model = feature_column_lib.LinearModel( feature_columns=feature_columns, units=1, sparse_combiner='sum') logits = linear_model(features) bias = linear_model.bias # We'd like to get all the non-bias variables associated with this # LinearModel. # TODO(rohanj): Figure out how to get shared embedding weights variable # here. variables = linear_model.variables variables.remove(bias) # Expand (potential) Partitioned variables bias = _get_expanded_variable_list([bias]) variables = _get_expanded_variable_list(variables) summary.scalar('bias', bias[0][0]) summary.scalar('fraction_of_zero_weights', _compute_fraction_of_zero(variables)) if mode == ModeKeys.TRAIN: sdca_model, train_op = optimizer.get_train_step( linear_model.layer._state_manager, # pylint: disable=protected-access head._weight_column, # pylint: disable=protected-access loss_type, feature_columns, features, labels, linear_model.bias, training.get_global_step()) update_weights_hook = _SDCAUpdateWeightsHook(sdca_model, train_op) model_fn_ops = head.create_estimator_spec( features=features, mode=mode, labels=labels, train_op_fn=lambda unused_loss_fn: train_op, logits=logits) return model_fn_ops._replace(training_chief_hooks=( model_fn_ops.training_chief_hooks + (update_weights_hook,))) else: return head.create_estimator_spec( features=features, mode=mode, labels=labels, logits=logits)