def testDenseFeatures_shareAcrossApplication(self):
features = {
"image": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]],
[[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]],
}
feature_columns = [
hub.image_embedding_column("image",
self.randomly_initialized_spec),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest(
"Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
# We want to test with dense_features_v2.DenseFeatures. This symbol was
# added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a.
feature_layer = tf.compat.v2.keras.layers.DenseFeatures(
feature_columns)
feature_layer_out_1 = feature_layer(features)
feature_layer_out_2 = feature_layer(features)
with tf_v1.train.MonitoredSession() as sess:
output_1 = sess.run(feature_layer_out_1)
output_2 = sess.run(feature_layer_out_2)
self.assertAllClose(output_1, output_2)
def testDenseFeatures_shareAcrossApplication(self):
features = {
"image": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]],
[[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]],
}
feature_columns = [
hub.image_embedding_column("image", self.spec),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest(
"Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
feature_layer = feature_column_v2.DenseFeatures(feature_columns)
global_vars_before = tf.global_variables()
feature_layer_out_1 = feature_layer(features)
global_vars_middle = tf.global_variables()
feature_layer_out_2 = feature_layer(features)
global_vars_after = tf.global_variables()
self.assertLen(global_vars_before, 0)
self.assertLen(global_vars_middle, 1)
self.assertLen(global_vars_after, 1)
with tf_v1.train.MonitoredSession() as sess:
output_1 = sess.run(feature_layer_out_1)
output_2 = sess.run(feature_layer_out_2)
self.assertAllClose(output_1,
[[0.5, 0.7, 0.9], [0.8, 0.9, 1.0]])
self.assertAllEqual(output_1, output_2)
def testDenseFeatures_shareAcrossApplication(self):
features = {
"text": ["hello world", "pair-programming"],
}
feature_columns = [
hub.text_embedding_column("text", self.spec, trainable=True),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest("Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
# We want to test with dense_features_v2.DenseFeatures. This symbol was
# added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a.
feature_layer = tf.compat.v2.keras.layers.DenseFeatures(feature_columns)
feature_layer_out_1 = feature_layer(features)
feature_layer_out_2 = feature_layer(features)
# We define loss only on the first layer. Since layers should have shared
# weights, we expect the second layer will change too.
loss = feature_layer_out_1 - tf.constant(0.005)
optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=0.7)
train_op = optimizer.minimize(loss)
with tf.compat.v1.train.MonitoredSession() as sess:
before_update_1 = sess.run(feature_layer_out_1)
sess.run(train_op)
after_update_1 = sess.run(feature_layer_out_1)
after_update_2 = sess.run(feature_layer_out_2)
self.assertAllEqual(before_update_1, [[1, 2, 3, 4],
[5, 5, 5, 5]])
self.assertAllEqual(after_update_1, after_update_2)
def testDenseFeatures_shareAcrossApplication(self):
features = {
"text": ["hello world", "pair-programming"],
}
feature_columns = [
hub.text_embedding_column("text", self.spec, trainable=True),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest("Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
feature_layer = _dense_features_module.DenseFeatures(feature_columns)
feature_layer_out_1 = feature_layer(features)
feature_layer_out_2 = feature_layer(features)
# We define loss only on the first layer. Since layers should have shared
# weights, we expect the second layer will change too.
loss = feature_layer_out_1 - tf.constant(0.005)
optimizer = tf_v1.train.GradientDescentOptimizer(learning_rate=0.7)
train_op = optimizer.minimize(loss)
with tf_v1.train.MonitoredSession() as sess:
before_update_1 = sess.run(feature_layer_out_1)
sess.run(train_op)
after_update_1 = sess.run(feature_layer_out_1)
after_update_2 = sess.run(feature_layer_out_2)
self.assertAllEqual(before_update_1, [[1, 2, 3, 4],
[5, 5, 5, 5]])
self.assertAllEqual(after_update_1, after_update_2)
def testDenseFeatures(self):
features = {
"image_a": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]],
[[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]],
"image_b": [[[[0.1, 0.2, 0.1], [0.2, 0.1, 0.2]]],
[[[0.1, 0.2, 0.3], [0.3, 0.2, 0.1]]]],
}
feature_columns = [
hub.image_embedding_column("image_a", self.spec),
hub.image_embedding_column("image_b", self.spec),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest(
"Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
# We want to test with dense_features_v2.DenseFeatures. This symbol was
# added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a.
feature_layer = tf.compat.v2.keras.layers.DenseFeatures(
feature_columns)
feature_layer_out = feature_layer(features)
with tf.compat.v1.train.MonitoredSession() as sess:
output = sess.run(feature_layer_out)
self.assertAllClose(output, [[0.5, 0.7, 0.9, 0.3, 0.3, 0.3],
[0.8, 0.9, 1.0, 0.4, 0.4, 0.4]])
def testFeatureColumnsWithResources(self, mock_add_resource):
feature_column = hub.text_embedding_column("text_a", self.spec)
if not isinstance(feature_column, feature_column_v2.FeatureColumn):
self.skipTest(
"Resources not implemented in the state manager of feature "
"column v2.")
self.assertTrue(
feature_column_v2.is_feature_column_v2([feature_column]))
def linear_logit_fn(features):
"""Linear model logit_fn.
Args:
features: This is the first item returned from the `input_fn`
passed to `train`, `evaluate`, and `predict`. This should be a
single `Tensor` or `dict` of same.
Returns:
A `Tensor` representing the logits.
"""
if feature_column_v2.is_feature_column_v2(feature_columns):
shared_state_manager = feature_column_v2.SharedEmbeddingStateManager(
)
linear_model = feature_column_v2.LinearModel(
feature_columns=feature_columns,
units=units,
sparse_combiner=sparse_combiner,
shared_state_manager=shared_state_manager)
logits = linear_model(features)
bias = linear_model.bias_variable
# We'd like to get all the non-bias variables associated with this
# LinearModel. This includes the shared embedding variables as well.
variables = linear_model.variables
variables.remove(bias)
variables.extend(shared_state_manager.variables)
# Expand (potential) Partitioned variables
bias = _get_expanded_variable_list([bias])
variables = _get_expanded_variable_list(variables)
else:
linear_model = feature_column._LinearModel( # pylint: disable=protected-access
feature_columns=feature_columns,
units=units,
sparse_combiner=sparse_combiner,
name='linear_model')
logits = linear_model(features)
cols_to_vars = linear_model.cols_to_vars()
bias = cols_to_vars.pop('bias')
variables = cols_to_vars.values()
if units > 1:
summary.histogram('bias', bias)
else:
# If units == 1, the bias value is a length-1 list of a scalar Tensor,
# so we should provide a scalar summary.
summary.scalar('bias', bias[0][0])
summary.scalar('fraction_of_zero_weights',
_compute_fraction_of_zero(variables))
return logits
def linear_logit_fn(features):
"""Linear model logit_fn.
Args:
features: This is the first item returned from the `input_fn`
passed to `train`, `evaluate`, and `predict`. This should be a
single `Tensor` or `dict` of same.
Returns:
A `Tensor` representing the logits.
"""
if feature_column_v2.is_feature_column_v2(feature_columns):
shared_state_manager = feature_column_v2.SharedEmbeddingStateManager()
linear_model = feature_column_v2.LinearModel(
feature_columns=feature_columns,
units=units,
sparse_combiner=sparse_combiner,
shared_state_manager=shared_state_manager)
logits = linear_model(features)
bias = linear_model.bias_variable
# We'd like to get all the non-bias variables associated with this
# LinearModel. This includes the shared embedding variables as well.
variables = linear_model.variables
variables.remove(bias)
variables.extend(shared_state_manager.variables)
# Expand (potential) Partitioned variables
bias = _get_expanded_variable_list([bias])
variables = _get_expanded_variable_list(variables)
else:
linear_model = feature_column._LinearModel( # pylint: disable=protected-access
feature_columns=feature_columns,
units=units,
sparse_combiner=sparse_combiner,
name='linear_model')
logits = linear_model(features)
cols_to_vars = linear_model.cols_to_vars()
bias = cols_to_vars.pop('bias')
variables = cols_to_vars.values()
if units > 1:
summary.histogram('bias', bias)
else:
# If units == 1, the bias value is a length-1 list of a scalar Tensor,
# so we should provide a scalar summary.
summary.scalar('bias', bias[0][0])
summary.scalar('fraction_of_zero_weights',
_compute_fraction_of_zero(variables))
return logits
def testDenseFeatures(self):
features = {
"text_a": ["hello world", "pair-programming"],
"text_b": ["hello world", "oov token"],
}
feature_columns = [
hub.text_embedding_column("text_a", self.spec, trainable=False),
hub.text_embedding_column("text_b", self.spec, trainable=False),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest("Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
feature_layer = _dense_features_module.DenseFeatures(feature_columns)
feature_layer_out = feature_layer(features)
with tf_v1.train.MonitoredSession() as sess:
output = sess.run(feature_layer_out)
self.assertAllEqual(
output, [[1, 2, 3, 4, 1, 2, 3, 4], [5, 5, 5, 5, 0, 0, 0, 0]])
def testDenseFeatures_shareAcrossApplication(self):
features = {
"image": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]],
[[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]],
}
feature_columns = [
hub.image_embedding_column("image", self.randomly_initialized_spec),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest("Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
feature_layer = _dense_features_module.DenseFeatures(feature_columns)
feature_layer_out_1 = feature_layer(features)
feature_layer_out_2 = feature_layer(features)
with tf_v1.train.MonitoredSession() as sess:
output_1 = sess.run(feature_layer_out_1)
output_2 = sess.run(feature_layer_out_2)
self.assertAllClose(output_1, output_2)
def testDenseFeatures(self):
features = {
"text_a": ["hello world", "pair-programming"],
"text_b": ["hello world", "oov token"],
}
feature_columns = [
hub.text_embedding_column("text_a", self.spec, trainable=False),
hub.text_embedding_column("text_b", self.spec, trainable=False),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest("Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
# We want to test with dense_features_v2.DenseFeatures. This symbol was
# added in https://github.com/tensorflow/tensorflow/commit/64586f18724f737393071125a91b19adf013cf8a.
feature_layer = tf.compat.v2.keras.layers.DenseFeatures(feature_columns)
feature_layer_out = feature_layer(features)
with tf.compat.v1.train.MonitoredSession() as sess:
output = sess.run(feature_layer_out)
self.assertAllEqual(
output, [[1, 2, 3, 4, 1, 2, 3, 4], [5, 5, 5, 5, 0, 0, 0, 0]])
def testDenseFeatures(self):
features = {
"image_a": [[[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]],
[[[0.7, 0.7, 0.7], [0.1, 0.2, 0.3]]]],
"image_b": [[[[0.1, 0.2, 0.1], [0.2, 0.1, 0.2]]],
[[[0.1, 0.2, 0.3], [0.3, 0.2, 0.1]]]],
}
feature_columns = [
hub.image_embedding_column("image_a", self.spec),
hub.image_embedding_column("image_b", self.spec),
]
if not feature_column_v2.is_feature_column_v2(feature_columns):
self.skipTest("Resources not implemented in the state manager of feature "
"column v2.")
with tf.Graph().as_default():
feature_layer = _dense_features_module.DenseFeatures(feature_columns)
feature_layer_out = feature_layer(features)
with tf_v1.train.MonitoredSession() as sess:
output = sess.run(feature_layer_out)
self.assertAllClose(
output,
[[0.5, 0.7, 0.9, 0.3, 0.3, 0.3], [0.8, 0.9, 1.0, 0.4, 0.4, 0.4]])
def __init__(self,
feature_columns,
model_dir=None,
n_classes=2,
weight_column=None,
label_vocabulary=None,
optimizer='Ftrl',
config=None,
partitioner=None,
warm_start_from=None,
loss_reduction=losses.Reduction.SUM,
sparse_combiner='sum'):
"""Construct a `LinearClassifier` estimator object.
Args:
feature_columns: An iterable containing all the feature columns used by
the model. All items in the set should be instances of classes derived
from `FeatureColumn`.
model_dir: Directory to save model parameters, graph and etc. This can
also be used to load checkpoints from the directory into a estimator
to continue training a previously saved model.
n_classes: number of label classes. Default is binary classification.
Note that class labels are integers representing the class index (i.e.
values from 0 to n_classes-1). For arbitrary label values (e.g. string
labels), convert to class indices first.
weight_column: A string or a `_NumericColumn` created by
`tf.feature_column.numeric_column` defining feature column representing
weights. It is used to down weight or boost examples during training. It
will be multiplied by the loss of the example. If it is a string, it is
used as a key to fetch weight tensor from the `features`. If it is a
`_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
then weight_column.normalizer_fn is applied on it to get weight tensor.
label_vocabulary: A list of strings represents possible label values. If
given, labels must be string type and have any value in
`label_vocabulary`. If it is not given, that means labels are
already encoded as integer or float within [0, 1] for `n_classes=2` and
encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 .
Also there will be errors if vocabulary is not provided and labels are
string.
optimizer: An instance of `tf.Optimizer` or
`tf.estimator.experimental.LinearSDCA` used to train the model. Can
also be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'),
or callable. Defaults to FTRL optimizer.
config: `RunConfig` object to configure the runtime settings.
partitioner: Optional. Partitioner for input layer.
warm_start_from: A string filepath to a checkpoint to warm-start from, or
a `WarmStartSettings` object to fully configure warm-starting. If the
string filepath is provided instead of a `WarmStartSettings`, then all
weights and biases are warm-started, and it is assumed that vocabularies
and Tensor names are unchanged.
loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
to reduce training loss over batch. Defaults to `SUM`.
sparse_combiner: A string specifying how to reduce if a categorical column
is multivalent. One of "mean", "sqrtn", and "sum" -- these are
effectively different ways to do example-level normalization, which can
be useful for bag-of-words features. for more details, see
`tf.feature_column.linear_model`.
Returns:
A `LinearClassifier` estimator.
Raises:
ValueError: if n_classes < 2.
"""
if isinstance(optimizer, LinearSDCA):
if sparse_combiner != 'sum':
raise ValueError(
'sparse_combiner must be "sum" when optimizer '
'is a LinearSDCA object.')
if not feature_column_v2.is_feature_column_v2(feature_columns):
raise ValueError('V2 feature columns required when optimizer '
'is a LinearSDCA object.')
if n_classes > 2:
raise ValueError(
'LinearSDCA cannot be used in a multi-class setting.')
if n_classes == 2:
head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss( # pylint: disable=protected-access
weight_column=weight_column,
label_vocabulary=label_vocabulary,
loss_reduction=loss_reduction)
else:
head = head_lib._multi_class_head_with_softmax_cross_entropy_loss( # pylint: disable=protected-access
n_classes,
weight_column=weight_column,
label_vocabulary=label_vocabulary,
loss_reduction=loss_reduction)
def _model_fn(features, labels, mode, config):
"""Call the defined shared _linear_model_fn."""
return _linear_model_fn(features=features,
labels=labels,
mode=mode,
head=head,
feature_columns=tuple(feature_columns
or []),
optimizer=optimizer,
partitioner=partitioner,
config=config,
sparse_combiner=sparse_combiner)
super(LinearClassifier, self).__init__(model_fn=_model_fn,
model_dir=model_dir,
config=config,
warm_start_from=warm_start_from)
def __init__(
self,
hidden_units,
feature_columns,
model_dir=None,
label_dimension=1,
weight_column=None,
optimizer='Adagrad',
activation_fn=nn.relu,
dropout=None,
input_layer_partitioner=None,
config=None,
warm_start_from=None,
loss_reduction=losses.Reduction.SUM,
batch_norm=False,
):
"""Initializes a `DNNRegressor` instance.
Args:
hidden_units: Iterable of number hidden units per layer. All layers are
fully connected. Ex. `[64, 32]` means first layer has 64 nodes and
second one has 32.
feature_columns: An iterable containing all the feature columns used by
the model. All items in the set should be instances of classes derived
from `_FeatureColumn`.
model_dir: Directory to save model parameters, graph and etc. This can
also be used to load checkpoints from the directory into a estimator to
continue training a previously saved model.
label_dimension: Number of regression targets per example. This is the
size of the last dimension of the labels and logits `Tensor` objects
(typically, these have shape `[batch_size, label_dimension]`).
weight_column: A string or a `_NumericColumn` created by
`tf.feature_column.numeric_column` defining feature column representing
weights. It is used to down weight or boost examples during training. It
will be multiplied by the loss of the example. If it is a string, it is
used as a key to fetch weight tensor from the `features`. If it is a
`_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
then weight_column.normalizer_fn is applied on it to get weight tensor.
optimizer: An instance of `tf.Optimizer` used to train the model. Can also
be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or
callable. Defaults to Adagrad optimizer.
activation_fn: Activation function applied to each layer. If `None`, will
use `tf.nn.relu`.
dropout: When not `None`, the probability we will drop out a given
coordinate.
input_layer_partitioner: Optional. Partitioner for input layer. Defaults
to `min_max_variable_partitioner` with `min_slice_size` 64 << 20.
config: `RunConfig` object to configure the runtime settings.
warm_start_from: A string filepath to a checkpoint to warm-start from, or
a `WarmStartSettings` object to fully configure warm-starting. If the
string filepath is provided instead of a `WarmStartSettings`, then all
weights are warm-started, and it is assumed that vocabularies and Tensor
names are unchanged.
loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
to reduce training loss over batch. Defaults to `SUM`.
batch_norm: Whether to use batch normalization after each hidden layer.
"""
shared_state_manager = None
if feature_column_v2.is_feature_column_v2(feature_columns):
shared_state_manager = feature_column_v2.SharedEmbeddingStateManager(
)
def _model_fn(features, labels, mode, config):
"""Call the defined shared _dnn_model_fn."""
return _dnn_model_fn(
features=features,
labels=labels,
mode=mode,
head=head_lib._regression_head( # pylint: disable=protected-access
label_dimension=label_dimension,
weight_column=weight_column,
loss_reduction=loss_reduction),
hidden_units=hidden_units,
feature_columns=tuple(feature_columns or []),
optimizer=optimizer,
activation_fn=activation_fn,
dropout=dropout,
input_layer_partitioner=input_layer_partitioner,
config=config,
batch_norm=batch_norm,
shared_state_manager=shared_state_manager)
super(DNNRegressor, self).__init__(model_fn=_model_fn,
model_dir=model_dir,
config=config,
warm_start_from=warm_start_from)
def testFeatureColumnsWithResources(self, mock_add_resource):
feature_column = hub.text_embedding_column("text_a", self.spec)
self.assertTrue(feature_column_v2.is_feature_column_v2([feature_column]))
def testFeatureColumnsWithNoResources(self, mock_add_resource):
mock_add_resource.side_effect = NotImplementedError
feature_column = hub.text_embedding_column("text_a", self.spec)
self.assertFalse(feature_column_v2.is_feature_column_v2([feature_column]))
def __init__(self,
units,
hidden_units,
feature_columns,
activation_fn,
dropout,
input_layer_partitioner,
batch_norm,
shared_state_manager,
name=None,
**kwargs):
super(_DNNModel, self).__init__(name=name, **kwargs)
if feature_column_v2.is_feature_column_v2(feature_columns):
self._input_layer = feature_column_v2.FeatureLayer(
feature_columns=feature_columns,
name='input_layer',
shared_state_manager=shared_state_manager)
else:
self._input_layer = feature_column.InputLayer(
feature_columns=feature_columns,
name='input_layer',
create_scope_now=False)
self._add_layer(self._input_layer, 'input_layer')
self._dropout = dropout
self._batch_norm = batch_norm
self._hidden_layers = []
self._dropout_layers = []
self._batch_norm_layers = []
self._hidden_layer_scope_names = []
for layer_id, num_hidden_units in enumerate(hidden_units):
with variable_scope.variable_scope(
'hiddenlayer_%d' % layer_id) as hidden_layer_scope:
hidden_layer = core_layers.Dense(
units=num_hidden_units,
activation=activation_fn,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=hidden_layer_scope,
_scope=hidden_layer_scope)
self._add_layer(hidden_layer, hidden_layer_scope.name)
self._hidden_layer_scope_names.append(hidden_layer_scope.name)
self._hidden_layers.append(hidden_layer)
if self._dropout is not None:
dropout_layer = core_layers.Dropout(rate=self._dropout)
self._add_layer(dropout_layer, dropout_layer.name)
self._dropout_layers.append(dropout_layer)
if self._batch_norm:
batch_norm_layer = normalization.BatchNormalization(
# The default momentum 0.99 actually crashes on certain
# problem, so here we use 0.999, which is the default of
# tf.contrib.layers.batch_norm.
momentum=0.999,
trainable=True,
name='batchnorm_%d' % layer_id,
_scope='batchnorm_%d' % layer_id)
self._add_layer(batch_norm_layer, batch_norm_layer.name)
self._batch_norm_layers.append(batch_norm_layer)
with variable_scope.variable_scope('logits') as logits_scope:
self._logits_layer = core_layers.Dense(
units=units,
activation=None,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=logits_scope,
_scope=logits_scope)
self._add_layer(self._logits_layer, logits_scope.name)
self._logits_scope_name = logits_scope.name
self._input_layer_partitioner = input_layer_partitioner
def __init__(self,
units,
hidden_units,
feature_columns,
activation_fn,
dropout,
input_layer_partitioner,
batch_norm,
name=None,
**kwargs):
super(_DNNModel, self).__init__(name=name, **kwargs)
if feature_column_v2.is_feature_column_v2(feature_columns):
self._input_layer = feature_column_v2.FeatureLayer(
feature_columns=feature_columns, name='input_layer')
else:
self._input_layer = feature_column.InputLayer(
feature_columns=feature_columns,
name='input_layer',
create_scope_now=False)
self._add_layer(self._input_layer, 'input_layer')
self._dropout = dropout
self._batch_norm = batch_norm
self._hidden_layers = []
self._dropout_layers = []
self._batch_norm_layers = []
self._hidden_layer_scope_names = []
for layer_id, num_hidden_units in enumerate(hidden_units):
with variable_scope.variable_scope('hiddenlayer_%d' %
layer_id) as hidden_layer_scope:
hidden_layer = core_layers.Dense(
units=num_hidden_units,
activation=activation_fn,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=hidden_layer_scope,
_scope=hidden_layer_scope)
self._add_layer(hidden_layer, hidden_layer_scope.name)
self._hidden_layer_scope_names.append(hidden_layer_scope.name)
self._hidden_layers.append(hidden_layer)
if self._dropout is not None:
dropout_layer = core_layers.Dropout(rate=self._dropout)
self._add_layer(dropout_layer, dropout_layer.name)
self._dropout_layers.append(dropout_layer)
if self._batch_norm:
batch_norm_layer = normalization.BatchNormalization(
# The default momentum 0.99 actually crashes on certain
# problem, so here we use 0.999, which is the default of
# tf.contrib.layers.batch_norm.
momentum=0.999,
trainable=True,
name='batchnorm_%d' % layer_id,
_scope='batchnorm_%d' % layer_id)
self._add_layer(batch_norm_layer, batch_norm_layer.name)
self._batch_norm_layers.append(batch_norm_layer)
with variable_scope.variable_scope('logits') as logits_scope:
self._logits_layer = core_layers.Dense(
units=units,
activation=None,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=logits_scope,
_scope=logits_scope)
self._add_layer(self._logits_layer, logits_scope.name)
self._logits_scope_name = logits_scope.name
self._input_layer_partitioner = input_layer_partitioner
def _sdca_model_fn(features, labels, mode, head, feature_columns, optimizer):
"""A model_fn for linear models that use the SDCA optimizer.
Args:
features: dict of `Tensor`.
labels: `Tensor` of shape `[batch_size]`.
mode: Defines whether this is training, evaluation or prediction.
See `ModeKeys`.
head: A `Head` instance.
feature_columns: An iterable containing all the feature columns used by
the model.
optimizer: a `LinearSDCA` instance.
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: mode or params are invalid, or features has the wrong type.
"""
assert feature_column_v2.is_feature_column_v2(feature_columns)
if isinstance(head,
head_lib._BinaryLogisticHeadWithSigmoidCrossEntropyLoss): # pylint: disable=protected-access
loss_type = 'logistic_loss'
elif isinstance(head, head_lib._RegressionHeadWithMeanSquaredErrorLoss): # pylint: disable=protected-access
assert head.logits_dimension == 1
loss_type = 'squared_loss'
else:
raise ValueError('Unsupported head type: {}'.format(head))
linear_model = feature_column_v2.LinearModel(
feature_columns=feature_columns, units=1, sparse_combiner='sum')
logits = linear_model(features)
bias = linear_model.bias_variable
# We'd like to get all the non-bias variables associated with this
# LinearModel.
# TODO(rohanj): Figure out how to get shared embedding weights variable
# here.
variables = linear_model.variables
variables.remove(bias)
# Expand (potential) Partitioned variables
bias = _get_expanded_variable_list([bias])
variables = _get_expanded_variable_list(variables)
summary.scalar('bias', bias[0][0])
summary.scalar('fraction_of_zero_weights',
_compute_fraction_of_zero(variables))
if mode == model_fn.ModeKeys.TRAIN:
sdca_model, train_op = optimizer.get_train_step(
linear_model._state_manager, # pylint: disable=protected-access
head._weight_column, # pylint: disable=protected-access
loss_type,
feature_columns,
features,
labels,
linear_model.bias_variable,
training.get_global_step())
update_weights_hook = _SDCAUpdateWeightsHook(sdca_model, train_op)
model_fn_ops = head.create_estimator_spec(
features=features,
mode=mode,
labels=labels,
train_op_fn=lambda unused_loss_fn: train_op,
logits=logits)
return model_fn_ops._replace(
training_chief_hooks=(model_fn_ops.training_chief_hooks +
(update_weights_hook, )))
else:
return head.create_estimator_spec(features=features,
mode=mode,
labels=labels,
logits=logits)
def __init__(
self,
hidden_units,
feature_columns,
model_dir=None,
label_dimension=1,
weight_column=None,
optimizer='Adagrad',
activation_fn=nn.relu,
dropout=None,
input_layer_partitioner=None,
config=None,
warm_start_from=None,
loss_reduction=losses.Reduction.SUM,
batch_norm=False,
):
"""Initializes a `DNNRegressor` instance.
Args:
hidden_units: Iterable of number hidden units per layer. All layers are
fully connected. Ex. `[64, 32]` means first layer has 64 nodes and
second one has 32.
feature_columns: An iterable containing all the feature columns used by
the model. All items in the set should be instances of classes derived
from `_FeatureColumn`.
model_dir: Directory to save model parameters, graph and etc. This can
also be used to load checkpoints from the directory into a estimator to
continue training a previously saved model.
label_dimension: Number of regression targets per example. This is the
size of the last dimension of the labels and logits `Tensor` objects
(typically, these have shape `[batch_size, label_dimension]`).
weight_column: A string or a `_NumericColumn` created by
`tf.feature_column.numeric_column` defining feature column representing
weights. It is used to down weight or boost examples during training. It
will be multiplied by the loss of the example. If it is a string, it is
used as a key to fetch weight tensor from the `features`. If it is a
`_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
then weight_column.normalizer_fn is applied on it to get weight tensor.
optimizer: An instance of `tf.Optimizer` used to train the model. Can also
be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or
callable. Defaults to Adagrad optimizer.
activation_fn: Activation function applied to each layer. If `None`, will
use `tf.nn.relu`.
dropout: When not `None`, the probability we will drop out a given
coordinate.
input_layer_partitioner: Optional. Partitioner for input layer. Defaults
to `min_max_variable_partitioner` with `min_slice_size` 64 << 20.
config: `RunConfig` object to configure the runtime settings.
warm_start_from: A string filepath to a checkpoint to warm-start from, or
a `WarmStartSettings` object to fully configure warm-starting. If the
string filepath is provided instead of a `WarmStartSettings`, then all
weights are warm-started, and it is assumed that vocabularies and Tensor
names are unchanged.
loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
to reduce training loss over batch. Defaults to `SUM`.
batch_norm: Whether to use batch normalization after each hidden layer.
"""
shared_state_manager = None
if feature_column_v2.is_feature_column_v2(feature_columns):
shared_state_manager = feature_column_v2.SharedEmbeddingStateManager()
def _model_fn(features, labels, mode, config):
"""Call the defined shared _dnn_model_fn."""
return _dnn_model_fn(
features=features,
labels=labels,
mode=mode,
head=head_lib._regression_head( # pylint: disable=protected-access
label_dimension=label_dimension,
weight_column=weight_column,
loss_reduction=loss_reduction),
hidden_units=hidden_units,
feature_columns=tuple(feature_columns or []),
optimizer=optimizer,
activation_fn=activation_fn,
dropout=dropout,
input_layer_partitioner=input_layer_partitioner,
config=config,
batch_norm=batch_norm,
shared_state_manager=shared_state_manager)
super(DNNRegressor, self).__init__(
model_fn=_model_fn, model_dir=model_dir, config=config,
warm_start_from=warm_start_from)
def testFeatureColumnsIsV2(self):
feature_column = hub.text_embedding_column_v2("text_a", self.model)
self.assertTrue(feature_column_v2.is_feature_column_v2([feature_column]))
def __init__(self,
feature_columns,
model_dir=None,
label_dimension=1,
weight_column=None,
optimizer='Ftrl',
config=None,
partitioner=None,
warm_start_from=None,
loss_reduction=losses.Reduction.SUM,
sparse_combiner='sum'):
"""Initializes a `LinearRegressor` instance.
Args:
feature_columns: An iterable containing all the feature columns used by
the model. All items in the set should be instances of classes derived
from `FeatureColumn`.
model_dir: Directory to save model parameters, graph and etc. This can
also be used to load checkpoints from the directory into a estimator
to continue training a previously saved model.
label_dimension: Number of regression targets per example. This is the
size of the last dimension of the labels and logits `Tensor` objects
(typically, these have shape `[batch_size, label_dimension]`).
weight_column: A string or a `_NumericColumn` created by
`tf.feature_column.numeric_column` defining feature column representing
weights. It is used to down weight or boost examples during training. It
will be multiplied by the loss of the example. If it is a string, it is
used as a key to fetch weight tensor from the `features`. If it is a
`_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
then weight_column.normalizer_fn is applied on it to get weight tensor.
optimizer: An instance of `tf.Optimizer` or
`tf.estimator.experimental.LinearSDCA` used to train the model. Can
also be a string (one of 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'),
or callable. Defaults to FTRL optimizer.
config: `RunConfig` object to configure the runtime settings.
partitioner: Optional. Partitioner for input layer.
warm_start_from: A string filepath to a checkpoint to warm-start from, or
a `WarmStartSettings` object to fully configure warm-starting. If the
string filepath is provided instead of a `WarmStartSettings`, then all
weights and biases are warm-started, and it is assumed that vocabularies
and Tensor names are unchanged.
loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how
to reduce training loss over batch. Defaults to `SUM`.
sparse_combiner: A string specifying how to reduce if a categorical column
is multivalent. One of "mean", "sqrtn", and "sum" -- these are
effectively different ways to do example-level normalization, which can
be useful for bag-of-words features. for more details, see
`tf.feature_column.linear_model`.
"""
if isinstance(optimizer, LinearSDCA):
if sparse_combiner != 'sum':
raise ValueError(
'sparse_combiner must be "sum" when optimizer '
'is a LinearSDCA object.')
if not feature_column_v2.is_feature_column_v2(feature_columns):
raise ValueError('V2 feature columns required when optimizer '
'is a LinearSDCA object.')
if label_dimension > 1:
raise ValueError(
'LinearSDCA can only be used with one-dimensional '
'label.')
head = head_lib._regression_head( # pylint: disable=protected-access
label_dimension=label_dimension,
weight_column=weight_column,
loss_reduction=loss_reduction)
def _model_fn(features, labels, mode, config):
"""Call the defined shared _linear_model_fn."""
return _linear_model_fn(features=features,
labels=labels,
mode=mode,
head=head,
feature_columns=tuple(feature_columns
or []),
optimizer=optimizer,
partitioner=partitioner,
config=config,
sparse_combiner=sparse_combiner)
super(LinearRegressor, self).__init__(model_fn=_model_fn,
model_dir=model_dir,
config=config,
warm_start_from=warm_start_from)
