def test_callable_returns_invalid(self):
def _optimizer_fn():
return (1, 2, 3)
with self.assertRaisesRegexp(
ValueError, 'The given object is not an Optimizer instance'):
optimizers.get_optimizer_instance(_optimizer_fn)
def _dnn_model_fn(features,
labels,
mode,
head,
hidden_units,
feature_columns,
optimizer='Adagrad',
activation_fn=nn.relu,
dropout=None,
input_layer_partitioner=None,
config=None,
use_tpu=False,
batch_norm=False):
"""Deep Neural Net model_fn v1.
Args:
features: dict of `Tensor`.
labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of dtype
`int32` or `int64` in the range `[0, n_classes)`.
mode: Defines whether this is training, evaluation or prediction. See
`ModeKeys`.
head: A `head_lib._Head` instance.
hidden_units: Iterable of integer number of hidden units per layer.
feature_columns: Iterable of `feature_column._FeatureColumn` model inputs.
optimizer: String, `tf.Optimizer` object, or callable that creates the
optimizer to use for training. If not specified, will use the Adagrad
optimizer with a default learning rate of 0.05.
activation_fn: Activation function applied to each layer.
dropout: When not `None`, the probability we will drop out a given
coordinate.
input_layer_partitioner: Partitioner for input layer. Defaults to
`min_max_variable_partitioner` with `min_slice_size` 64 << 20.
config: `RunConfig` object to configure the runtime settings.
use_tpu: Whether to make a DNN model able to run on TPU. Will make function
return a `_TPUEstimatorSpec` instance and disable variable partitioning.
batch_norm: Whether to use batch normalization after each hidden layer.
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: If features has the wrong type.
"""
optimizer = optimizers.get_optimizer_instance(
optimizer, learning_rate=_LEARNING_RATE)
return _dnn_model_fn_core(
features,
labels,
mode,
head,
hidden_units,
feature_columns,
optimizer=optimizer,
activation_fn=activation_fn,
dropout=dropout,
input_layer_partitioner=input_layer_partitioner,
use_tpu=use_tpu,
batch_norm=batch_norm)
def _linear_model_fn(features,
labels,
mode,
head,
feature_columns,
optimizer,
partitioner,
config,
sparse_combiner='sum'):
"""A model_fn for linear models that use a gradient-based optimizer.
Args:
features: dict of `Tensor`.
labels: `Tensor` of shape `[batch_size, logits_dimension]`.
mode: Defines whether this is training, evaluation or prediction.
See `ModeKeys`.
head: A `Head` instance.
feature_columns: An iterable containing all the feature columns used by
the model.
optimizer: string, `Optimizer` object, or callable that defines the
optimizer to use for training. If `None`, will use a FTRL optimizer.
partitioner: Partitioner for variables.
config: `RunConfig` object to configure the runtime settings.
sparse_combiner: A string specifying how to reduce if a categorical column
is multivalent. One of "mean", "sqrtn", and "sum".
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: mode or params are invalid, or features has the wrong type.
"""
if not isinstance(features, dict):
raise ValueError('features should be a dictionary of `Tensor`s. '
'Given type: {}'.format(type(features)))
optimizer = optimizers.get_optimizer_instance(
optimizer or _get_default_optimizer(feature_columns),
learning_rate=_LEARNING_RATE)
num_ps_replicas = config.num_ps_replicas if config else 0
partitioner = partitioner or (
partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas, min_slice_size=64 << 20))
with variable_scope.variable_scope('linear',
values=tuple(six.itervalues(features)),
partitioner=partitioner):
logit_fn = linear_logit_fn_builder(units=head.logits_dimension,
feature_columns=feature_columns,
sparse_combiner=sparse_combiner)
logits = logit_fn(features=features)
return head.create_estimator_spec(features=features,
mode=mode,
labels=labels,
optimizer=optimizer,
logits=logits)
def _rnn_model_fn(features,
labels,
mode,
head,
rnn_cell_fn,
sequence_feature_columns,
context_feature_columns,
return_sequences=False,
optimizer='Adagrad',
input_layer_partitioner=None,
config=None):
"""Recurrent Neural Net model_fn.
Args:
features: dict of `Tensor` and `SparseTensor` objects returned from
`input_fn`.
labels: `Tensor` of shape [batch_size, 1] or [batch_size] with labels.
mode: Defines whether this is training, evaluation or prediction.
See `ModeKeys`.
head: A `Head` instance.
rnn_cell_fn: A function with one argument, a `tf.estimator.ModeKeys`, and
returns an object of type `tf.nn.rnn_cell.RNNCell`.
sequence_feature_columns: Iterable containing `FeatureColumn`s that
represent sequential model inputs.
context_feature_columns: Iterable containing `FeatureColumn`s that
represent model inputs not associated with a specific timestep.
return_sequences: A boolean indicating whether to return the last output
in the output sequence, or the full sequence.
optimizer: String, `tf.Optimizer` object, or callable that creates the
optimizer to use for training. If not specified, will use the Adagrad
optimizer with a default learning rate of 0.05 and gradient clip norm of
5.0.
input_layer_partitioner: Partitioner for input layer. Defaults
to `min_max_variable_partitioner` with `min_slice_size` 64 << 20.
config: `RunConfig` object to configure the runtime settings.
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: If mode or optimizer is invalid, or features has the wrong type.
"""
if not isinstance(features, dict):
raise ValueError('features should be a dictionary of `Tensor`s. '
'Given type: {}'.format(type(features)))
# If user does not provide an optimizer instance, use the optimizer specified
# by the string with default learning rate and gradient clipping.
if not isinstance(optimizer, optimizer_lib.Optimizer):
optimizer = optimizers.get_optimizer_instance(
optimizer, learning_rate=_DEFAULT_LEARNING_RATE)
optimizer = extenders.clip_gradients_by_norm(optimizer,
_DEFAULT_CLIP_NORM)
num_ps_replicas = config.num_ps_replicas if config else 0
partitioner = partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas)
with variable_scope.variable_scope('rnn',
values=tuple(six.itervalues(features)),
partitioner=partitioner):
input_layer_partitioner = input_layer_partitioner or (
partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas, min_slice_size=64 << 20))
logit_fn = _rnn_logit_fn_builder(
output_units=head.logits_dimension,
rnn_cell_fn=rnn_cell_fn,
sequence_feature_columns=sequence_feature_columns,
context_feature_columns=context_feature_columns,
input_layer_partitioner=input_layer_partitioner,
return_sequences=return_sequences)
logits, sequence_length_mask = logit_fn(features=features, mode=mode)
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
return optimizer.minimize(
loss, global_step=training_util.get_global_step())
if return_sequences and head.input_sequence_mask_key not in features:
features[head.input_sequence_mask_key] = sequence_length_mask
return head.create_estimator_spec(features=features,
mode=mode,
labels=labels,
train_op_fn=_train_op_fn,
logits=logits)
def __init__(self,
periodicities,
input_window_size,
output_window_size,
model_dir=None,
num_features=1,
extra_feature_columns=None,
num_timesteps=10,
loss=ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS,
num_units=128,
optimizer="Adam",
config=None):
"""Initialize the Estimator.
Args:
periodicities: periodicities of the input data, in the same units as the
time feature (for example 24 if feeding hourly data with a daily
periodicity, or 60 * 24 if feeding minute-level data with daily
periodicity). Note this can be a single value or a list of values for
multiple periodicities.
input_window_size: Number of past time steps of data to look at when doing
the regression.
output_window_size: Number of future time steps to predict. Note that
setting this value to > 1 empirically seems to give a better fit.
model_dir: Directory to save model parameters, graph and etc. This can
also be used to load checkpoints from the directory into a estimator to
continue training a previously saved model.
num_features: The dimensionality of the time series (default value is one
for univariate, more than one for multivariate).
extra_feature_columns: A list of `tf.feature_column`s (for example
`tf.feature_column.embedding_column`) corresponding to features which
provide extra information to the model but are not part of the series to
be predicted.
num_timesteps: Number of buckets into which to divide (time %
periodicity). This value multiplied by the number of periodicities is
the number of time features added to the model.
loss: Loss function to use for training. Currently supported values are
SQUARED_LOSS and NORMAL_LIKELIHOOD_LOSS. Note that for
NORMAL_LIKELIHOOD_LOSS, we train the covariance term as well. For
SQUARED_LOSS, the evaluation loss is reported based on un-scaled
observations and predictions, while the training loss is computed on
normalized data.
num_units: The size of the hidden state in the encoder and decoder LSTM
cells.
optimizer: string, `tf.train.Optimizer` object, or callable that defines
the optimizer algorithm to use for training. Defaults to the Adam
optimizer with a learning rate of 0.01.
config: Optional `estimator.RunConfig` object to configure the runtime
settings.
"""
optimizer = optimizers.get_optimizer_instance(optimizer,
learning_rate=0.01)
model = ar_model.ARModel(
periodicities=periodicities,
input_window_size=input_window_size,
output_window_size=output_window_size,
num_features=num_features,
exogenous_feature_columns=extra_feature_columns,
num_time_buckets=num_timesteps,
loss=loss,
prediction_model_factory=functools.partial(
ar_model.LSTMPredictionModel, num_units=num_units))
state_manager = state_management.FilteringOnlyStateManager()
super(LSTMAutoRegressor,
self).__init__(model=model,
state_manager=state_manager,
optimizer=optimizer,
model_dir=model_dir,
config=config,
head_type=ts_head_lib.OneShotPredictionHead)
def test_lambda(self):
opt = optimizers.get_optimizer_instance(lambda: _TestOptimizer()) # pylint: disable=unnecessary-lambda
self.assertIsInstance(opt, _TestOptimizer)
def test_sgd(self):
opt = optimizers.get_optimizer_instance('SGD', learning_rate=0.1)
self.assertIsInstance(opt, gradient_descent.GradientDescentOptimizer)
self.assertAlmostEqual(0.1, opt._learning_rate)
def _rnn_model_fn(features,
labels,
mode,
head,
rnn_layer_fn,
sequence_feature_columns,
context_feature_columns,
return_sequences=False,
optimizer='Adagrad'):
"""Recurrent Neural Net model_fn.
Args:
features: dict of `Tensor` and `SparseTensor` objects returned from
`input_fn`.
labels: `Tensor` of shape [batch_size, 1] or [batch_size] with labels.
mode: Defines whether this is training, evaluation or prediction.
See `ModeKeys`.
head: A `Head` instance.
rnn_layer_fn: A function that returns a tf.keras.layers.RNN layer.
sequence_feature_columns: Iterable containing `FeatureColumn`s that
represent sequential model inputs.
context_feature_columns: Iterable containing `FeatureColumn`s that
represent model inputs not associated with a specific timestep.
return_sequences: A boolean indicating whether to return the last output
in the output sequence, or the full sequence.
optimizer: String, `tf.Optimizer` object, or callable that creates the
optimizer to use for training. If not specified, will use the Adagrad
optimizer with a default learning rate of 0.05 and gradient clip norm of
5.0.
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: If mode or optimizer is invalid, or features has the wrong type.
"""
if not isinstance(features, dict):
raise ValueError('features should be a dictionary of `Tensor`s. '
'Given type: {}'.format(type(features)))
# If user does not provide an optimizer instance, use the optimizer specified
# by the string with default learning rate and gradient clipping.
if not isinstance(optimizer, optimizer_lib.Optimizer):
optimizer = optimizers.get_optimizer_instance(
optimizer, learning_rate=_DEFAULT_LEARNING_RATE)
optimizer = extenders.clip_gradients_by_norm(optimizer,
_DEFAULT_CLIP_NORM)
logit_fn = _rnn_logit_fn_builder(
output_units=head.logits_dimension,
rnn_layer_fn=rnn_layer_fn,
sequence_feature_columns=sequence_feature_columns,
context_feature_columns=context_feature_columns)
logits, sequence_length_mask = logit_fn(features=features, mode=mode)
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
return optimizer.minimize(loss,
global_step=training_util.get_global_step())
if return_sequences and head.input_sequence_mask_key not in features:
features[head.input_sequence_mask_key] = sequence_length_mask
return head.create_estimator_spec(features=features,
mode=mode,
labels=labels,
train_op_fn=_train_op_fn,
logits=logits)
def test_ftrl(self):
opt = optimizers.get_optimizer_instance('Ftrl', learning_rate=0.1)
self.assertIsInstance(opt, ftrl.FtrlOptimizer)
self.assertAlmostEqual(0.1, opt._learning_rate)
def test_rmsprop(self):
opt = optimizers.get_optimizer_instance('RMSProp', learning_rate=0.1)
self.assertIsInstance(opt, rmsprop.RMSPropOptimizer)
self.assertAlmostEqual(0.1, opt._learning_rate)
def test_adam(self):
opt = optimizers.get_optimizer_instance('Adam', learning_rate=0.1)
self.assertIsInstance(opt, adam.AdamOptimizer)
self.assertAlmostEqual(0.1, opt._lr)
def test_supported_name_but_learning_rate_none(self):
with self.assertRaisesRegexp(
ValueError,
'learning_rate must be specified when opt is string'):
optimizers.get_optimizer_instance('Adagrad', learning_rate=None)
def test_unsupported_name(self):
with self.assertRaisesRegexp(
ValueError, 'Unsupported optimizer name: unsupported_name'):
optimizers.get_optimizer_instance('unsupported_name',
learning_rate=0.1)
def _dnn_linear_combined_model_fn(features,
labels,
mode,
head,
linear_feature_columns=None,
linear_optimizer='Ftrl',
dnn_feature_columns=None,
dnn_optimizer='Adagrad',
dnn_hidden_units=None,
dnn_activation_fn=nn.relu,
dnn_dropout=None,
input_layer_partitioner=None,
config=None,
batch_norm=False,
linear_sparse_combiner='sum'):
"""Deep Neural Net and Linear combined model_fn.
Args:
features: dict of `Tensor`.
labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of dtype
`int32` or `int64` in the range `[0, n_classes)`.
mode: Defines whether this is training, evaluation or prediction.
See `ModeKeys`.
head: A `Head` instance.
linear_feature_columns: An iterable containing all the feature columns used
by the Linear model.
linear_optimizer: string, `Optimizer` object, or callable that defines the
optimizer to use for training the Linear model. Defaults to the Ftrl
optimizer.
dnn_feature_columns: An iterable containing all the feature columns used by
the DNN model.
dnn_optimizer: string, `Optimizer` object, or callable that defines the
optimizer to use for training the DNN model. Defaults to the Adagrad
optimizer.
dnn_hidden_units: List of hidden units per DNN layer.
dnn_activation_fn: Activation function applied to each DNN layer. If `None`,
will use `tf.nn.relu`.
dnn_dropout: When not `None`, the probability we will drop out a given DNN
coordinate.
input_layer_partitioner: Partitioner for input layer.
config: `RunConfig` object to configure the runtime settings.
batch_norm: Whether to use batch normalization after each hidden layer.
linear_sparse_combiner: A string specifying how to reduce the linear model
if a categorical column is multivalent. One of "mean", "sqrtn", and
"sum".
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: If both `linear_feature_columns` and `dnn_features_columns`
are empty at the same time, or `input_layer_partitioner` is missing,
or features has the wrong type.
"""
if not isinstance(features, dict):
raise ValueError('features should be a dictionary of `Tensor`s. '
'Given type: {}'.format(type(features)))
if not linear_feature_columns and not dnn_feature_columns:
raise ValueError(
'Either linear_feature_columns or dnn_feature_columns must be defined.'
)
num_ps_replicas = config.num_ps_replicas if config else 0
input_layer_partitioner = input_layer_partitioner or (
partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas, min_slice_size=64 << 20))
# Build DNN Logits.
dnn_parent_scope = 'dnn'
if not dnn_feature_columns:
dnn_logits = None
else:
dnn_optimizer = optimizers.get_optimizer_instance(
dnn_optimizer, learning_rate=_DNN_LEARNING_RATE)
_check_no_sync_replicas_optimizer(dnn_optimizer)
if not dnn_hidden_units:
raise ValueError(
'dnn_hidden_units must be defined when dnn_feature_columns is '
'specified.')
dnn_partitioner = (partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas))
with variable_scope.variable_scope(
dnn_parent_scope,
values=tuple(six.itervalues(features)),
partitioner=dnn_partitioner) as scope:
dnn_absolute_scope = scope.name
dnn_logit_fn = dnn.dnn_logit_fn_builder(
units=head.logits_dimension,
hidden_units=dnn_hidden_units,
feature_columns=dnn_feature_columns,
activation_fn=dnn_activation_fn,
dropout=dnn_dropout,
batch_norm=batch_norm,
input_layer_partitioner=input_layer_partitioner)
dnn_logits = dnn_logit_fn(features=features, mode=mode)
linear_parent_scope = 'linear'
if not linear_feature_columns:
linear_logits = None
else:
linear_optimizer = optimizers.get_optimizer_instance(
linear_optimizer,
learning_rate=_linear_learning_rate(len(linear_feature_columns)))
_check_no_sync_replicas_optimizer(linear_optimizer)
with variable_scope.variable_scope(
linear_parent_scope,
values=tuple(six.itervalues(features)),
partitioner=input_layer_partitioner) as scope:
linear_absolute_scope = scope.name
logit_fn = linear.linear_logit_fn_builder(
units=head.logits_dimension,
feature_columns=linear_feature_columns,
sparse_combiner=linear_sparse_combiner)
linear_logits = logit_fn(features=features)
_add_layer_summary(linear_logits, scope.name)
# Combine logits and build full model.
if dnn_logits is not None and linear_logits is not None:
logits = dnn_logits + linear_logits
elif dnn_logits is not None:
logits = dnn_logits
else:
logits = linear_logits
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
train_ops = []
global_step = training_util.get_global_step()
if dnn_logits is not None:
train_ops.append(
dnn_optimizer.minimize(loss,
var_list=ops.get_collection(
ops.GraphKeys.TRAINABLE_VARIABLES,
scope=dnn_absolute_scope)))
if linear_logits is not None:
train_ops.append(
linear_optimizer.minimize(
loss,
var_list=ops.get_collection(
ops.GraphKeys.TRAINABLE_VARIABLES,
scope=linear_absolute_scope)))
train_op = control_flow_ops.group(*train_ops)
with ops.control_dependencies([train_op]):
return state_ops.assign_add(global_step, 1).op
return head.create_estimator_spec(features=features,
mode=mode,
labels=labels,
train_op_fn=_train_op_fn,
logits=logits)
def test_object(self):
opt = optimizers.get_optimizer_instance(_TestOptimizer())
self.assertIsInstance(opt, _TestOptimizer)
if 'batch' in model:
batch_size = model['batch']
else:
batch_size = 128
num_steps = 10000
times = model['times'] # 1
# Network Parameters
num_input = coding['num_input'] # 686 # HPPI data input
num_classes = coding['num_classes'] # 2 # HPPI total classes
hidden_units = model['hidden_units'] # [256, 256, 256]
activation_fn = tf.nn.relu
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
if 'optimizer' in model:
optimizer_name = model['optimizer']
from tensorflow_estimator.python.estimator.canned import optimizers
optimizer = optimizers.get_optimizer_instance(
optimizer_name, learning_rate=learning_rate)
else:
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
cwd = os.getcwd()
data_sets_dir = cwd + "/" + data_set['dir'] # "/data/02-ct-bin"
# model_info = "_{0}({1:d}x{2:d})_{3}_{4}_{5}_{6:g}_dropout_{7:g}".format(model['data_sets_info'], num_input, num_classes, 'x'.join([str(n) for n in hidden_units]), activation_fn.__name__, optimizer.get_name(), learning_rate, dropout)
model_info = "_{0}".format(target)
model_dir = cwd + "/model/train_with_tf_estimator" + model_info
result_file = cwd + "/model/result_of_tf_estimator" + model_info + ".txt"
for _ in range(times):
main()
def test_object_invalid(self):
with self.assertRaisesRegexp(
ValueError, 'The given object is not an Optimizer instance'):
optimizers.get_optimizer_instance((1, 2, 3))
def _dnn_model_fn(features,
labels,
mode,
head,
hidden_units,
feature_columns,
optimizer='Adagrad',
activation_fn=nn.relu,
dropout=None,
input_layer_partitioner=None,
config=None,
use_tpu=False,
batch_norm=False):
"""Deep Neural Net model_fn.
Args:
features: dict of `Tensor`.
labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of
dtype `int32` or `int64` in the range `[0, n_classes)`.
mode: Defines whether this is training, evaluation or prediction.
See `ModeKeys`.
head: A `head_lib._Head` instance.
hidden_units: Iterable of integer number of hidden units per layer.
feature_columns: Iterable of `feature_column._FeatureColumn` model inputs.
optimizer: String, `tf.Optimizer` object, or callable that creates the
optimizer to use for training. If not specified, will use the Adagrad
optimizer with a default learning rate of 0.05.
activation_fn: Activation function applied to each layer.
dropout: When not `None`, the probability we will drop out a given
coordinate.
input_layer_partitioner: Partitioner for input layer. Defaults
to `min_max_variable_partitioner` with `min_slice_size` 64 << 20.
config: `RunConfig` object to configure the runtime settings.
use_tpu: Whether to make a DNN model able to run on TPU. Will make function
return a `_TPUEstimatorSpec` instance and disable variable partitioning.
batch_norm: Whether to use batch normalization after each hidden layer.
Returns:
An `EstimatorSpec` instance.
Raises:
ValueError: If features has the wrong type.
"""
if not isinstance(features, dict):
raise ValueError('features should be a dictionary of `Tensor`s. '
'Given type: {}'.format(type(features)))
optimizer = optimizers.get_optimizer_instance(optimizer,
learning_rate=_LEARNING_RATE)
num_ps_replicas = config.num_ps_replicas if config else 0
partitioner = (None if use_tpu else
partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas))
with variable_scope.variable_scope('dnn',
values=tuple(six.itervalues(features)),
partitioner=partitioner):
input_layer_partitioner = input_layer_partitioner or (
None
if use_tpu else partitioned_variables.min_max_variable_partitioner(
max_partitions=num_ps_replicas, min_slice_size=64 << 20))
logit_fn = dnn_logit_fn_builder(
units=head.logits_dimension,
hidden_units=hidden_units,
feature_columns=feature_columns,
activation_fn=activation_fn,
dropout=dropout,
input_layer_partitioner=input_layer_partitioner,
batch_norm=batch_norm)
logits = logit_fn(features=features, mode=mode)
if use_tpu:
return head._create_tpu_estimator_spec( # pylint: disable=protected-access
features=features,
mode=mode,
labels=labels,
optimizer=optimizer,
logits=logits)
else:
return head.create_estimator_spec(features=features,
mode=mode,
labels=labels,
optimizer=optimizer,
logits=logits)
def test_callable(self):
def _optimizer_fn():
return _TestOptimizer()
opt = optimizers.get_optimizer_instance(_optimizer_fn)
self.assertIsInstance(opt, _TestOptimizer)
def train_op_fn(loss):
opt = optimizers.get_optimizer_instance(optimizer,
learning_rate=_LEARNING_RATE)
return opt.minimize(loss, global_step=training_util.get_global_step())
def test_adagrad(self):
opt = optimizers.get_optimizer_instance('Adagrad', learning_rate=0.1)
self.assertIsInstance(opt, tf.compat.v1.train.AdagradOptimizer)
self.assertAlmostEqual(0.1, opt._learning_rate)