def gradient_clipping(grads_and_vars):
"""Internal function for adaptive clipping."""
grads, variables = zip(*grads_and_vars)
norm = clip_ops.global_norm(grads)
max_norm, log_mean = _adaptive_max_norm(norm, std_factor, decay,
global_step, epsilon, name)
# reports the max gradient norm for debugging
if report_summary:
logging_ops.scalar_summary(
"global_norm/adaptive_max_gradient_norm", max_norm)
# factor will be 1. if norm is smaller than max_norm
factor = math_ops.select(norm < max_norm, array_ops.ones_like(norm),
math_ops.exp(log_mean) / norm)
if static_max_norm is not None:
factor = math_ops.minimum(static_max_norm / norm, factor)
# apply factor
clipped_grads = []
for grad in grads:
if grad is None:
clipped_grads.append(None)
elif isinstance(grad, ops.IndexedSlices):
clipped_grads.append(
ops.IndexedSlices(grad.values * factor, grad.indices,
grad.dense_shape))
else:
clipped_grads.append(grad * factor)
return list(zip(clipped_grads, variables))
def batch(tensor_list, batch_size, num_threads=1, capacity=32,
enqueue_many=False, shapes=None, shared_name=None, name=None):
"""Creates batches of tensors in `tensor_list`.
This function is implemented using a queue. A `QueueRunner` for the
queue is added to the current `Graph`'s `QUEUE_RUNNER` collection.
If `enqueue_many` is `False`, `tensor_list` is assumed to represent a
single example. An input tensor with shape `[x, y, z]` will be output
as a tensor with shape `[batch_size, x, y, z]`.
If `enqueue_many` is `True`, `tensor_list` is assumed to represent a
batch of examples, where the first dimension is indexed by example,
and all members of `tensor_list` should have the same size in the
first dimension. If an input tensor has shape `[*, x, y, z]`, the
output will have shape `[batch_size, x, y, z]`. The `capacity` argument
controls the how long the prefetching is allowed to grow the queues.
The returned operation is a dequeue operation and will throw
`tf.errors.OutOfRangeError` if the input queue is exhausted. If this
operation is feeding another input queue, its queue runner will catch
this exception, however, if this operation is used in your main thread
you are responsible for catching this yourself.
*N.B.:* You must ensure that either (i) the `shapes` argument is
passed, or (ii) all of the tensors in `tensor_list` must have
fully-defined shapes. `ValueError` will be raised if neither of
these conditions holds.
Args:
tensor_list: The list of tensors to enqueue.
batch_size: The new batch size pulled from the queue.
num_threads: The number of threads enqueuing `tensor_list`.
capacity: An integer. The maximum number of elements in the queue.
enqueue_many: Whether each tensor in `tensor_list` is a single example.
shapes: (Optional) The shapes for each example. Defaults to the
inferred shapes for `tensor_list`.
shared_name: (optional). If set, this queue will be shared under the given
name across multiple sessions.
name: (Optional) A name for the operations.
Returns:
A list of tensors with the same number and types as `tensor_list`.
Raises:
ValueError: If the `shapes` are not specified, and cannot be
inferred from the elements of `tensor_list`.
"""
with ops.op_scope(tensor_list, name, "batch") as name:
tensor_list = _validate(tensor_list)
types = _dtypes([tensor_list])
shapes = _shapes([tensor_list], shapes, enqueue_many)
# TODO(josh11b,mrry): Switch to BatchQueue once it is written.
queue = data_flow_ops.FIFOQueue(
capacity=capacity, dtypes=types, shapes=shapes, shared_name=shared_name)
_enqueue(queue, tensor_list, num_threads, enqueue_many)
logging_ops.scalar_summary(
"queue/%s/fraction_of_%d_full" % (queue.name, capacity),
math_ops.cast(queue.size(), dtypes.float32) * (1. / capacity))
return queue.dequeue_many(batch_size, name=name)
def _get_batch(per_class_queues, probs, batch_size):
"""Generates batches according to per-class-probabilities."""
num_classes = probs.size
# Number of examples per class is governed by a multinomial distribution.
# Note: multinomial takes unnormalized log probabilities for its first
# argument, of dimension [batch_size, num_classes].
examples = random_ops.multinomial(
np.expand_dims(np.log(probs), 0), batch_size)
# Prepare the data and label batches.
val_list = []
label_list = []
for i in range(num_classes):
num_examples = math_ops.reduce_sum(
math_ops.cast(math_ops.equal(examples, i), dtypes.int32))
val_list.append(per_class_queues[i].dequeue_many(num_examples))
label_list.append(array_ops.ones([num_examples], dtype=dtypes.int32) * i)
# Create a tensor of labels.
batch_labels = array_ops.concat(0, label_list)
batch_labels.set_shape([batch_size])
# Debug instrumentation.
sample_tags = ['stratified_sample/samples_class%i' % i for i in
range(num_classes)]
logging_ops.scalar_summary(sample_tags, math_ops.reduce_sum(
array_ops.one_hot(batch_labels, num_classes), 0))
return array_ops.concat(0, val_list), batch_labels
def _make_per_class_queues(data, labels, num_classes, queue_capacity,
threads_per_queue):
"""Creates per-class-queues based on data and labels."""
# Create one queue per class.
queues = []
per_data_shape = data.get_shape().with_rank_at_least(1)[1:]
per_data_shape.assert_is_fully_defined()
for i in range(num_classes):
q = data_flow_ops.FIFOQueue(capacity=queue_capacity,
shapes=per_data_shape, dtypes=[data.dtype],
name='stratified_sample_class%d_queue' % i)
logging_ops.scalar_summary('queue/stratified_sample_class%d' % i, q.size())
queues.append(q)
# Partition tensors according to labels.
partitions = data_flow_ops.dynamic_partition(data, labels, num_classes)
# Enqueue each tensor on the per-class-queue.
for i in range(num_classes):
enqueue_op = queues[i].enqueue_many(partitions[i]),
queue_runner.add_queue_runner(queue_runner.QueueRunner(
queues[i], [enqueue_op] * threads_per_queue))
return queues
def _get_train_ops(self, features, targets):
"""See base class."""
self._validate_linear_feature_columns(features)
if not isinstance(self._linear_optimizer,
sdca_optimizer.SDCAOptimizer):
return super(LinearClassifier,
self)._get_train_ops(features, targets)
# SDCA currently supports binary classification only.
if self._target_column.num_label_columns > 2:
raise ValueError(
"SDCA does not currently support multi-class classification.")
global_step = contrib_variables.get_global_step()
assert global_step
logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=self._linear_feature_columns,
num_outputs=self._target_column.num_label_columns,
weight_collections=[self._linear_weight_collection],
name="linear")
with ops.control_dependencies([self._centered_bias()]):
loss = self._loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
train_ops = self._linear_optimizer.get_train_step(
self._linear_feature_columns,
self._target_column.weight_column_name, "logistic_loss", features,
targets, columns_to_variables, global_step)
return train_ops, loss
def _get_train_ops(self, features, targets):
"""See base class."""
self._validate_linear_feature_columns(features)
if not isinstance(self._linear_optimizer, sdca_optimizer.SDCAOptimizer):
return super(LinearClassifier, self)._get_train_ops(features, targets)
# SDCA currently supports binary classification only.
if self._target_column.num_label_columns > 2:
raise ValueError(
"SDCA does not currently support multi-class classification.")
global_step = contrib_variables.get_global_step()
assert global_step
logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=self._linear_feature_columns,
num_outputs=self._target_column.num_label_columns,
weight_collections=[self._linear_weight_collection],
name="linear")
with ops.control_dependencies([self._centered_bias()]):
loss = self._loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
train_ops = self._linear_optimizer.get_train_step(
self._linear_feature_columns, self._target_column.weight_column_name,
"logistic_loss", features, targets, columns_to_variables, global_step)
return train_ops, loss
def gradient_clipping(grads_and_vars):
"""Internal function for adaptive clipping."""
grads, variables = zip(*grads_and_vars)
norm = clip_ops.global_norm(grads)
max_norm, log_mean = _adaptive_max_norm(
norm, std_factor, decay, global_step, epsilon, name)
# reports the max gradient norm for debugging
if report_summary:
logging_ops.scalar_summary(
"global_norm/adaptive_max_gradient_norm", max_norm)
# factor will be 1. if norm is smaller than max_norm
factor = math_ops.select(norm < max_norm,
array_ops.ones_like(norm),
math_ops.exp(log_mean) / norm)
if static_max_norm is not None:
factor = math_ops.minimum(static_max_norm / norm, factor)
# apply factor
clipped_grads = []
for grad in grads:
if grad is None:
clipped_grads.append(None)
elif isinstance(grad, ops.IndexedSlices):
clipped_grads.append(ops.IndexedSlices(
grad.values * factor, grad.indices, grad.dense_shape))
else:
clipped_grads.append(grad * factor)
return list(zip(clipped_grads, variables))
def testMergeSummary(self):
with self.cached_session():
c = constant_op.constant(3)
a = logging_ops.scalar_summary('a', c)
b = logging_ops.scalar_summary('b', c)
s = logging_ops.merge_summary([a, b])
self.assertEqual(s.op.type, u'MergeSummary')
def testMergeSummary(self):
with self.cached_session():
c = constant_op.constant(3)
a = logging_ops.scalar_summary('a', c)
b = logging_ops.scalar_summary('b', c)
s = logging_ops.merge_summary([a, b])
self.assertEqual(s.op.type, u'MergeSummary')
def _model_fn(features, labels, mode):
"""Model function."""
assert labels is None, labels
(all_scores, model_predictions, losses,
training_op) = clustering_ops.KMeans(
self._parse_tensor_or_dict(features),
self._num_clusters,
self._training_initial_clusters,
self._distance_metric,
self._use_mini_batch,
random_seed=self._random_seed,
kmeans_plus_plus_num_retries=self.
_kmeans_plus_plus_num_retries).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
logging_ops.scalar_summary('loss/raw', loss)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
KMeansClustering.ALL_SCORES: all_scores[0],
KMeansClustering.CLUSTER_IDX: model_predictions[0],
}
eval_metric_ops = {KMeansClustering.SCORES: loss,}
if self._relative_tolerance is not None:
training_hooks = [self.LossRelativeChangeHook(self._relative_tolerance)]
else:
training_hooks = None
return ModelFnOps(
mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op,
training_hooks=training_hooks)
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._logits(features, is_training=True)
if self._enable_centered_bias:
centered_bias_step = [self._centered_bias_step(targets, features)]
else:
centered_bias_step = []
with ops.control_dependencies(centered_bias_step):
loss = self._loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
linear_vars = self._get_linear_vars()
dnn_vars = self._get_dnn_vars()
grads = gradients.gradients(loss, dnn_vars + linear_vars)
if self._gradient_clip_norm:
grads, _ = clip_ops.clip_by_global_norm(grads, self._gradient_clip_norm)
dnn_grads = grads[0 : len(dnn_vars)]
linear_grads = grads[len(dnn_vars) :]
train_ops = self._get_linear_training_ops(linear_grads, linear_vars) + self._get_dnn_training_ops(
dnn_grads, dnn_vars
)
train_step = control_flow_ops.group(*train_ops, name="combined_training_op")
with ops.control_dependencies([train_step]):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def _get_train_ops(self, features, targets):
"""See base class."""
if not isinstance(self._linear_optimizer, sdca_optimizer.SDCAOptimizer):
return super(LinearRegressor, self)._get_train_ops(features, targets)
assert not self._joint_weights, ("_joint_weights is incompatible with"
" SDCAOptimizer.")
global_step = contrib_variables.get_or_create_global_step()
logits, columns_to_variables, bias = (
layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=self._linear_feature_columns,
num_outputs=self._target_column.num_label_columns,
weight_collections=[self._linear_model.get_scope_name()],
scope=self._linear_model.get_scope_name()))
with ops.control_dependencies([self._centered_bias()]):
loss = self._target_column.loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
_add_bias_column(self._linear_feature_columns, features, bias, targets,
columns_to_variables)
train_op = self._linear_optimizer.get_train_step(
columns_to_variables, self._target_column.weight_column_name,
self._loss_type(), features, targets, global_step)
return train_op, loss
def _make_per_class_queues(tensor_list, labels, num_classes, queue_capacity, threads_per_queue):
"""Creates per-class-queues based on data and labels."""
# Create one queue per class.
queues = []
data_shapes = []
data_dtypes = []
for data_tensor in tensor_list:
per_data_shape = data_tensor.get_shape().with_rank_at_least(1)[1:]
per_data_shape.assert_is_fully_defined()
data_shapes.append(per_data_shape)
data_dtypes.append(data_tensor.dtype)
for i in range(num_classes):
q = data_flow_ops.FIFOQueue(
capacity=queue_capacity, shapes=data_shapes, dtypes=data_dtypes, name="stratified_sample_class%d_queue" % i
)
logging_ops.scalar_summary("queue/%s/stratified_sample_class%d" % (q.name, i), q.size())
queues.append(q)
# Partition tensors according to labels. `partitions` is a list of lists, of
# size num_classes X len(tensor_list). The number of tensors in partition `i`
# should be the same for all tensors.
all_partitions = [data_flow_ops.dynamic_partition(data, labels, num_classes) for data in tensor_list]
partitions = [[cur_partition[i] for cur_partition in all_partitions] for i in range(num_classes)]
# Enqueue each tensor on the per-class-queue.
for i in range(num_classes):
enqueue_op = (queues[i].enqueue_many(partitions[i]),)
queue_runner.add_queue_runner(queue_runner.QueueRunner(queues[i], [enqueue_op] * threads_per_queue))
return queues
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
features = self._get_feature_dict(features)
logits = self._logits(features, is_training=True)
if self._enable_centered_bias:
centered_bias_step = [self._centered_bias_step(targets, features)]
else:
centered_bias_step = []
with ops.control_dependencies(centered_bias_step):
training_loss = self._target_column.training_loss(logits, targets,
features)
weighted_average_loss = self._target_column.loss(logits, targets,
features)
logging_ops.scalar_summary("loss", weighted_average_loss)
linear_train_step = self._linear_model.get_train_step(training_loss)
dnn_train_step = (self._dnn_model.get_train_step(training_loss) if
self._dnn_model else [])
with ops.control_dependencies(linear_train_step + dnn_train_step):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, weighted_average_loss
def _model_fn(features, labels, mode):
"""Model function."""
assert labels is None, labels
(all_scores, model_predictions, losses,
training_op) = clustering_ops.KMeans(
self._parse_tensor_or_dict(features),
self._num_clusters,
self._training_initial_clusters,
self._distance_metric,
self._use_mini_batch,
random_seed=self._random_seed,
kmeans_plus_plus_num_retries=self.
_kmeans_plus_plus_num_retries).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
logging_ops.scalar_summary('loss/raw', loss)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
KMeansClustering.ALL_SCORES: all_scores[0],
KMeansClustering.CLUSTER_IDX: model_predictions[0],
}
eval_metric_ops = {KMeansClustering.SCORES: loss,}
return ModelFnOps(
mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op)
def _get_train_ops(self, features, targets):
"""See base class."""
if not isinstance(self._linear_optimizer,
sdca_optimizer.SDCAOptimizer):
return super(LinearRegressor,
self)._get_train_ops(features, targets)
global_step = contrib_variables.get_or_create_global_step()
logits, columns_to_variables, bias = (
layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=self._linear_feature_columns,
num_outputs=self._target_column.num_label_columns,
weight_collections=[self._linear_model.get_scope_name()],
scope=self._linear_model.get_scope_name()))
with ops.control_dependencies([self._centered_bias()]):
loss = self._target_column.loss(logits, targets, features)
logging_ops.scalar_summary("loss", loss)
train_feature_columns = _maybe_add_bias_column(
self._linear_feature_columns, features, bias, targets,
self._enable_centered_bias, columns_to_variables)
train_op = self._linear_optimizer.get_train_step(
train_feature_columns, self._target_column.weight_column_name,
self._loss_type(), features, targets, columns_to_variables,
global_step)
return train_op, loss
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = contrib_variables.get_global_step()
assert global_step
logits = self._logits(features, is_training=True)
with ops.control_dependencies([
self._centered_bias_step(targets,
self._get_weight_tensor(features))
]):
loss = self._loss(logits, targets,
self._get_weight_tensor(features))
logging_ops.scalar_summary("loss", loss)
linear_vars = self._get_linear_vars()
dnn_vars = self._get_dnn_vars()
grads = gradients.gradients(loss, dnn_vars + linear_vars)
if self._gradient_clip_norm:
grads, _ = clip_ops.clip_by_global_norm(grads,
self._gradient_clip_norm)
dnn_grads = grads[0:len(dnn_vars)]
linear_grads = grads[len(dnn_vars):]
train_ops = self._get_linear_training_ops(
linear_grads, linear_vars) + self._get_dnn_training_ops(
dnn_grads, dnn_vars)
train_step = control_flow_ops.group(*train_ops,
name="combined_training_op")
with ops.control_dependencies([train_step]):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def input_producer(input_tensor, element_shape=None, num_epochs=None,
shuffle=True, seed=None, capacity=32, shared_name=None,
summary_name=None, name=None):
"""Output the rows of `input_tensor` to a queue for an input pipeline.
Args:
input_tensor: A tensor with the rows to produce. Must be at least
one-dimensional. Must either have a fully-defined shape, or
`element_shape` must be defined.
element_shape: (Optional.) A `TensorShape` representing the shape of a
row of `input_tensor`, if it cannot be inferred.
num_epochs: (Optional.) An integer. If specified `input_producer` produces
each row of `input_tensor` `num_epochs` times before generating an
`OutOfRange` error. If not specified, `input_producer` can cycle through
the rows of `input_tensor` an unlimited number of times.
shuffle: (Optional.) A boolean. If true, the rows are randomly shuffled
within each epoch.
seed: (Optional.) An integer. The seed to use if `shuffle` is true.
capacity: (Optional.) The capacity of the queue to be used for buffering
the input.
shared_name: (Optional.) If set, this queue will be shared under the given
name across multiple sessions.
summary_name: (Optional.) If set, a scalar summary for the current queue
size will be generated, using this name as part of the tag.
name: (Optional.) A name for queue.
Returns:
A queue with the output rows. A `QueueRunner` for the queue is
added to the current `QUEUE_RUNNER` collection of the current
graph.
Raises:
ValueError: If the shape of the input cannot be inferred from the arguments.
"""
with ops.name_scope(name, "input_producer", [input_tensor]):
input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")
element_shape = input_tensor.get_shape()[1:].merge_with(element_shape)
if not element_shape.is_fully_defined():
raise ValueError("Either `input_tensor` must have a fully defined shape "
"or `element_shape` must be specified")
if shuffle:
input_tensor = random_ops.random_shuffle(input_tensor, seed=seed)
input_tensor = limit_epochs(input_tensor, num_epochs)
q = data_flow_ops.FIFOQueue(capacity=capacity,
dtypes=[input_tensor.dtype.base_dtype],
shapes=[element_shape],
shared_name=shared_name, name=name)
enq = q.enqueue_many([input_tensor])
queue_runner.add_queue_runner(queue_runner.QueueRunner(q, [enq]))
if summary_name is not None:
logging_ops.scalar_summary("queue/%s/%s" % (q.name, summary_name),
math_ops.cast(q.size(), dtypes.float32) *
(1. / capacity))
return q
def input_producer(input_tensor, element_shape=None, num_epochs=None,
shuffle=True, seed=None, capacity=32, shared_name=None,
summary_name=None, name=None):
"""Output the rows of `input_tensor` to a queue for an input pipeline.
Args:
input_tensor: A tensor with the rows to produce. Must be at
one-dimensional. Must either have a fully-defined shape, or
`element_shape` must be defined.
element_shape: (Optional.) A `TensorShape` representing the shape of a
row of `input_tensor`, if it cannot be inferred.
num_epochs: (Optional.) An integer. If specified `input_producer` produces
each row of `input_tensor` `num_epochs` times before generating an
`OutOfRange` error. If not specified, `input_producer` can cycle through
the rows of `input_tensor` an unlimited number of times.
shuffle: (Optional.) A boolean. If true, the rows are randomly shuffled
within each eopch.
seed: (Optional.) An integer. The seed to use if `shuffle` is true.
capacity: (Optional.) The capacity of the queue to be used for buffering
the input.
shared_name: (Optional.) If set, this queue will be shared under the given
name across multiple sessions.
summary_name: (Optional.) If set, a scalar summary for the current queue
size will be generated, using this name as part of the tag.
name: (Optional.) A name for queue.
Returns:
A queue with the output rows. A `QueueRunner` for the queue is
added to the current `QUEUE_RUNNER` collection of the current
graph.
Raises:
ValueError: If the shape of the input cannot be inferred from the arguments.
"""
with ops.op_scope([input_tensor], name, "input_producer"):
input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")
element_shape = input_tensor.get_shape()[1:].merge_with(element_shape)
if not element_shape.is_fully_defined():
raise ValueError("Either `input_tensor` must have a fully defined shape "
"or `element_shape` must be specified")
if shuffle:
input_tensor = random_ops.random_shuffle(input_tensor, seed=seed)
input_tensor = limit_epochs(input_tensor, num_epochs)
q = data_flow_ops.FIFOQueue(capacity=capacity,
dtypes=[input_tensor.dtype.base_dtype],
shapes=[element_shape],
shared_name=shared_name, name=name)
enq = q.enqueue_many([input_tensor])
queue_runner.add_queue_runner(queue_runner.QueueRunner(q, [enq]))
if summary_name is not None:
logging_ops.scalar_summary("queue/%s/%s" % (q.name, summary_name),
math_ops.cast(q.size(), dtypes.float32) *
(1. / capacity))
return q
def _centered_bias(num_label_columns):
centered_bias = variables.Variable(
array_ops.zeros([num_label_columns]),
collections=[_CENTERED_BIAS, ops.GraphKeys.VARIABLES],
name=_CENTERED_BIAS_WEIGHT)
logging_ops.scalar_summary(
["centered_bias %d" % cb for cb in range(num_label_columns)],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(num_outputs):
centered_bias = variables.Variable(
array_ops.zeros([num_outputs]),
collections=["centered_bias", ops.GraphKeys.VARIABLES],
name="centered_bias_weight")
logging_ops.scalar_summary(
["centered_bias_%d" % cb for cb in range(num_outputs)],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(num_outputs):
centered_bias = variables.Variable(
array_ops.zeros([num_outputs]),
collections=["centered_bias", ops.GraphKeys.VARIABLES],
name="centered_bias_weight")
logging_ops.scalar_summary(
["centered_bias_%d" % cb for cb in range(num_outputs)],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(num_label_columns):
centered_bias = variables.Variable(
array_ops.zeros([num_label_columns]),
collections=[_CENTERED_BIAS, ops.GraphKeys.VARIABLES],
name=_CENTERED_BIAS_WEIGHT)
logging_ops.scalar_summary(
["centered_bias %d" % cb for cb in range(num_label_columns)],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(self):
centered_bias = variables.Variable(
array_ops.zeros([self._num_label_columns()]),
collections=[self._centered_bias_weight_collection,
ops.GraphKeys.VARIABLES],
name="centered_bias_weight")
logging_ops.scalar_summary(
["centered_bias_%d" % cb for cb in range(self._num_label_columns())],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(self):
centered_bias = variables.Variable(
array_ops.zeros([self._num_label_columns()]),
collections=[self._centered_bias_weight_collection,
ops.GraphKeys.VARIABLES],
name="centered_bias_weight")
logging_ops.scalar_summary(
["centered_bias_%d" % cb for cb in range(self._num_label_columns())],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(logits_dimension, weight_collection):
"""Creates and returns centered bias."""
centered_bias = variables.Variable(
array_ops.zeros([logits_dimension]),
collections=[weight_collection, ops.GraphKeys.VARIABLES],
name="centered_bias_weight")
logging_ops.scalar_summary(
["centered_bias_%d" % cb for cb in range(logits_dimension)],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def _centered_bias(logits_dimension, weight_collection):
"""Creates and returns centered bias."""
centered_bias = variables.Variable(
array_ops.zeros([logits_dimension]),
collections=[weight_collection, ops.GraphKeys.VARIABLES],
name="centered_bias_weight")
logging_ops.scalar_summary(
["centered_bias_%d" % cb for cb in range(logits_dimension)],
array_ops.reshape(centered_bias, [-1]))
return centered_bias
def prefetch_queue(tensors,
capacity=8,
shared_name=None,
name=None):
"""Creates a queue to prefetech tensors from `tensors`.
A queue runner for enqueing tensors into the prefetch_queue is automatically
added to the TF QueueRunners collection.
Example:
This is for example useful to pre-assemble input batches read with
`tf.train.batch()` and enqueue the pre-assembled batches. Ops that dequeue
from the pre-assembled queue will not pay the cost of assembling the batch.
images, labels = tf.train.batch([image, label], batch_size=32, num_threads=4)
batch_queue = prefetch_queue([images, labels])
images, labels = batch_queue.dequeue()
logits = Net(images)
loss = Loss(logits, labels)
Args:
tensors: A list or dictionary of `Tensors` to enqueue in the buffer.
capacity: An integer. The maximum number of elements in the queue.
shared_name: (optional). If set, this queue will be shared under the given
name across multiple sessions.
name: (Optional) A name for the operations.
Returns:
A queue from which you can dequeue tensors with the same type and shape
as `tensors`.
"""
if isinstance(tensors, dict):
# Need to wrap the keys and values in list() since Python3 returns views.
# We sort the keys so the order is consistent across runs.
names = list(sorted(tensors.keys()))
tensor_list = list([tensors[n] for n in names])
else:
names = None
tensor_list = tensors
with ops.name_scope(name, "prefetch_queue", tensor_list) as name:
dtypes = [t.dtype for t in tensor_list]
shapes = [t.get_shape() for t in tensor_list]
queue = data_flow_ops.FIFOQueue(capacity=capacity,
dtypes=dtypes,
shapes=shapes,
names=names,
shared_name=shared_name)
enqueue_op = queue.enqueue(tensors, name=name)
queue_runner.add_queue_runner(
queue_runner.QueueRunner(queue, [enqueue_op]))
logging_ops.scalar_summary(
"queue/%s/fraction_of_%d_full" % (queue.name, capacity),
math_ops.to_float(queue.size()) * (1. / capacity))
return queue
def create_estimator_spec(
self, features, mode, logits, labels=None, train_op_fn=None):
"""See `Head`."""
with variable_scope.variable_scope(
None,
default_name='regression_head',
values=(tuple(six.itervalues(features)) + (labels, logits))):
# Predict.
logits = _check_logits(logits, self._logits_dimension)
predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits}
if mode == model_fn.ModeKeys.PREDICT:
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.PREDICT,
predictions=predictions,
export_outputs={'': export_output.RegressionOutput(value=logits)})
# Eval.
labels = _check_labels(_maybe_expand_dim(math_ops.to_float(labels)),
self._logits_dimension)
unweighted_loss = losses.mean_squared_error(
labels=labels, predictions=logits, reduction=losses.Reduction.NONE)
weights = (
1. if (self._weight_feature_key is None) else
features[self._weight_feature_key])
weights = _maybe_expand_dim(math_ops.to_float(weights, name='weights'))
training_loss = losses.compute_weighted_loss(
unweighted_loss, weights=weights, reduction=losses.Reduction.SUM)
if mode == model_fn.ModeKeys.EVAL:
# Estimator already adds a metric for loss.
eval_metric_ops = {
metric_keys.MetricKeys.LOSS_MEAN: metrics_lib.mean(
unweighted_loss, weights=weights)
}
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=training_loss,
eval_metric_ops=eval_metric_ops)
# Train.
if train_op_fn is None:
raise ValueError('train_op_fn can not be None.')
logging_ops.scalar_summary(metric_keys.MetricKeys.LOSS, training_loss)
logging_ops.scalar_summary(
metric_keys.MetricKeys.LOSS_MEAN,
losses.compute_weighted_loss(
unweighted_loss, weights=weights,
reduction=losses.Reduction.MEAN))
return model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.TRAIN,
predictions=predictions,
loss=training_loss,
train_op=train_op_fn(training_loss))
def __init__(self, examples, variables, options):
"""Create a new sdca optimizer."""
if not examples or not variables or not options:
raise ValueError(
'examples, variables and options must all be specified.')
supported_losses = ('logistic_loss', 'squared_loss', 'hinge_loss',
'smooth_hinge_loss')
if options['loss_type'] not in supported_losses:
raise ValueError('Unsupported loss_type: ', options['loss_type'])
self._assertSpecified([
'example_labels', 'example_weights', 'example_ids',
'sparse_features', 'dense_features'
], examples)
self._assertList(['sparse_features', 'dense_features'], examples)
self._assertSpecified(
['sparse_features_weights', 'dense_features_weights'], variables)
self._assertList(['sparse_features_weights', 'dense_features_weights'],
variables)
self._assertSpecified([
'loss_type', 'symmetric_l2_regularization',
'symmetric_l1_regularization'
], options)
for name in [
'symmetric_l1_regularization', 'symmetric_l2_regularization'
]:
value = options[name]
if value < 0.0:
raise ValueError('%s should be non-negative. Found (%f)' %
(name, value))
self._examples = examples
self._variables = variables
self._options = options
self._create_slots()
self._hashtable = _ShardedMutableDenseHashTable(
key_dtype=dtypes.int64,
value_dtype=dtypes.float32,
num_shards=self._num_table_shards(),
default_value=[0.0, 0.0, 0.0, 0.0],
# SdcaFprint never returns 0 or 1 for the low64 bits, so this a safe
# empty_key (that will never collide with actual payloads).
empty_key=[0, 0])
logging_ops.scalar_summary('approximate_duality_gap',
self.approximate_duality_gap())
logging_ops.scalar_summary('examples_seen', self._hashtable.size())
def _get_batch_from_per_class_queues(per_class_queues, probs, batch_size):
"""Generates batches according to per-class-probabilities."""
num_classes = probs.get_shape().num_elements()
# Number of examples per class is governed by a multinomial distribution.
# Note: multinomial takes unnormalized log probabilities for its first
# argument, of dimension [batch_size, num_classes].
examples = random_ops.multinomial(
array_ops.expand_dims(math_ops.log(probs), 0), batch_size)
# Prepare the data and label batches.
val_list = []
label_list = []
for i in range(num_classes):
num_examples = math_ops.reduce_sum(
math_ops.cast(math_ops.equal(examples, i), dtypes.int32))
tensors = per_class_queues[i].dequeue_many(num_examples)
# If you enqueue a list with a single tensor, only a single tensor is
# returned. If you enqueue a list with multiple tensors, then a list is
# returned. We want to handle both cases, so reduce the case of the single
# tensor to the case of multiple tensors.
if not isinstance(tensors, list):
tensors = [tensors]
val_list.append(tensors)
label_list.append(
array_ops.ones([num_examples], dtype=dtypes.int32) * i)
# Create a list of tensor of values. val_list is of dimension
# [num_classes x len(tensors)]. We want list_batch_vals to be of dimension
# [len(tensors)].
num_data = len(val_list[0])
list_batch_vals = [
array_ops.concat(0, [val_list[i][j] for i in range(num_classes)])
for j in range(num_data)
]
# Create a tensor of labels.
batch_labels = array_ops.concat(0, label_list)
batch_labels.set_shape([batch_size])
# Debug instrumentation.
sample_tags = [
'stratified_sample/%s/samples_class%i' % (batch_labels.name, i)
for i in range(num_classes)
]
logging_ops.scalar_summary(
sample_tags,
math_ops.reduce_sum(array_ops.one_hot(batch_labels, num_classes), 0))
return list_batch_vals, batch_labels
def _training_loss(self,
features,
target,
logits=None,
logits_input=None,
name="training_loss"):
"""Returns training loss tensor for this head.
Training loss is different from the loss reported on the tensorboard as we
should respect the example weights when computing the gradient.
L = sum_{i} w_{i} * l_{i} / B
where B is the number of examples in the batch, l_{i}, w_{i} are individual
losses, and example weight.
Args:
features: features dict.
target: either a tensor for labels or in multihead case, a dict of string
to target tensor.
logits: logits, a float tensor.
logits_input: Output of last hidden layer.
name: Op name.
Returns:
A tuple of training Loss and additional_train_op (possibly None)
"""
target = _check_target(target, self._label_name)
centered_bias_step = None
if self._enable_centered_bias:
logits = nn.bias_add(
logits,
_centered_bias(self.logits_dimension,
self._centered_bias_weight_collection))
centered_bias_step = [
_centered_bias_step(self.logits_dimension,
self._centered_bias_weight_collection,
target, self._train_loss_fn)
]
loss_unweighted = self._train_loss_fn(logits, target)
loss, weighted_average_loss = _loss(loss_unweighted,
_weight_tensor(
features,
self._weight_column_name),
name=name)
logging_ops.scalar_summary(_head_prefixed(self._head_name, "loss"),
weighted_average_loss)
return loss, centered_bias_step
def _input_producer(input_tensor, dtype, num_epochs, shuffle, seed, capacity,
shared_name, name, summary_name):
if shuffle:
input_tensor = random_ops.random_shuffle(input_tensor, seed=seed)
input_tensor = limit_epochs(input_tensor, num_epochs)
q = data_flow_ops.FIFOQueue(capacity=capacity, dtypes=[dtype], shapes=[[]],
shared_name=shared_name, name=name)
enq = q.enqueue_many([input_tensor])
queue_runner.add_queue_runner(queue_runner.QueueRunner(q, [enq]))
logging_ops.scalar_summary("queue/%s/%s" % (q.name, summary_name),
math_ops.cast(q.size(), dtypes.float32) *
(1. / capacity))
return q
def _conditional_batch(tensors, accept_prob, batch_size, queue_threads=10):
"""Conditionally enqueue tensors based on accept_prob.
Specifically, enqueue the element if accept_prob > rand_unif([0, 1]).
Args:
tensors: List of tensors to enqueue.
accept_prob: Acceptance probability per example.
batch_size: Size of batch.
queue_threads: Number of threads enqueuing in the final queue.
Returns:
List of batched tensors.
Raises:
ValueError: `accept_prob` isn't 0D.
"""
accept_prob.get_shape().assert_has_rank(0)
# Determine shapes and types of to-be-enqueued-tensors.
shapes_list = []
dtypes_list = []
for tensor in tensors:
cur_shape = tensor.get_shape()
cur_shape.assert_is_fully_defined()
shapes_list.append(cur_shape)
dtypes_list.append(tensor.dtype)
final_q = data_flow_ops.FIFOQueue(capacity=batch_size,
shapes=shapes_list,
dtypes=dtypes_list,
name='batched_queue')
logging_ops.scalar_summary('queue/%s/size' % final_q.name, final_q.size())
# Conditionally enqueue.
# Reshape enqueue op to match no_op's shape.
eq_tf = math_ops.less(random_ops.random_uniform([]), accept_prob)
conditional_enqueue = control_flow_ops.cond(
eq_tf,
lambda: final_q.enqueue(tensors),
control_flow_ops.no_op)
queue_runner.add_queue_runner(queue_runner.QueueRunner(
final_q, [conditional_enqueue] * queue_threads))
out_tensor = final_q.dequeue_many(batch_size)
# Queues return a single tensor if the list of enqued tensors is one. Since we
# want the type to be the same in all cases, always return a list.
if isinstance(out_tensor, ops.Tensor):
out_tensor = [out_tensor]
return out_tensor
def __init__(self,
examples,
variables,
options):
"""Create a new sdca optimizer."""
if not examples or not variables or not options:
raise ValueError('examples, variables and options must all be specified.')
supported_losses = ('logistic_loss', 'squared_loss', 'hinge_loss',
'smooth_hinge_loss')
if options['loss_type'] not in supported_losses:
raise ValueError('Unsupported loss_type: ', options['loss_type'])
self._assertSpecified(['example_labels', 'example_weights', 'example_ids',
'sparse_features', 'dense_features'], examples)
self._assertList(['sparse_features', 'dense_features'], examples)
self._assertSpecified(['sparse_features_weights', 'dense_features_weights'],
variables)
self._assertList(['sparse_features_weights', 'dense_features_weights'],
variables)
self._assertSpecified(['loss_type', 'symmetric_l2_regularization',
'symmetric_l1_regularization'], options)
for name in ['symmetric_l1_regularization', 'symmetric_l2_regularization']:
value = options[name]
if value < 0.0:
raise ValueError('%s should be non-negative. Found (%f)' %
(name, value))
self._examples = examples
self._variables = variables
self._options = options
self._create_slots()
self._hashtable = _ShardedMutableDenseHashTable(
key_dtype=dtypes.int64,
value_dtype=dtypes.float32,
num_shards=self._num_table_shards(),
default_value=[0.0, 0.0, 0.0, 0.0],
# SdcaFprint never returns 0 or 1 for the low64 bits, so this a safe
# empty_key (that will never collide with actual payloads).
empty_key=[0, 0])
logging_ops.scalar_summary('approximate_duality_gap',
self.approximate_duality_gap())
logging_ops.scalar_summary('examples_seen', self._hashtable.size())
def testMergeSummary(self):
with self.cached_session() as sess:
const = constant_op.constant(10.0)
summ1 = summary.histogram("h", const)
summ2 = logging_ops.scalar_summary("c", const)
merge = summary.merge([summ1, summ2])
value = self.evaluate(merge)
self.assertEqual([], merge.get_shape())
self.assertProtoEquals(
"""
value {
tag: "h"
histo {
min: 10.0
max: 10.0
num: 1.0
sum: 10.0
sum_squares: 100.0
bucket_limit: 9.93809490288
bucket_limit: 10.9319043932
bucket_limit: 1.7976931348623157e+308
bucket: 0.0
bucket: 1.0
bucket: 0.0
}
}
value { tag: "c" simple_value: 10.0 }
""", self._AsSummary(value))
def testMergeSummary(self):
with self.cached_session() as sess:
const = constant_op.constant(10.0)
summ1 = summary.histogram("h", const)
summ2 = logging_ops.scalar_summary("c", const)
merge = summary.merge([summ1, summ2])
value = sess.run(merge)
self.assertEqual([], merge.get_shape())
self.assertProtoEquals("""
value {
tag: "h"
histo {
min: 10.0
max: 10.0
num: 1.0
sum: 10.0
sum_squares: 100.0
bucket_limit: 9.93809490288
bucket_limit: 10.9319043932
bucket_limit: 1.7976931348623157e+308
bucket: 0.0
bucket: 1.0
bucket: 0.0
}
}
value { tag: "c" simple_value: 10.0 }
""", self._AsSummary(value))
def _input_producer(input_tensor, dtype, num_epochs, shuffle, seed, capacity,
shared_name, name, summary_name):
if shuffle:
input_tensor = random_ops.random_shuffle(input_tensor, seed=seed)
input_tensor = limit_epochs(input_tensor, num_epochs)
q = data_flow_ops.FIFOQueue(capacity=capacity,
dtypes=[dtype],
shapes=[[]],
shared_name=shared_name,
name=name)
enq = q.enqueue_many([input_tensor])
queue_runner.add_queue_runner(queue_runner.QueueRunner(q, [enq]))
logging_ops.scalar_summary(
"queue/%s/%s" % (q.name, summary_name),
math_ops.cast(q.size(), dtypes.float32) * (1. / capacity))
return q
def sdca_classifier_model_fn(features, targets, mode, params):
"""Estimator's linear model_fn."""
feature_columns = params["feature_columns"]
optimizer = params["optimizer"]
weight_column_name = params["weight_column_name"]
loss_type = params["loss_type"]
enable_centered_bias = params.get("enable_centered_bias", True)
if not isinstance(optimizer, sdca_optimizer.SDCAOptimizer):
raise ValueError("Optimizer must be of type SDCAOptimizer")
loss_fn = {
"logistic_loss": _log_loss_with_two_classes,
"hinge_loss": _hinge_loss,
}[loss_type]
logits, columns_to_variables, bias = (
layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=feature_columns,
num_outputs=1))
if enable_centered_bias:
_add_bias_column(feature_columns, features, bias, targets,
columns_to_variables)
loss = None
if mode != estimator.ModeKeys.INFER:
loss = math_ops.reduce_mean(loss_fn(logits, targets), name="loss")
logging_ops.scalar_summary("loss", loss)
train_op = None
if mode == estimator.ModeKeys.TRAIN:
global_step = contrib_variables.get_global_step()
train_op = optimizer.get_train_step(columns_to_variables,
weight_column_name, loss_type,
features, targets, global_step)
predictions = {}
predictions[_LOGISTIC] = math_ops.sigmoid(logits)
logits = array_ops.concat(1, [array_ops.zeros_like(logits), logits])
predictions[_PROBABILITIES] = nn.softmax(logits)
predictions[_CLASSES] = math_ops.argmax(logits, 1)
return predictions, loss, train_op
def _get_batch_from_per_class_queues(per_class_queues, probs, batch_size):
"""Generates batches according to per-class-probabilities."""
num_classes = probs.get_shape().num_elements()
# Number of examples per class is governed by a multinomial distribution.
# Note: multinomial takes unnormalized log probabilities for its first
# argument, of dimension [batch_size, num_classes].
examples = random_ops.multinomial(
array_ops.expand_dims(math_ops.log(probs), 0), batch_size)
# Prepare the data and label batches.
val_list = []
label_list = []
for i in range(num_classes):
num_examples = math_ops.reduce_sum(
math_ops.cast(math_ops.equal(examples, i), dtypes.int32))
tensors = per_class_queues[i].dequeue_many(num_examples)
# If you enqueue a list with a single tensor, only a single tensor is
# returned. If you enqueue a list with multiple tensors, then a list is
# returned. We want to handle both cases, so reduce the case of the single
# tensor to the case of multiple tensors.
if not isinstance(tensors, list):
tensors = [tensors]
val_list.append(tensors)
label_list.append(array_ops.ones([num_examples], dtype=dtypes.int32) * i)
# Create a list of tensor of values. val_list is of dimension
# [num_classes x len(tensors)]. We want list_batch_vals to be of dimension
# [len(tensors)].
num_data = len(val_list[0])
list_batch_vals = [array_ops.concat(
0, [val_list[i][j] for i in range(num_classes)]) for j in range(num_data)]
# Create a tensor of labels.
batch_labels = array_ops.concat(0, label_list)
batch_labels.set_shape([batch_size])
# Debug instrumentation.
sample_tags = ['stratified_sample/%s/samples_class%i' % (batch_labels.name, i)
for i in range(num_classes)]
logging_ops.scalar_summary(sample_tags, math_ops.reduce_sum(
array_ops.one_hot(batch_labels, num_classes), 0))
return list_batch_vals, batch_labels
def sdca_classifier_model_fn(features, targets, mode, params):
"""Estimator's linear model_fn."""
feature_columns = params["feature_columns"]
optimizer = params["optimizer"]
weight_column_name = params["weight_column_name"]
loss_type = params["loss_type"]
enable_centered_bias = params.get("enable_centered_bias", True)
if not isinstance(optimizer, sdca_optimizer.SDCAOptimizer):
raise ValueError("Optimizer must be of type SDCAOptimizer")
loss_fn = {
"logistic_loss": _log_loss_with_two_classes,
"hinge_loss": _hinge_loss,
}[loss_type]
logits, columns_to_variables, bias = (
layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=feature_columns,
num_outputs=1))
if enable_centered_bias:
_add_bias_column(feature_columns, features, bias, targets,
columns_to_variables)
loss = None
if mode != estimator.ModeKeys.INFER:
loss = math_ops.reduce_mean(loss_fn(logits, targets), name="loss")
logging_ops.scalar_summary("loss", loss)
train_op = None
if mode == estimator.ModeKeys.TRAIN:
global_step = contrib_variables.get_global_step()
train_op = optimizer.get_train_step(
columns_to_variables, weight_column_name, loss_type, features,
targets, global_step)
predictions = {}
predictions[_LOGISTIC] = math_ops.sigmoid(logits)
logits = array_ops.concat(1, [array_ops.zeros_like(logits), logits])
predictions[_PROBABILITIES] = nn.softmax(logits)
predictions[_CLASSES] = math_ops.argmax(logits, 1)
return predictions, loss, train_op
def _training_loss(self, features, target, logits=None,
logits_input=None, name="training_loss"):
"""Returns training loss tensor for this head.
Training loss is different from the loss reported on the tensorboard as we
should respect the example weights when computing the gradient.
L = sum_{i} w_{i} * l_{i} / B
where B is the number of examples in the batch, l_{i}, w_{i} are individual
losses, and example weight.
Args:
features: features dict.
target: either a tensor for labels or in multihead case, a dict of string
to target tensor.
logits: logits, a float tensor.
logits_input: Output of last hidden layer.
name: Op name.
Returns:
A tuple of training Loss and additional_train_op (possibly None)
"""
target = _check_target(target, self._label_name)
centered_bias_step = None
if self._enable_centered_bias:
logits = nn.bias_add(logits, _centered_bias(
self.logits_dimension,
self._centered_bias_weight_collection))
centered_bias_step = [_centered_bias_step(
self.logits_dimension,
self._centered_bias_weight_collection,
target,
self._train_loss_fn)]
loss_unweighted = self._train_loss_fn(logits, target)
loss, weighted_average_loss = _loss(
loss_unweighted,
_weight_tensor(features, self._weight_column_name),
name=name)
logging_ops.scalar_summary(_head_prefixed(self._head_name, "loss"),
weighted_average_loss)
return loss, centered_bias_step
def testScalarSummaryDefaultName(self):
with self.cached_session() as sess:
const = constant_op.constant([10.0, 20.0])
summ = logging_ops.scalar_summary(["c1", "c2"], const)
value = self.evaluate(summ)
self.assertEqual([], summ.get_shape())
self.assertProtoEquals("""
value { tag: "c1" simple_value: 10.0 }
value { tag: "c2" simple_value: 20.0 }
""", self._AsSummary(value))
def testScalarSummaryDefaultName(self):
with self.cached_session() as sess:
const = constant_op.constant([10.0, 20.0])
summ = logging_ops.scalar_summary(["c1", "c2"], const)
value = sess.run(summ)
self.assertEqual([], summ.get_shape())
self.assertProtoEquals("""
value { tag: "c1" simple_value: 10.0 }
value { tag: "c2" simple_value: 20.0 }
""", self._AsSummary(value))
def define_graph(self):
"""
Sets up the model graph in TensorFlow.
"""
with tf.name_scope('discriminator'):
##
# Setup scale networks. Each will make the predictions for images at a given scale.
##
self.scale_nets = []
for scale_num in range(self.num_scale_nets):
with tf.name_scope('scale_net_' + str(scale_num)):
scale_factor = 1. / 2**(
(self.num_scale_nets - 1) - scale_num)
self.scale_nets.append(
DScaleModel(scale_num, int(self.height * scale_factor),
int(self.width * scale_factor),
self.scale_conv_layer_fms[scale_num],
self.scale_kernel_sizes[scale_num],
self.scale_fc_layer_sizes[scale_num]))
# A list of the prediction tensors for each scale network
self.scale_preds = []
for scale_num in range(self.num_scale_nets):
self.scale_preds.append(self.scale_nets[scale_num].preds)
##
# Data
##
self.labels = tf.placeholder(tf.float32,
shape=[None, 1],
name='labels')
##
# Training
##
with tf.name_scope('training'):
# global loss is the combined loss from every scale network
self.global_loss = adv_loss(self.scale_preds, self.labels)
self.global_step = tf.Variable(0,
trainable=False,
name='global_step')
self.optimizer = tf.train.GradientDescentOptimizer(
c.LRATE_D, name='optimizer')
self.train_op = self.optimizer.minimize(
self.global_loss,
global_step=self.global_step,
name='train_op')
# add summaries to visualize in TensorBoard
loss_summary = scalar_summary('loss_D', self.global_loss)
self.summaries = merge_summary([loss_summary])
def _get_train_ops(self, features, targets):
"""See base class."""
global_step = variables.get_global_step()
assert global_step
loss = self._loss(self._logits(features), targets, self._get_weight_tensor(features))
logging_ops.scalar_summary("loss", loss)
linear_vars = self._get_linear_vars()
dnn_vars = self._get_dnn_vars()
grads = gradients.gradients(loss, dnn_vars + linear_vars)
dnn_grads = grads[0 : len(dnn_vars)]
linear_grads = grads[len(dnn_vars) :]
train_ops = self._get_linear_training_ops(linear_grads, linear_vars) + self._get_dnn_training_ops(
dnn_grads, dnn_vars
)
train_step = control_flow_ops.group(*train_ops, name="combined_training_op")
with ops.control_dependencies([train_step]):
with ops.get_default_graph().colocate_with(global_step):
return state_ops.assign_add(global_step, 1).op, loss
def _make_per_class_queues(tensor_list, labels, num_classes, queue_capacity,
threads_per_queue):
"""Creates per-class-queues based on data and labels."""
# Create one queue per class.
queues = []
data_shapes = []
data_dtypes = []
for data_tensor in tensor_list:
per_data_shape = data_tensor.get_shape().with_rank_at_least(1)[1:]
per_data_shape.assert_is_fully_defined()
data_shapes.append(per_data_shape)
data_dtypes.append(data_tensor.dtype)
for i in range(num_classes):
q = data_flow_ops.FIFOQueue(capacity=queue_capacity,
shapes=data_shapes,
dtypes=data_dtypes,
name='stratified_sample_class%d_queue' % i)
logging_ops.scalar_summary(
'queue/%s/stratified_sample_class%d' % (q.name, i), q.size())
queues.append(q)
# Partition tensors according to labels. `partitions` is a list of lists, of
# size num_classes X len(tensor_list). The number of tensors in partition `i`
# should be the same for all tensors.
all_partitions = [
data_flow_ops.dynamic_partition(data, labels, num_classes)
for data in tensor_list
]
partitions = [[cur_partition[i] for cur_partition in all_partitions]
for i in range(num_classes)]
# Enqueue each tensor on the per-class-queue.
for i in range(num_classes):
enqueue_op = queues[i].enqueue_many(partitions[i]),
queue_runner.add_queue_runner(
queue_runner.QueueRunner(queues[i],
[enqueue_op] * threads_per_queue))
return queues
def create_queue(self, shared_name=None, name=None):
from tensorflow.python.ops import data_flow_ops, logging_ops, math_ops
from tensorflow.python.framework import dtypes
assert self.dtypes is not None and self.shapes is not None
assert len(self.dtypes) == len(self.shapes)
capacity = self.queue_size
self._queue = data_flow_ops.FIFOQueue(capacity=capacity,
dtypes=self.dtypes,
shapes=self.shapes,
shared_name=shared_name,
name=name)
enq = self._queue.enqueue_many(self.batch_phs)
# create a queue runner
queue_runner.add_queue_runner(
queue_runner.QueueRunner(self._queue, [enq] * self.nthreads,
feed_dict_op=[lambda: self.next_batch()],
feed_dict_key=self.batch_phs))
summary_name = 'fraction_of_%d_full' % capacity
logging_ops.scalar_summary(
"queue/%s/%s" % (self._queue.name, summary_name),
math_ops.cast(self._queue.size(), dtypes.float32) *
(1. / capacity))
def testScalarSummary(self):
with self.test_session() as sess:
const = constant_op.constant([10.0, 20.0])
summ = logging_ops.scalar_summary(["c1", "c2"],
const,
name="mysumm")
value = sess.run(summ)
self.assertEqual([], summ.get_shape())
self.assertProtoEquals(
"""
value { tag: "c1" simple_value: 10.0 }
value { tag: "c2" simple_value: 20.0 }
""", self._AsSummary(value))
def _get_linear_train_and_loss_ops(features, target, linear_feature_columns,
target_column, linear_optimizer, loss_type,
centered_bias, scope_name):
"""Returns train and loss ops for SDCAOptimizer."""
global_step = contrib_variables.get_global_step()
assert global_step
logits, columns_to_variables, _ = layers.weighted_sum_from_feature_columns(
columns_to_tensors=features,
feature_columns=linear_feature_columns,
num_outputs=target_column.num_label_columns,
weight_collections=[scope_name],
scope=scope_name)
with ops.control_dependencies([centered_bias]):
loss = target_column.loss(logits, target, features)
logging_ops.scalar_summary("loss", loss)
train_op = linear_optimizer.get_train_step(linear_feature_columns,
target_column.weight_column_name,
loss_type, features, target,
columns_to_variables, global_step)
return train_op, loss
def __init__(self, examples, variables, options):
"""Create a new sdca optimizer."""
if not examples or not variables or not options:
raise ValueError("examples, variables and options must all be specified.")
supported_losses = ("logistic_loss", "squared_loss", "hinge_loss")
if options["loss_type"] not in supported_losses:
raise ValueError("Unsupported loss_type: ", options["loss_type"])
self._assertSpecified(
["example_labels", "example_weights", "example_ids", "sparse_features", "dense_features"], examples
)
self._assertList(["sparse_features", "dense_features"], examples)
self._assertSpecified(["sparse_features_weights", "dense_features_weights"], variables)
self._assertList(["sparse_features_weights", "dense_features_weights"], variables)
self._assertSpecified(["loss_type", "symmetric_l2_regularization", "symmetric_l1_regularization"], options)
for name in ["symmetric_l1_regularization", "symmetric_l2_regularization"]:
value = options[name]
if value < 0.0:
raise ValueError("%s should be non-negative. Found (%f)" % (name, value))
self._examples = examples
self._variables = variables
self._options = options
self._create_slots()
self._hashtable = _ShardedMutableHashTable(
key_dtype=dtypes.string,
value_dtype=dtypes.float32,
num_shards=self._num_table_shards(),
default_value=[0.0, 0.0, 0.0, 0.0],
)
logging_ops.scalar_summary("approximate_duality_gap", self.approximate_duality_gap())
logging_ops.scalar_summary("examples_seen", self._hashtable.size())
def _kmeans_clustering_model_fn(features, labels, mode, params, config):
"""Model function for KMeansClustering estimator."""
assert labels is None, labels
(all_scores, model_predictions, losses,
is_initialized, init_op, training_op) = clustering_ops.KMeans(
_parse_tensor_or_dict(features),
params.get('num_clusters'),
initial_clusters=params.get('training_initial_clusters'),
distance_metric=params.get('distance_metric'),
use_mini_batch=params.get('use_mini_batch'),
mini_batch_steps_per_iteration=params.get(
'mini_batch_steps_per_iteration'),
random_seed=params.get('random_seed'),
kmeans_plus_plus_num_retries=params.get(
'kmeans_plus_plus_num_retries')).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
logging_ops.scalar_summary('loss/raw', loss)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
KMeansClustering.ALL_SCORES: all_scores[0],
KMeansClustering.CLUSTER_IDX: model_predictions[0],
}
eval_metric_ops = {KMeansClustering.SCORES: loss}
training_hooks = [_InitializeClustersHook(
init_op, is_initialized, config.is_chief)]
relative_tolerance = params.get('relative_tolerance')
if relative_tolerance is not None:
training_hooks.append(_LossRelativeChangeHook(relative_tolerance))
return ModelFnOps(
mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op,
training_hooks=training_hooks)
def _kmeans_clustering_model_fn(features, labels, mode, params, config):
"""Model function for KMeansClustering estimator."""
assert labels is None, labels
(all_scores, model_predictions, losses, is_initialized, init_op,
training_op) = clustering_ops.KMeans(
_parse_tensor_or_dict(features),
params.get('num_clusters'),
initial_clusters=params.get('training_initial_clusters'),
distance_metric=params.get('distance_metric'),
use_mini_batch=params.get('use_mini_batch'),
mini_batch_steps_per_iteration=params.get(
'mini_batch_steps_per_iteration'),
random_seed=params.get('random_seed'),
kmeans_plus_plus_num_retries=params.get(
'kmeans_plus_plus_num_retries')).training_graph()
incr_step = state_ops.assign_add(variables.get_global_step(), 1)
loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
logging_ops.scalar_summary('loss/raw', loss)
training_op = with_dependencies([training_op, incr_step], loss)
predictions = {
KMeansClustering.ALL_SCORES: all_scores[0],
KMeansClustering.CLUSTER_IDX: model_predictions[0],
}
eval_metric_ops = {KMeansClustering.SCORES: loss}
training_hooks = [
_InitializeClustersHook(init_op, is_initialized, config.is_chief)
]
relative_tolerance = params.get('relative_tolerance')
if relative_tolerance is not None:
training_hooks.append(_LossRelativeChangeHook(relative_tolerance))
return ModelFnOps(mode=mode,
predictions=predictions,
eval_metric_ops=eval_metric_ops,
loss=loss,
train_op=training_op,
training_hooks=training_hooks)
