def create_mean_metric(value, name=None):
# TODO(psv): Remove this import when b/110718070 is fixed.
from tensorflow.python.keras import metrics as metrics_module # pylint: disable=g-import-not-at-top
from tensorflow.python.keras.distribute import distributed_training_utils # pylint: disable=g-import-not-at-top
metric_obj = metrics_module.Mean(name=name)
return (metric_obj,
distributed_training_utils.call_replica_local_fn(
metric_obj, value))
def test_multiple_keras_metrics_experimental_run(self, distribution):
with distribution.scope():
loss_metric = metrics.Mean("loss", dtype=np.float32)
loss_metric_2 = metrics.Mean("loss_2", dtype=np.float32)
@def_function.function
def train_step():
def step_fn():
loss = constant_op.constant(5.0, dtype=np.float32)
loss_metric.update_state(loss)
loss_metric_2.update_state(loss)
distribution.run(step_fn)
train_step()
self.assertEqual(loss_metric.result().numpy(),
loss_metric_2.result().numpy())
self.assertEqual(loss_metric.result().numpy(), 5.0)
def metrics(self, regularization_losses=None):
"""Creates metrics. See `base_head.Head` for details."""
keys = metric_keys.MetricKeys
with ops.name_scope('metrics', values=(regularization_losses, )):
# Mean metric.
eval_metrics = {}
eval_metrics[self._loss_mean_key] = metrics.Mean(
name=keys.LOSS_MEAN)
eval_metrics[self._prediction_mean_key] = metrics.Mean(
name=keys.PREDICTION_MEAN)
eval_metrics[self._label_mean_key] = metrics.Mean(
name=keys.LABEL_MEAN)
if regularization_losses is not None:
eval_metrics[self._loss_regularization_key] = metrics.Mean(
name=keys.LOSS_REGULARIZATION)
eval_metrics[self._mean_abs_error] = metrics.MeanAbsoluteError(
name=MEAN_ABS_ERROR, dtype=tf.float32)
return eval_metrics
def test_update_keras_metric_outside_strategy_scope_cross_replica(
self, distribution):
metric = metrics.Mean("test_metric", dtype=np.float32)
with distribution.scope():
for i in range(10):
metric.update_state(i)
# This should be the mean of integers 0-9 which has a sum of 45 and a count
# of 10 resulting in mean of 4.5.
self.assertEqual(metric.result().numpy(), 4.5)
def testEvalMetricOpsWithoutUpdates(self):
with ops.Graph().as_default():
eval_metric_ops = {'mean': metrics.Mean()}
with ops.Graph().as_default(), self.cached_session():
loss = constant_op.constant(1.)
with self.assertRaisesRegexp(ValueError,
'Please call update_state(...)'):
model_fn.EstimatorSpec(mode=ModeKeys.EVAL,
predictions={'loss': loss},
loss=loss,
eval_metric_ops=eval_metric_ops)
def __init__(self, losses, loss_weights=None, output_names=None):
# Keep user-supplied values untouched for recompiling and serialization.
self._user_losses = losses
self._user_loss_weights = loss_weights
self._losses = losses
self._loss_weights = loss_weights
self._output_names = output_names
self._per_output_metrics = None # Per-output losses become metrics.
self._loss_metric = metrics_mod.Mean(name='loss') # Total loss.
self._built = False
def __init__(self, losses, loss_weights=None, output_names=None):
super(LossesContainer, self).__init__(output_names=output_names)
# Keep user-supplied values untouched for recompiling and serialization.
self._user_losses = losses.copy()
self._user_loss_weights = loss_weights
self._losses = losses
self._loss_weights = loss_weights
self._per_output_metrics = None # Per-output losses become metrics.
self._loss_metric = metrics_mod.Mean(name="loss") # Total loss.
self._built = False
def __init__(self):
self.batch_size = batch_size
self.z_dim = z_dim
self.y_dim = 4
self.len_continuous_code = 4
self.learning_rate = learning_rate
self.epochs = epoch
self.cp = np.load(cp_path)
self.cl_cd = np.load(cl_cd_path)
self.datasets = tf.data.Dataset.from_tensor_slices(
(self.cp, self.cl_cd)).shuffle(buffer_size).batch(batch_size, drop_remainder=True)
self.g = generator_model()
self.d, self.c = discriminator_and_classifier_model(self.y_dim)
self.g_optimizer = keras.optimizers.Adam(lr=5 * self.learning_rate, beta_1=0.5)
self.d_optimizer = keras.optimizers.Adam(lr=self.learning_rate, beta_1=0.5)
self.q_optimizer = keras.optimizers.Adam(lr=5 * self.learning_rate, beta_1=0.5)
self.g_loss_metric = metrics.Mean('g_loss', dtype=tf.float32)
self.d_loss_metric = metrics.Mean('d_loss', dtype=tf.float32)
self.q_loss_metric = metrics.Mean('q_loss', dtype=tf.float32)
def _create_metrics(self):
"""Creates per-output loss metrics, but only for multi-output Models."""
if len(self._output_names) == 1:
self._per_output_metrics = [None]
else:
self._per_output_metrics = []
for loss_obj, output_name in zip(self._losses, self._output_names):
if loss_obj is None:
self._per_output_metrics.append(None)
else:
self._per_output_metrics.append(
metrics_mod.Mean(output_name + '_loss'))
def model_fn(features, labels, mode):
_, _ = features, labels
mean = metrics_module.Mean()
mean.update_state(constant_op.constant(2.))
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant(3.),
scaffold=training.Scaffold(saver=training.Saver()),
train_op=constant_op.constant(5.),
eval_metric_ops={
'mean_of_features': mean,
})
def testEvalMetricOpsFromDifferentGraphWithMetricObject(self):
with ops.Graph().as_default():
metric_obj = metrics.Mean()
metric_obj.update_state(constant_op.constant(1.))
eval_metric_ops = {'metric': metric_obj}
with ops.Graph().as_default(), self.cached_session():
loss = constant_op.constant(1.)
with self.assertRaisesRegexp(ValueError,
'must be from the default graph'):
model_fn.EstimatorSpec(mode=ModeKeys.EVAL,
predictions={'loss': loss},
loss=loss,
eval_metric_ops=eval_metric_ops)
def train(self, dataset):
z = tf.constant(
random.normal((self.params.n_samples, 1, 1, self.z_dim)))
g_train_loss = metrics.Mean()
d_train_loss = metrics.Mean()
for epoch in range(self.epochs):
for batch in dataset:
for _ in range(self.n_critic):
self.train_d(batch)
d_loss = self.train_d(batch)
d_train_loss(d_loss)
g_loss = self.train_g()
g_train_loss(g_loss)
self.train_g()
g_train_loss.reset_states()
d_train_loss.reset_states()
samples = self.generate_samples(z)
image_grid = img_merge(samples, n_rows=8).squeeze()
save_image_grid(image_grid, epoch + 1)
def model_fn(features, labels, mode):
_, _ = features, labels
def init_fn(scaffold, session):
_, _ = scaffold, session
mean = metrics_module.Mean()
mean.update_state(tf.constant(2.))
return estimator_lib.EstimatorSpec(
mode,
loss=tf.constant(3.),
scaffold=tf.compat.v1.train.Scaffold(init_fn=init_fn),
train_op=tf.constant(5.),
eval_metric_ops={
'mean_of_features': mean,
})
def _create_metrics(self):
"""Creates per-output loss metrics, but only for multi-output Models."""
# if len(self._output_names) == 1:
# self._per_output_metrics = [None]
# else:
# self._per_output_metrics = []
# for loss_obj, output_name in zip(self._losses, self._output_names):
# if loss_obj is None:
# self._per_output_metrics.append(None)
# else:
# self._per_output_metrics.append(
# metrics_mod.Mean(output_name + '_loss'))
self._per_output_metrics = {}
for key in self.loss_items:
self._per_output_metrics[key] = metrics_mod.Mean(
"{}_loss".format(key))
def model_fn(features, labels, mode):
_ = labels
if estimator_lib.ModeKeys.TRAIN == mode:
with ops.control_dependencies([features]):
train_op = state_ops.assign_add(training.get_global_step(),
1)
return estimator_lib.EstimatorSpec(
mode, loss=constant_op.constant(3.), train_op=train_op)
if estimator_lib.ModeKeys.EVAL == mode:
mean = metrics_module.Mean()
mean.update_state(features)
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant(5.),
eval_metric_ops={
'mean_of_features': mean,
})
def test_update_keras_metric_declared_in_strategy_scope(self, distribution):
with distribution.scope():
metric = metrics.Mean("test_metric", dtype=np.float32)
dataset = dataset_ops.Dataset.range(10).batch(2)
dataset = distribution.experimental_distribute_dataset(dataset)
@def_function.function
def step_fn(i):
metric.update_state(i)
for i in dataset:
distribution.run(step_fn, args=(i,))
# This should be the mean of integers 0-9 which has a sum of 45 and a count
# of 10 resulting in mean of 4.5.
self.assertEqual(metric.result().numpy(), 4.5)
def test_mean_with_sample_weight(self):
m = metrics.Mean(dtype=dtypes.float64)
self.assertEqual(m.dtype, dtypes.float64)
self.evaluate(variables.variables_initializer(m.variables))
# check scalar weight
result_t = m(100, sample_weight=0.5)
self.assertEqual(self.evaluate(result_t), 50 / 0.5)
self.assertEqual(self.evaluate(m.total), 50)
self.assertEqual(self.evaluate(m.count), 0.5)
# check weights not scalar and weights rank matches values rank
result_t = m([1, 5], sample_weight=[1, 0.2])
result = self.evaluate(result_t)
self.assertAlmostEqual(result, 52 / 1.7, 2)
self.assertAlmostEqual(self.evaluate(m.total), 52,
2) # 50 + 1 + 5 * 0.2
self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2) # 0.5 + 1.2
# check weights broadcast
result_t = m([1, 2], sample_weight=0.5)
self.assertAlmostEqual(self.evaluate(result_t), 53.5 / 2.7, 2)
self.assertAlmostEqual(self.evaluate(m.total), 53.5, 2) # 52 + 0.5 + 1
self.assertAlmostEqual(self.evaluate(m.count), 2.7,
2) # 1.7 + 0.5 + 0.5
# check weights squeeze
result_t = m([1, 5], sample_weight=[[1], [0.2]])
self.assertAlmostEqual(self.evaluate(result_t), 55.5 / 3.9, 2)
self.assertAlmostEqual(self.evaluate(m.total), 55.5, 2) # 53.5 + 1 + 1
self.assertAlmostEqual(self.evaluate(m.count), 3.9, 2) # 2.7 + 1.2
# check weights expand
result_t = m([[1], [5]], sample_weight=[1, 0.2])
self.assertAlmostEqual(self.evaluate(result_t), 57.5 / 5.1, 2)
self.assertAlmostEqual(self.evaluate(m.total), 57.5, 2) # 55.5 + 1 + 1
self.assertAlmostEqual(self.evaluate(m.count), 5.1, 2) # 3.9 + 1.2
# check values reduced to the dimensions of weight
result_t = m([[[1., 2.], [3., 2.], [0.5, 4.]]], sample_weight=[0.5])
result = np.round(self.evaluate(result_t), decimals=2) # 58.5 / 5.6
self.assertEqual(result, 10.45)
self.assertEqual(np.round(self.evaluate(m.total), decimals=2), 58.54)
self.assertEqual(np.round(self.evaluate(m.count), decimals=2), 5.6)
def test_mean_graph_with_placeholder(self):
with context.graph_mode(), self.cached_session() as sess:
m = metrics.Mean()
v = array_ops.placeholder(dtypes.float32)
w = array_ops.placeholder(dtypes.float32)
sess.run(variables.variables_initializer(m.variables))
# check __call__()
result_t = m(v, sample_weight=w)
result = sess.run(result_t, feed_dict=({v: 100, w: 0.5}))
self.assertEqual(sess.run(m.total), 50)
self.assertEqual(sess.run(m.count), 0.5)
self.assertEqual(result, 50 / 0.5)
# check update_state() and result()
result = sess.run(result_t, feed_dict=({v: [1, 5], w: [1, 0.2]}))
self.assertAlmostEqual(sess.run(m.total), 52, 2) # 50 + 1 + 5 * 0.2
self.assertAlmostEqual(sess.run(m.count), 1.7, 2) # 0.5 + 1.2
self.assertAlmostEqual(result, 52 / 1.7, 2)
def _build(self, y_pred):
"""One-time setup of loss objects."""
if self._output_names is None:
# In Subclass API, output names like 'output_1' are used for
# `Metric` names.
self._output_names = create_output_names(y_pred)
# Accept a dict of losses keyed by output_name when outputs are a flat
# list.
self._losses = map_to_output_names(y_pred, self._output_names,
self._losses)
self._loss_weights = map_to_output_names(y_pred, self._output_names,
self._loss_weights)
# Broadcast single config values to apply to each output.
if not nest.is_sequence(self._losses):
self._losses = nest.map_structure(lambda output: self._losses,
y_pred)
if not nest.is_sequence(self._loss_weights):
self._loss_weights = nest.map_structure(
lambda output: self._loss_weights, y_pred)
self._losses = nest.map_structure(self._get_loss_object, self._losses)
# Now that structures have been checked, it is safe to flatten.
self._losses = nest.flatten(self._losses)
self._loss_weights = nest.flatten(self._loss_weights)
# Create per-output loss metrics, but only for multi-output Models.
if len(self._output_names) == 1:
self._per_output_metrics = [None]
else:
self._per_output_metrics = []
for loss_obj, output_name in zip(self._losses, self._output_names):
if loss_obj is None:
self._per_output_metrics.append(None)
else:
self._per_output_metrics.append(
metrics_mod.Mean(output_name + '_loss'))
self._built = True
def test_supervised_outputs_no_prepend(self):
"""Tests that no errors are raised when provided outputs are valid."""
loss = {"loss": constant_op.constant([0])}
predictions = {u"predictions": constant_op.constant(["foo"])}
metric_obj = metrics_module.Mean()
metric_obj.update_state(constant_op.constant([0]))
metrics = {
"metrics_1": metric_obj,
"metrics_2": (constant_op.constant([0]), constant_op.constant([10]))
}
outputter = MockSupervisedOutput(loss, predictions, metrics)
self.assertEqual(set(outputter.loss.keys()), set(["loss"]))
self.assertEqual(set(outputter.predictions.keys()), set(["predictions"]))
self.assertEqual(
set(outputter.metrics.keys()),
set([
"metrics_1/value", "metrics_1/update_op", "metrics_2/update_op",
"metrics_2/value"
]))
def model_fn(features, labels, mode):
_, _ = features, labels
w = variables.VariableV1(
initial_value=[0.],
trainable=False,
collections=[ops.GraphKeys.SAVEABLE_OBJECTS])
init_op = control_flow_ops.group(
[w.initializer,
training.get_global_step().initializer])
mean = metrics_module.Mean()
mean.update_state(constant_op.constant(2.))
return estimator_lib.EstimatorSpec(
mode,
loss=constant_op.constant(3.),
scaffold=training.Scaffold(init_op=init_op),
train_op=constant_op.constant(5.),
eval_metric_ops={
'mean_of_features': mean,
})
def test_update_keras_metrics_dynamic_shape(self, distribution):
with distribution.scope():
metric = metrics.Mean("test_metric", dtype=np.float32)
dataset = dataset_ops.Dataset.range(10).batch(2, drop_remainder=False)
@def_function.function
def train_fn(dataset):
weights = constant_op.constant([0.1, 0.1])
def step_fn(i):
metric.update_state(i, weights)
for i in dataset:
distribution.run(step_fn, args=(i,))
train_fn(dataset)
# This should be the mean of integers 0-9 which has a sum of 45 and a count
# of 10 resulting in mean of 4.5.
self.assertEqual(metric.result().numpy(), 4.5)
def test_train_signature_def(self):
loss = {"my_loss": constant_op.constant([0])}
predictions = {u"output1": constant_op.constant(["foo"])}
metric_obj = metrics_module.Mean()
metric_obj.update_state(constant_op.constant([0]))
metrics = {
"metrics_1": metric_obj,
"metrics_2": (constant_op.constant([0]), constant_op.constant([10]))
}
outputter = export_output_lib.TrainOutput(loss, predictions, metrics)
receiver = {u"features": constant_op.constant(100, shape=(100, 2)),
"labels": constant_op.constant(100, shape=(100, 1))}
sig_def = outputter.as_signature_def(receiver)
self.assertTrue("loss/my_loss" in sig_def.outputs)
self.assertTrue("metrics_1/value" in sig_def.outputs)
self.assertTrue("metrics_2/value" in sig_def.outputs)
self.assertTrue("predictions/output1" in sig_def.outputs)
self.assertTrue("features" in sig_def.inputs)
def extract_model_metrics(model):
"""Convert metrics from a Keras model to (value, update) ops.
This is used for converting Keras models to Estimators and SavedModels.
Args:
model: A `tf.keras.Model` object.
Returns:
Dictionary mapping metric names to tuples of (value, update) ops. May return
`None` if the model does not contain any metrics.
"""
if not getattr(model, '_compile_metrics', None):
return None
# TODO(psv/kathywu): use this implementation in model to estimator flow.
eval_metric_ops = {}
for metric_name in model.metrics_names[1:]: # Index 0 is `loss`.
m = metrics.Mean()
m(model._compile_metrics_tensors[metric_name])
eval_metric_ops[metric_name] = m
return eval_metric_ops
def test_supervised_outputs_tuples(self):
"""Tests that no errors are raised when provided outputs are valid."""
loss = {("my", "loss"): constant_op.constant([0])}
predictions = {(u"output1", "2"): constant_op.constant(["foo"])}
metric_obj = metrics_module.Mean()
metric_obj.update_state(constant_op.constant([0]))
metrics = {
("metrics", "1"):
metric_obj,
("metrics", "2"):
(constant_op.constant([0]), constant_op.constant([10]))
}
outputter = MockSupervisedOutput(loss, predictions, metrics)
self.assertEqual(set(outputter.loss.keys()), set(["loss/my/loss"]))
self.assertEqual(set(outputter.predictions.keys()),
set(["predictions/output1/2"]))
self.assertEqual(
set(outputter.metrics.keys()),
set([
"metrics/1/value", "metrics/1/update_op", "metrics/2/value",
"metrics/2/update_op"
]))
def test_mean_with_sample_weight(self):
m = metrics.Mean(dtype=dtypes.float64)
self.assertEqual(m.dtype, dtypes.float64)
self.evaluate(variables.global_variables_initializer())
# check scalar weight
result_t = m(100, sample_weight=0.5)
self.assertEqual(self.evaluate(result_t), 50 / 0.5)
self.assertEqual(self.evaluate(m.total), 50)
self.assertEqual(self.evaluate(m.count), 0.5)
# check weights not scalar and weights rank matches values rank
result_t = m([1, 5], sample_weight=[1, 0.2])
result = self.evaluate(result_t)
self.assertAlmostEqual(result, 52 / 1.7, 2)
self.assertAlmostEqual(self.evaluate(m.total), 52,
2) # 50 + 1 + 5 * 0.2
self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2) # 0.5 + 1.2
# check weights broadcast
result_t = m([1, 2], sample_weight=0.5)
self.assertAlmostEqual(self.evaluate(result_t), 53.5 / 2.7, 2)
self.assertAlmostEqual(self.evaluate(m.total), 53.5, 2) # 52 + 0.5 + 1
self.assertAlmostEqual(self.evaluate(m.count), 2.7,
2) # 1.7 + 0.5 + 0.5
# check weights squeeze
result_t = m([1, 5], sample_weight=[[1], [0.2]])
self.assertAlmostEqual(self.evaluate(result_t), 55.5 / 3.9, 2)
self.assertAlmostEqual(self.evaluate(m.total), 55.5, 2) # 53.5 + 1 + 1
self.assertAlmostEqual(self.evaluate(m.count), 3.9, 2) # 2.7 + 1.2
# check weights expand
result_t = m([[1], [5]], sample_weight=[1, 0.2])
self.assertAlmostEqual(self.evaluate(result_t), 57.5 / 5.1, 2)
self.assertAlmostEqual(self.evaluate(m.total), 57.5, 2) # 55.5 + 1 + 1
self.assertAlmostEqual(self.evaluate(m.count), 5.1, 2) # 3.9 + 1.2
def test_mean(self):
m = metrics.Mean(name='my_mean')
# check config
self.assertEqual(m.name, 'my_mean')
self.assertTrue(m.stateful)
self.assertEqual(m.dtype, dtypes.float32)
self.assertEqual(len(m.variables), 2)
self.evaluate(variables.variables_initializer(m.variables))
# check initial state
self.assertEqual(self.evaluate(m.total), 0)
self.assertEqual(self.evaluate(m.count), 0)
# check __call__()
self.assertEqual(self.evaluate(m(100)), 100)
self.assertEqual(self.evaluate(m.total), 100)
self.assertEqual(self.evaluate(m.count), 1)
# check update_state() and result() + state accumulation + tensor input
update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
self.evaluate(update_op)
self.assertAlmostEqual(self.evaluate(m.result()), 106 / 3, 2)
self.assertEqual(self.evaluate(m.total), 106) # 100 + 1 + 5
self.assertEqual(self.evaluate(m.count), 3)
# check reset_states()
m.reset_states()
self.assertEqual(self.evaluate(m.total), 0)
self.assertEqual(self.evaluate(m.count), 0)
# Check save and restore config
m2 = metrics.Mean.from_config(m.get_config())
self.assertEqual(m2.name, 'my_mean')
self.assertTrue(m2.stateful)
self.assertEqual(m2.dtype, dtypes.float32)
self.assertEqual(len(m2.variables), 2)
def test_supervised_outputs_tuples(self):
"""Tests that no errors are raised when provided outputs are valid."""
with context.graph_mode():
loss = {('my', 'loss'): constant_op.constant([0])}
predictions = {(u'output1', '2'): constant_op.constant(['foo'])}
metric_obj = metrics_module.Mean()
metric_obj.update_state(constant_op.constant([0]))
metrics = {
('metrics', '1'):
metric_obj,
('metrics', '2'):
(constant_op.constant([0]), constant_op.constant([10]))
}
outputter = MockSupervisedOutput(loss, predictions, metrics)
self.assertEqual(set(outputter.loss.keys()), set(['loss/my/loss']))
self.assertEqual(set(outputter.predictions.keys()),
set(['predictions/output1/2']))
self.assertEqual(
set(outputter.metrics.keys()),
set([
'metrics/1/value', 'metrics/1/update_op',
'metrics/2/value', 'metrics/2/update_op'
]))
def test_supervised_outputs_no_prepend(self):
"""Tests that no errors are raised when provided outputs are valid."""
with context.graph_mode():
loss = {'loss': constant_op.constant([0])}
predictions = {u'predictions': constant_op.constant(['foo'])}
metric_obj = metrics_module.Mean()
metric_obj.update_state(constant_op.constant([0]))
metrics = {
'metrics_1':
metric_obj,
'metrics_2':
(constant_op.constant([0]), constant_op.constant([10]))
}
outputter = MockSupervisedOutput(loss, predictions, metrics)
self.assertEqual(set(outputter.loss.keys()), set(['loss']))
self.assertEqual(set(outputter.predictions.keys()),
set(['predictions']))
self.assertEqual(
set(outputter.metrics.keys()),
set([
'metrics_1/value', 'metrics_1/update_op',
'metrics_2/update_op', 'metrics_2/value'
]))
def testAllArgumentsSet(self):
"""Tests that no errors are raised when all arguments are set."""
with ops.Graph().as_default(), self.cached_session():
loss = constant_op.constant(1.)
predictions = {'loss': loss}
classes = constant_op.constant('hello')
metric_obj = metrics.Mean()
metric_obj.update_state(loss)
model_fn.EstimatorSpec(
mode=model_fn.ModeKeys.EVAL,
predictions=predictions,
loss=loss,
train_op=control_flow_ops.no_op(),
eval_metric_ops={
'loss': (control_flow_ops.no_op(), loss),
'mean': metric_obj,
},
export_outputs={
'head_name': export_output.ClassificationOutput(classes=classes)
},
training_chief_hooks=[_FakeHook()],
training_hooks=[_FakeHook()],
scaffold=monitored_session.Scaffold(),
evaluation_hooks=[_FakeHook()])
