def test_build_ensemble_spec(
self,
want_logits,
want_loss=None,
want_adanet_loss=None,
want_ensemble_trainable_vars=None,
adanet_lambda=0.,
adanet_beta=0.,
ensemble_spec_fn=lambda: None,
use_bias=False,
use_logits_last_layer=False,
mixture_weight_type=MixtureWeightType.MATRIX,
mixture_weight_initializer=tf_compat.v1.zeros_initializer(),
warm_start_mixture_weights=True,
subnetwork_builder_class=_Builder,
mode=tf.estimator.ModeKeys.TRAIN,
multi_head=False,
want_subnetwork_trainable_vars=2,
ensembler_class=ComplexityRegularizedEnsembler,
my_ensemble_index=None,
want_replay_indices=None,
want_predictions=None,
export_subnetworks=False,
previous_ensemble_spec=None,
previous_iteration_checkpoint=None):
seed = 64
if multi_head:
head = multi_head_lib.MultiHead(heads=[
binary_class_head.BinaryClassHead(
name="head1", loss_reduction=tf_compat.SUM),
binary_class_head.BinaryClassHead(name="head2",
loss_reduction=tf_compat.SUM)
])
else:
head = binary_class_head.BinaryClassHead(
loss_reduction=tf_compat.SUM)
builder = _EnsembleBuilder(
head=head,
export_subnetwork_logits=export_subnetworks,
export_subnetwork_last_layer=export_subnetworks)
def _subnetwork_train_op_fn(loss, var_list):
self.assertLen(var_list, want_subnetwork_trainable_vars)
self.assertEqual(
var_list,
tf_compat.v1.get_collection(
tf_compat.v1.GraphKeys.TRAINABLE_VARIABLES))
# Subnetworks get iteration steps instead of global steps.
self.assertEqual("subnetwork_test/iteration_step",
tf_compat.v1.train.get_global_step().op.name)
# Subnetworks get scoped summaries.
self.assertEqual("fake_scalar",
tf_compat.v1.summary.scalar("scalar", 1.))
self.assertEqual("fake_image",
tf_compat.v1.summary.image("image", 1.))
self.assertEqual("fake_histogram",
tf_compat.v1.summary.histogram("histogram", 1.))
self.assertEqual("fake_audio",
tf_compat.v1.summary.audio("audio", 1., 1.))
optimizer = tf_compat.v1.train.GradientDescentOptimizer(
learning_rate=.1)
return optimizer.minimize(loss, var_list=var_list)
def _mixture_weights_train_op_fn(loss, var_list):
self.assertLen(var_list, want_ensemble_trainable_vars)
self.assertEqual(
var_list,
tf_compat.v1.get_collection(
tf_compat.v1.GraphKeys.TRAINABLE_VARIABLES))
# Subnetworks get iteration steps instead of global steps.
self.assertEqual("ensemble_test/iteration_step",
tf_compat.v1.train.get_global_step().op.name)
# Subnetworks get scoped summaries.
self.assertEqual("fake_scalar",
tf_compat.v1.summary.scalar("scalar", 1.))
self.assertEqual("fake_image",
tf_compat.v1.summary.image("image", 1.))
self.assertEqual("fake_histogram",
tf_compat.v1.summary.histogram("histogram", 1.))
self.assertEqual("fake_audio",
tf_compat.v1.summary.audio("audio", 1., 1.))
if not var_list:
return tf.no_op()
optimizer = tf_compat.v1.train.GradientDescentOptimizer(
learning_rate=.1)
return optimizer.minimize(loss, var_list=var_list)
previous_ensemble = None
previous_ensemble_spec = ensemble_spec_fn()
if previous_ensemble_spec:
previous_ensemble = previous_ensemble_spec.ensemble
subnetwork_manager = _SubnetworkManager(head)
subnetwork_builder = subnetwork_builder_class(
_subnetwork_train_op_fn,
_mixture_weights_train_op_fn,
use_logits_last_layer,
seed,
multi_head=multi_head)
with tf.Graph().as_default() as g:
tf_compat.v1.train.get_or_create_global_step()
# A trainable variable to later verify that creating models does not
# affect the global variables collection.
_ = tf_compat.v1.get_variable("some_var", shape=0, trainable=True)
features = {"x": tf.constant([[1.], [2.]])}
if multi_head:
labels = {
"head1": tf.constant([0, 1]),
"head2": tf.constant([0, 1])
}
else:
labels = tf.constant([0, 1])
session_config = tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True))
subnetwork_spec = subnetwork_manager.build_subnetwork_spec(
name="test",
subnetwork_builder=subnetwork_builder,
summary=_FakeSummary(),
features=features,
mode=mode,
labels=labels,
previous_ensemble=previous_ensemble)
ensembler_kwargs = {}
if ensembler_class is ComplexityRegularizedEnsembler:
ensembler_kwargs.update({
"mixture_weight_type": mixture_weight_type,
"mixture_weight_initializer": mixture_weight_initializer,
"warm_start_mixture_weights": warm_start_mixture_weights,
"model_dir": self.test_subdirectory,
"adanet_lambda": adanet_lambda,
"adanet_beta": adanet_beta,
"use_bias": use_bias
})
if ensembler_class is MeanEnsembler:
ensembler_kwargs.update(
{"add_mean_last_layer_predictions": True})
ensemble_spec = builder.build_ensemble_spec(
# Note: when ensemble_spec is not None and warm_start_mixture_weights
# is True, we need to make sure that the bias and mixture weights are
# already saved to the checkpoint_dir.
name="test",
previous_ensemble_spec=previous_ensemble_spec,
candidate=EnsembleCandidate("foo", [subnetwork_builder], None),
ensembler=ensembler_class(**ensembler_kwargs),
subnetwork_specs=[subnetwork_spec],
summary=_FakeSummary(),
features=features,
iteration_number=1,
labels=labels,
my_ensemble_index=my_ensemble_index,
mode=mode,
previous_iteration_checkpoint=previous_iteration_checkpoint)
if want_replay_indices:
self.assertAllEqual(want_replay_indices,
ensemble_spec.architecture.replay_indices)
with tf_compat.v1.Session(
graph=g, config=session_config).as_default() as sess:
sess.run(tf_compat.v1.global_variables_initializer())
# Equals the number of subnetwork and ensemble trainable variables,
# plus the one 'some_var' created earlier.
self.assertLen(
tf_compat.v1.trainable_variables(),
want_subnetwork_trainable_vars +
want_ensemble_trainable_vars + 1)
# Get the real global step outside a subnetwork's context.
self.assertEqual("global_step",
tf_compat.v1.train.get_global_step().op.name)
self.assertEqual("global_step",
train.get_global_step().op.name)
self.assertEqual("global_step",
tf_v1.train.get_global_step().op.name)
self.assertEqual("global_step",
training_util.get_global_step().op.name)
self.assertEqual(
"global_step",
tf_compat.v1.train.get_or_create_global_step().op.name)
self.assertEqual("global_step",
train.get_or_create_global_step().op.name)
self.assertEqual(
"global_step",
tf_v1.train.get_or_create_global_step().op.name)
self.assertEqual(
"global_step",
training_util.get_or_create_global_step().op.name)
# Get global tf.summary outside a subnetwork's context.
self.assertNotEqual("fake_scalar",
tf_compat.v1.summary.scalar("scalar", 1.))
self.assertNotEqual("fake_image",
tf_compat.v1.summary.image("image", 1.))
self.assertNotEqual(
"fake_histogram",
tf_compat.v1.summary.histogram("histogram", 1.))
self.assertNotEqual(
"fake_audio", tf_compat.v1.summary.audio("audio", 1., 1.))
if mode == tf.estimator.ModeKeys.PREDICT:
self.assertAllClose(want_logits,
sess.run(
ensemble_spec.ensemble.logits),
atol=1e-3)
self.assertIsNone(ensemble_spec.loss)
self.assertIsNone(ensemble_spec.adanet_loss)
self.assertIsNone(ensemble_spec.train_op)
self.assertIsNotNone(ensemble_spec.export_outputs)
if not export_subnetworks:
return
if not multi_head:
subnetwork_logits = sess.run(
ensemble_spec.export_outputs[
_EnsembleBuilder.
_SUBNETWORK_LOGITS_EXPORT_SIGNATURE].outputs)
self.assertAllClose(
subnetwork_logits["test"],
sess.run(subnetwork_spec.subnetwork.logits))
subnetwork_last_layer = sess.run(
ensemble_spec.export_outputs[
_EnsembleBuilder.
_SUBNETWORK_LAST_LAYER_EXPORT_SIGNATURE].
outputs)
self.assertAllClose(
subnetwork_last_layer["test"],
sess.run(subnetwork_spec.subnetwork.last_layer))
else:
self.assertIn("subnetwork_logits_head2",
ensemble_spec.export_outputs)
subnetwork_logits_head1 = sess.run(
ensemble_spec.
export_outputs["subnetwork_logits_head1"].outputs)
self.assertAllClose(
subnetwork_logits_head1["test"],
sess.run(
subnetwork_spec.subnetwork.logits["head1"]))
self.assertIn("subnetwork_logits_head2",
ensemble_spec.export_outputs)
subnetwork_last_layer_head1 = sess.run(
ensemble_spec.export_outputs[
"subnetwork_last_layer_head1"].outputs)
self.assertAllClose(
subnetwork_last_layer_head1["test"],
sess.run(subnetwork_spec.subnetwork.
last_layer["head1"]))
return
# Verify that train_op works, previous loss should be greater than loss
# after a train op.
loss = sess.run(ensemble_spec.loss)
train_op = tf.group(subnetwork_spec.train_op.train_op,
ensemble_spec.train_op.train_op)
for _ in range(3):
sess.run(train_op)
self.assertGreater(loss, sess.run(ensemble_spec.loss))
self.assertAllClose(want_logits,
sess.run(ensemble_spec.ensemble.logits),
atol=1e-3)
if ensembler_class is ComplexityRegularizedEnsembler:
# Bias should learn a non-zero value when used.
bias = sess.run(ensemble_spec.ensemble.bias)
if isinstance(bias, dict):
bias = sum(abs(b) for b in bias.values())
if use_bias:
self.assertNotEqual(0., bias)
else:
self.assertAlmostEqual(0., bias)
self.assertAlmostEqual(want_loss,
sess.run(ensemble_spec.loss),
places=3)
self.assertAlmostEqual(want_adanet_loss,
sess.run(ensemble_spec.adanet_loss),
places=3)
if want_predictions:
self.assertAllClose(
want_predictions,
sess.run(ensemble_spec.ensemble.predictions),
atol=1e-3)
def testConvertTfjsLayersModelIntoShardedWeights(self):
with tf.Graph().as_default(), tf.compat.v1.Session():
x = np.random.randn(8, 10)
# 1. Run the model.predict(), store the result. Then saved the model
# as a SavedModel.
model = self._createNestedSequentialModel()
y = model.predict(x)
weights = model.get_weights()
total_weight_bytes = sum(np.size(w) for w in weights) * 4
tf.keras.models.save_model(model, self._tmp_dir)
# 2. Convert the keras saved model to tfjs_layers_model format.
tfjs_output_dir = os.path.join(self._tmp_dir, 'tfjs')
# Implicit value of --output_format: tfjs_layers_model
process = subprocess.Popen([
'tensorflowjs_converter', '--input_format', 'keras_saved_model',
self._tmp_dir, tfjs_output_dir
])
process.communicate()
self.assertEqual(0, process.returncode)
# 3. Convert the tfjs_layers_model to another tfjs_layers_model,
# with sharded weights.
weight_shard_size_bytes = int(total_weight_bytes * 0.3)
# Due to the shard size, there ought to be 4 shards after conversion.
sharded_model_dir = os.path.join(self._tmp_dir, 'tfjs_sharded')
process = subprocess.Popen([
'tensorflowjs_converter', '--input_format', 'tfjs_layers_model',
'--output_format', 'tfjs_layers_model',
'--weight_shard_size_bytes', str(weight_shard_size_bytes),
os.path.join(tfjs_output_dir, 'model.json'), sharded_model_dir
])
process.communicate()
self.assertEqual(0, process.returncode)
# 4. Check the sharded weight files and their sizes.
weight_files = sorted(
glob.glob(os.path.join(sharded_model_dir, 'group*.bin')))
self.assertEqual(len(weight_files), 4)
weight_file_sizes = [os.path.getsize(f) for f in weight_files]
self.assertEqual(sum(weight_file_sizes), total_weight_bytes)
self.assertEqual(weight_file_sizes[0], weight_file_sizes[1])
self.assertEqual(weight_file_sizes[0], weight_file_sizes[2])
self.assertLess(weight_file_sizes[3], weight_file_sizes[0])
# 5. Convert the sharded tfjs_layers_model back into a keras h5 file.
new_h5_path = os.path.join(self._tmp_dir, 'new_h5.h5')
process = subprocess.Popen([
'tensorflowjs_converter', '--input_format', 'tfjs_layers_model',
os.path.join(sharded_model_dir, 'model.json'), new_h5_path
])
process.communicate()
self.assertEqual(0, process.returncode)
with tf.Graph().as_default(), tf.compat.v1.Session():
# 6. Load the keras model and check the predict() output is close to
# before.
new_model = keras.models.load_model(new_h5_path)
new_y = new_model.predict(x)
self.assertAllClose(new_y, y)
def test_generate_candidates(self,
want_names,
want_subnetwork_losses,
want_mixture_weight_losses,
want_complexities,
learn_mixture_weights=False,
initial_num_layers=0,
previous_ensemble=None):
feature_columns = [tf.feature_column.numeric_column("x")]
generator = simple_dnn.Generator(
feature_columns=feature_columns,
optimizer=tf.compat.v1.train.GradientDescentOptimizer(.1),
layer_size=3,
initial_num_layers=initial_num_layers,
learn_mixture_weights=learn_mixture_weights,
seed=42)
with context.graph_mode(), tf.Graph().as_default() as g:
iteration_step = tf.compat.v1.train.create_global_step()
features = {"x": [[1.], [2.]]}
labels = tf.constant([[0.], [1.]])
names = []
subnetwork_losses = []
mixture_weight_losses = []
complexities = []
for builder in generator.generate_candidates(
previous_ensemble,
# The following arguments are not used by
# simple_dnn.BuilderGenerator's generate_candidates.
iteration_number=0,
previous_ensemble_reports=[],
all_reports=[]):
names.append(builder.name)
# 1. Build subnetwork graph.
subnetwork = builder.build_subnetwork(
features,
logits_dimension=1,
training=True,
iteration_step=iteration_step,
summary=tf.summary,
previous_ensemble=previous_ensemble)
# 2. Build subnetwork train ops.
subnetwork_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=subnetwork.logits, labels=labels))
subnetwork_train_op = builder.build_subnetwork_train_op(
subnetwork,
subnetwork_loss,
var_list=None,
labels=labels,
iteration_step=iteration_step,
summary=tf.summary,
previous_ensemble=None)
# 3. Build mixture weight train ops.
# Stop gradients since mixture weights should have not propagate
# beyond top layer.
subnetwork_logits = tf.stop_gradient(subnetwork.logits)
# Mixture weight will initialize to a one-valued scalar.
mixture_weight_logits = tf.compat.v1.layers.dense(
subnetwork_logits,
units=1,
use_bias=False,
kernel_initializer=tf.ones_initializer())
mixture_weight_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=mixture_weight_logits, labels=labels))
mixture_weight_train_op = builder.build_mixture_weights_train_op(
mixture_weight_loss,
var_list=None,
labels=labels,
logits=mixture_weight_logits,
iteration_step=iteration_step,
summary=tf.summary)
with self.test_session(graph=g) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(subnetwork_train_op)
sess.run(mixture_weight_train_op)
subnetwork_losses.append(sess.run(subnetwork_loss))
mixture_weight_losses.append(sess.run(mixture_weight_loss))
complexities.append(sess.run(subnetwork.complexity))
self.assertEqual(want_names, names)
self.assertAllClose(want_subnetwork_losses,
subnetwork_losses,
atol=1e-3)
self.assertAllClose(want_mixture_weight_losses,
mixture_weight_losses,
atol=1e-3)
self.assertAllClose(want_complexities, complexities, atol=1e-3)
def testDenseWithLearningRateInverseTimeDecay(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
with tf.Graph().as_default():
var0_np = np.array([1.0, 2.0])
grads0_np = np.array([0.1, 0.2])
var1_np = np.array([3.0, 4.0])
grads1_np = np.array([0.01, 0.2])
var0 = tf.Variable(var0_np)
var1 = tf.Variable(var1_np)
grads0 = tf.constant(grads0_np)
grads1 = tf.constant(grads1_np)
learning_rate = 0.01
rho = 0.9
momentum = 0.0
epsilon = 1e-7
centered = False
decay = 0.5
lr_schedule = learning_rate_schedule.InverseTimeDecay(
learning_rate, decay_steps=1.0, decay_rate=decay)
opt = rmsprop.RMSprop(learning_rate=lr_schedule,
rho=rho,
momentum=momentum,
epsilon=epsilon,
centered=centered)
update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
rms0 = opt.get_slot(var0, "rms")
self.assertIsNotNone(rms0)
rms1 = opt.get_slot(var1, "rms")
self.assertIsNotNone(rms1)
if momentum > 0.:
mom0 = opt.get_slot(var0, "momentum")
mom1 = opt.get_slot(var1, "momentum")
else:
mom0 = None
mom1 = None
mg0_np = np.array([0.0, 0.0])
mg1_np = np.array([0.0, 0.0])
rms0_np = np.array([0.0, 0.0])
rms1_np = np.array([0.0, 0.0])
mom0_np = np.array([0.0, 0.0])
mom1_np = np.array([0.0, 0.0])
# Fetch params to validate initial values
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Run 4 steps of RMSprop
for t in range(2):
self.evaluate(update)
lr = learning_rate / (1 + decay * t)
var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
var0_np, grads0_np, mg0_np, rms0_np, mom0_np, lr, rho,
momentum, epsilon, centered)
var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
var1_np, grads1_np, mg1_np, rms1_np, mom1_np, lr, rho,
momentum, epsilon, centered)
# Validate updated params
self.assertAllCloseAccordingToType(rms0_np,
self.evaluate(rms0))
self.assertAllCloseAccordingToType(rms1_np,
self.evaluate(rms1))
if momentum > 0.:
self.assertAllCloseAccordingToType(mom0_np,
self.evaluate(mom0))
self.assertAllCloseAccordingToType(mom1_np,
self.evaluate(mom1))
self.assertAllCloseAccordingToType(var0_np,
self.evaluate(var0))
self.assertAllCloseAccordingToType(var1_np,
self.evaluate(var1))
def _import_and_infer(save_dir, inputs, signature_key="serving_default"):
"""Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
graph = tf.Graph()
with graph.as_default(), tf.compat.v1.Session() as session:
model = tf.compat.v1.saved_model.load(session, ["serve"], save_dir)
return _run_signature(session, model, inputs, signature_key)
def _export_mode(mode, has_saved_vars, builder, model, custom_objects,
checkpoint_path, input_signature):
"""Exports a model, and optionally saves new vars from the clone model.
Args:
mode: A `tf.estimator.ModeKeys` string.
has_saved_vars: A `boolean` indicating whether the SavedModel has already
exported variables.
builder: A `SavedModelBuilder` object.
model: A `tf.keras.Model` object.
custom_objects: A dictionary mapping string names to custom classes
or functions.
checkpoint_path: String path to checkpoint.
input_signature: Nested TensorSpec containing the expected inputs. Can be
`None`, in which case the signature will be inferred from the model.
Raises:
ValueError: If the train/eval mode is being exported, but the model does
not have an optimizer.
"""
compile_clone = (mode != mode_keys.ModeKeys.PREDICT)
if compile_clone and not model.optimizer:
raise ValueError(
'Model does not have an optimizer. Cannot export mode %s' % mode)
model_graph = tf.compat.v1.get_default_graph()
with tf.Graph().as_default() as g, backend.learning_phase_scope(
mode == mode_keys.ModeKeys.TRAIN):
if input_signature is None:
input_tensors = None
else:
input_tensors = tf.nest.map_structure(create_placeholder,
input_signature)
# Clone the model into blank graph. This will create placeholders for inputs
# and targets.
clone = models_lib.clone_and_build_model(model,
input_tensors=input_tensors,
custom_objects=custom_objects,
compile_clone=compile_clone)
# Make sure that iterations variable is added to the global step collection,
# to ensure that, when the SavedModel graph is loaded, the iterations
# variable is returned by `tf.compat.v1.train.get_global_step()`. This is
# required for compatibility with the SavedModelEstimator.
if compile_clone:
g.add_to_collection(tf.compat.v1.GraphKeys.GLOBAL_STEP,
clone.optimizer.iterations)
# Extract update and train ops from train/test/predict functions.
train_op = None
if mode == mode_keys.ModeKeys.TRAIN:
clone._make_train_function() # pylint: disable=protected-access
train_op = clone.train_function.updates_op
elif mode == mode_keys.ModeKeys.TEST:
clone._make_test_function() # pylint: disable=protected-access
else:
clone._make_predict_function() # pylint: disable=protected-access
g.get_collection_ref(tf.compat.v1.GraphKeys.UPDATE_OPS).extend(
clone.state_updates)
with tf.compat.v1.Session().as_default():
clone_var_list = _get_var_list(clone)
if has_saved_vars:
# Confirm all variables in the clone have an entry in the checkpoint.
status = clone.load_weights(checkpoint_path)
status.assert_existing_objects_matched()
else:
# Confirm that variables between the clone and model match up exactly,
# not counting optimizer objects. Optimizer objects are ignored because
# if the model has not trained, the slot variables will not have been
# created yet.
# TODO(b/113179535): Replace with trackable equivalence.
_assert_same_non_optimizer_objects(model, model_graph, clone,
g)
# TODO(b/113178242): Use value transfer for trackable objects.
clone.load_weights(checkpoint_path)
# Add graph and variables to SavedModel.
# TODO(b/113134168): Switch to add_meta_graph_and_variables.
clone.save_weights(checkpoint_path,
save_format='tf',
overwrite=True)
builder._has_saved_variables = True # pylint: disable=protected-access
# Add graph to the SavedModel builder.
builder.add_meta_graph(
model_utils.EXPORT_TAG_MAP[mode],
signature_def_map=_create_signature_def_map(clone, mode),
saver=tf.compat.v1.train.Saver(
clone_var_list,
# Allow saving Models with no variables. This is somewhat odd, but
# it's not necessarily a bug.
allow_empty=True),
init_op=tf.compat.v1.local_variables_initializer(),
train_op=train_op)
return None
def test_with_1d_unknown_shape_sparse_tensor(self):
embedding_values = (
(1.0, 2.0), # id 0
(6.0, 7.0), # id 1
(11.0, 12.0), # id 2
)
def _initializer(shape, dtype, partition_info=None):
del shape, dtype, partition_info
return embedding_values
# price has 1 dimension in dense_features
price = tf.feature_column.numeric_column("price")
# one_hot_body_style has 3 dims in dense_features.
body_style = tf.feature_column.categorical_column_with_vocabulary_list(
"body-style", vocabulary_list=["hardtop", "wagon", "sedan"]
)
one_hot_body_style = tf.feature_column.indicator_column(body_style)
# embedded_body_style has 5 dims in dense_features.
country = tf.feature_column.categorical_column_with_vocabulary_list(
"country", vocabulary_list=["US", "JP", "CA"]
)
embedded_country = tf.feature_column.embedding_column(
country, dimension=2, initializer=_initializer
)
# Provides 1-dim tensor and dense tensor.
with tf.Graph().as_default():
features = {
"price": tf.compat.v1.placeholder(tf.float32),
"body-style": tf.compat.v1.sparse_placeholder(tf.string),
# This is dense tensor for the categorical_column.
"country": tf.compat.v1.placeholder(tf.string),
}
self.assertIsNone(features["price"].shape.ndims)
self.assertIsNone(features["body-style"].get_shape().ndims)
self.assertIsNone(features["country"].shape.ndims)
price_data = np.array([11.0, 12.0])
body_style_data = tf.compat.v1.SparseTensorValue(
indices=((0,), (1,)),
values=("sedan", "hardtop"),
dense_shape=(2,),
)
country_data = np.array([["US"], ["CA"]])
net = df.DenseFeatures(
[price, one_hot_body_style, embedded_country]
)(features)
self.assertEqual(1 + 3 + 2, net.shape[1])
with _initialized_session() as sess:
# Each row is formed by concatenating `embedded_body_style`,
# `one_hot_body_style`, and `price` in order.
self.assertAllEqual(
[
[0.0, 0.0, 1.0, 1.0, 2.0, 11.0],
[1.0, 0.0, 0.0, 11.0, 12.0, 12.0],
],
sess.run(
net,
feed_dict={
features["price"]: price_data,
features["body-style"]: body_style_data,
features["country"]: country_data,
},
),
)
def main():
#download google pre-trained neural network
local_zip_file = 'inception5h.zip'
if not os.path.exists(local_zip_file):
#download
model_url = urllib.request.urlopen(url)
with open(local_zip_file, 'wb') as output:
output.write(model_url.read())
#extract
with zipfile.ZipFile(local_zip_file, 'r') as zip_ref:
zip_ref.extractall(data_dir)
model_fn = 'tensorflow_inseption_graph.pb'
#Creating tf session and loading the model
graph = tf.Graph()
sess = tfc.InteractiveSession(graph=graph)
with tfc.gfile.FastGFile((local_zip_file), 'rb') as f:
graph_def = tf.io.gfile.GFile()
graph_def.ParseFromString(f.read())
t_input = tf.placeholder(np.float32, name='input') #define input tensor
imagenet_mean = 117.0
t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0)
tf.import_graph_def(graph_def, {'input': t_preprocessed})
layers = [
op.name for op in graph.get_operations()
if op.type == 'Cony2D' and 'import/' in op.name
]
feature_nums = [
int(graph.get_tensor_by_name(model_name + ':0').get_shape()[-1])
for name in layers
]
print('Number of layers: ', len(layers))
print('Total numbers of feature channels:', sum(feature_nums))
def render_deepdream(t_obj,
img0=img_noise,
iter_n=10,
step=1.5,
octave_n=4,
octave_scale=1.4):
t_score = tf.reduce_mean(t_obj) #defining optimization objective
t_grad = tf.gradients(t_score, t_input)[0]
#split the image into a number of octaves
img = img0
octaves = []
for _ in range(octave_n - 1):
hw = img.shape[:2]
lo = resize(img, np.int32(np.float32(hw) / octave_scale))
hi = img - resize(low, hw)
img = lo
octaves.append(hi)
#generate details octave by octave
for octave in range(octave_n):
if octave > 0:
hi = octaves[-octave]
img = resize(img, hi.shape[:2]) + hi
for _ in range(iter_n):
g = calc_grad_tiled(img, t_grad)
img += g * (step / (np.abs(g).mean() + 1e-7))
#output deep dreamed image
showarray(img / 255.0)
#Pick a layer to enchance my image
layer = 'mixed4d_3x3_bottleneck_pre_relu'
channel = 139
img0 = PIL.Image.open('image.jpg')
img0 = np.float32(img0)
#Apply gradient ascent to the layer
render_deepdream(tf.square(T('mixed4c')), img0)
def testSparse(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
with tf.Graph().as_default():
for dtype in [tf.half, tf.float32, tf.float64]:
var0 = tf.Variable(tf.zeros([4, 2], dtype=dtype))
var1 = tf.Variable(tf.constant(1.0, dtype, [4, 2]))
grads0 = tf.IndexedSlices(tf.constant([[.1, .1]], dtype=dtype),
tf.constant([1]), tf.constant([4,
2]))
grads1 = tf.IndexedSlices(
tf.constant([[.01, .01], [.01, .01]], dtype=dtype),
tf.constant([2, 3]), tf.constant([4, 2]))
mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9)
mom_update = mom_opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
# Check we have slots
slot0 = mom_opt.get_slot(var0, "momentum")
self.assertEqual(slot0.shape, var0.shape)
slot1 = mom_opt.get_slot(var1, "momentum")
self.assertEqual(slot1.shape, var1.shape)
# Fetch params to validate initial values
self.assertAllClose([0, 0], self.evaluate(var0)[0])
self.assertAllClose([0, 0], self.evaluate(var0)[1])
self.assertAllClose([1, 1], self.evaluate(var1)[2])
# Step 1: the momentum accumulators are 0. So we should see a normal
# update: v -= grad * learning_rate
self.evaluate(mom_update)
# Check that the momentum accumulators have been updated.
self.assertAllCloseAccordingToType(np.array([0, 0]),
self.evaluate(slot0)[0])
self.assertAllCloseAccordingToType(
np.array([-2.0 * .1, -2.0 * .1]),
self.evaluate(slot0)[1])
self.assertAllCloseAccordingToType(
np.array([-2.0 * .01, -2.0 * .01]),
self.evaluate(slot1)[2])
# Check that the parameters have been updated.
self.assertAllCloseAccordingToType(np.array([0, 0]),
self.evaluate(var0)[0])
self.assertAllCloseAccordingToType(
np.array([-(0.1 * 2.0), -(0.1 * 2.0)]),
self.evaluate(var0)[1])
self.assertAllCloseAccordingToType(
np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]),
self.evaluate(var1)[2])
# Step 2: the momentum accumulators contain the previous update.
self.evaluate(mom_update)
# Check that the momentum accumulators have been updated.
self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0])
self.assertAllCloseAccordingToType(
np.array([(0.9 * (-0.2) - 2.0 * 0.1),
(0.9 * (-0.2) - 2.0 * 0.1)]),
self.evaluate(slot0)[1])
self.assertAllCloseAccordingToType(
np.array([(0.9 * (-0.02) - 2.0 * 0.01),
(0.9 * (-0.02) - 2.0 * 0.01)]),
self.evaluate(slot1)[2])
# Check that the parameters have been updated.
self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0])
self.assertAllCloseAccordingToType(
np.array([
-(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
-(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
]),
self.evaluate(var0)[1])
self.assertAllCloseAccordingToType(
np.array([
0.98 - ((0.9 * 0.01 + 0.01) * 2.0),
0.98 - ((0.9 * 0.01 + 0.01) * 2.0)
]),
self.evaluate(var1)[2])
def test_import_tower(self, shared_input):
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
phoenix_spec.is_input_shared = shared_input
features = {}
shared_input_tensor = None
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_1 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1")
tf.random.set_seed(1234)
tower_spec_1 = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor_1,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_1 = sess.run(tower_spec_1.logits_spec.logits,
feed_dict={input_tensor_1: test_input})
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_2 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2")
if shared_input:
shared_input_tensor = input_tensor_2
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del features, is_training, scope_name, lengths_feature_name
return None, None
else:
features = {"x": input_tensor_2}
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del is_training, lengths_feature_name
with tf.compat.v1.variable_scope(scope_name):
return tf.cast(features["x"],
dtype=tf.float32), None
tf.random.set_seed(1234)
tower_spec_2 = architecture_utils.import_tower(
features=features,
input_layer_fn=_input_layer_fn,
phoenix_spec=phoenix_spec,
shared_input_tensor=shared_input_tensor,
original_tower_name="test_tower",
new_tower_name="imported_tower",
model_directory=directory,
is_training=True,
logits_dimension=10,
shared_lengths=None,
force_snapshot=False,
force_freeze=False)
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_2 = sess.run(tower_spec_2.logits_spec.logits,
feed_dict={input_tensor_2: test_input})
self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
def testSaveAndLoadSavedModelExport(
self, model_builder, uses_learning_phase, optimizer_cls,
train_before_export):
optimizer = None if optimizer_cls is None else optimizer_cls()
saved_model_dir = self._save_model_dir()
np.random.seed(130)
input_arr = np.random.random((1, 3))
target_arr = np.random.random((1, 3))
model = model_builder(uses_learning_phase)
if optimizer is not None:
model.compile(
loss='mse',
optimizer=optimizer,
metrics=['mae'])
if train_before_export:
model.train_on_batch(input_arr, target_arr)
ref_loss, ref_mae = model.evaluate(input_arr, target_arr)
ref_predict = model.predict(input_arr)
# Export SavedModel
keras_saved_model.export_saved_model(model, saved_model_dir)
input_name = model.input_names[0]
output_name = model.output_names[0]
target_name = output_name + '_target'
# Load predict graph, and test predictions
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
inputs, outputs, _ = load_model(sess, saved_model_dir,
mode_keys.ModeKeys.PREDICT)
predictions = sess.run(outputs[output_name],
{inputs[input_name]: input_arr})
self.assertAllClose(ref_predict, predictions, atol=1e-05)
if optimizer:
# Load eval graph, and test predictions, loss and metric values
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
inputs, outputs, _ = load_model(sess, saved_model_dir,
mode_keys.ModeKeys.TEST)
# First obtain the loss and predictions, and run the metric update op by
# feeding in the inputs and targets.
metrics_name = 'mae' if tf.__internal__.tf2.enabled() else 'mean_absolute_error'
metrics_update_op_key = 'metrics/' + metrics_name + '/update_op'
metrics_value_op_key = 'metrics/' + metrics_name + '/value'
loss, predictions, _ = sess.run(
(outputs['loss'], outputs['predictions/' + output_name],
outputs[metrics_update_op_key]), {
inputs[input_name]: input_arr,
inputs[target_name]: target_arr
})
# The metric value should be run after the update op, to ensure that it
# reflects the correct value.
metric_value = sess.run(outputs[metrics_value_op_key])
self.assertEqual(int(train_before_export),
sess.run(tf.compat.v1.train.get_global_step()))
self.assertAllClose(ref_loss, loss, atol=1e-05)
self.assertAllClose(ref_mae, metric_value, atol=1e-05)
self.assertAllClose(ref_predict, predictions, atol=1e-05)
# Load train graph, and check for the train op, and prediction values
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
inputs, outputs, meta_graph_def = load_model(
sess, saved_model_dir, mode_keys.ModeKeys.TRAIN)
self.assertEqual(int(train_before_export),
sess.run(tf.compat.v1.train.get_global_step()))
self.assertIn('loss', outputs)
self.assertIn(metrics_update_op_key, outputs)
self.assertIn(metrics_value_op_key, outputs)
self.assertIn('predictions/' + output_name, outputs)
# Train for a step
train_op = get_train_op(meta_graph_def)
train_outputs, _ = sess.run(
[outputs, train_op], {inputs[input_name]: input_arr,
inputs[target_name]: target_arr})
self.assertEqual(int(train_before_export) + 1,
sess.run(tf.compat.v1.train.get_global_step()))
if uses_learning_phase:
self.assertAllClose(
[[0, 0, 0]], train_outputs['predictions/' + output_name],
atol=1e-05)
else:
self.assertNotAllClose(
[[0, 0, 0]], train_outputs['predictions/' + output_name],
atol=1e-05)
"""Script to generate fake 'Lost and Found' data."""
import tensorflow.compat.v2 as tf
import tensorflow_datasets.testing.cityscapes as cityscapes
if __name__ == '__main__':
example_dir = ('tensorflow_datasets/testing/test_data/fake_examples/'
'lost_and_found')
base_path = example_dir + '/{}.zip'
# generate image ids matching between zipfiles
train_ids = list(cityscapes.generate_ids('01_Turmstr_17')) + list(
cityscapes.generate_ids('02_Goethe_Str_6'))
test_ids = list(cityscapes.generate_ids('03_Schlossallee_1'))
splits = {'train': train_ids, 'test': test_ids}
with tf.Graph().as_default():
cityscapes.create_zipfile(
base_path.format('leftImg8bit'),
splits_with_ids=splits,
suffixes=['leftImg8bit'])
cityscapes.create_zipfile(
base_path.format('gtCoarse'),
splits_with_ids=splits,
suffixes=[
'gtCoarse_instanceIds', 'gtCoarse_labelIds', 'gtCoarse_color'
])
cityscapes.create_zipfile(
base_path.format('rightImg8bit'),
splits_with_ids=splits,
suffixes=['rightImg8bit'])
cityscapes.create_zipfile(
def testBasicWithLearningRateDecay(self):
for i, dtype in enumerate([tf.half, tf.float32, tf.float64]):
with self.session(graph=tf.Graph(), use_gpu=True):
# Initialize variables for numpy implementation.
m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np, name="var0_%d" % i)
var1 = tf.Variable(var1_np, name="var1_%d" % i)
grads0 = tf.constant(grads0_np)
grads1 = tf.constant(grads1_np)
learning_rate = 0.001
decay = 0.002
opt = adamax.Adamax(learning_rate=learning_rate, decay=decay)
if not tf.executing_eagerly():
update = opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
if not tf.executing_eagerly():
self.evaluate(tf.compat.v1.global_variables_initializer())
# Fetch params to validate initial values
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Run 3 steps of Adamax
for t in range(3):
beta_1_power = get_beta_accumulators(opt, dtype)
self.assertAllCloseAccordingToType(
0.9**(t + 1), self.evaluate(beta_1_power))
if not tf.executing_eagerly():
self.evaluate(update)
else:
opt.apply_gradients(zip([grads0, grads1],
[var0, var1]))
lr = learning_rate / (1 + decay * t)
var0_np, m0, v0 = adamax_update_numpy(var0_np,
grads0_np,
t,
m0,
v0,
alpha=lr)
var1_np, m1, v1 = adamax_update_numpy(var1_np,
grads1_np,
t,
m1,
v1,
alpha=lr)
# Validate updated params
self.assertAllCloseAccordingToType(var0_np,
self.evaluate(var0),
rtol=1e-2)
self.assertAllCloseAccordingToType(var1_np,
self.evaluate(var1),
rtol=1e-2)
def testSparseBasic(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
with tf.Graph().as_default():
for dtype in _DATA_TYPES:
var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0, 0.01],
dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np)
var1 = tf.Variable(var1_np)
grads0_np_indices = np.array([0, 2], dtype=np.int32)
grads0 = tf.IndexedSlices(
tf.constant(grads0_np[grads0_np_indices]),
tf.constant(grads0_np_indices),
tf.constant([3]),
)
grads1_np_indices = np.array([0, 2], dtype=np.int32)
grads1 = tf.IndexedSlices(
tf.constant(grads1_np[grads1_np_indices]),
tf.constant(grads1_np_indices),
tf.constant([3]),
)
learning_rate = 3.0
ada_opt = adagrad.Adagrad(learning_rate)
ada_update = ada_opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
# Fetch params to validate initial values
self.assertAllClose([1.0, 1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 3.0, 4.0], self.evaluate(var1))
accum0_np = np.array([0.1, 0.1, 0.1],
dtype=dtype.as_numpy_dtype)
accum1_np = np.array([0.1, 0.1, 0.1],
dtype=dtype.as_numpy_dtype)
# Run 3 step of sgd
for _ in range(3):
self.evaluate(ada_update)
var0_np, accum0_np = sparse_adagrad_update_numpy(
var0_np,
accum0_np,
grads0_np_indices,
grads0_np[grads0_np_indices],
learning_rate,
)
var1_np, accum1_np = sparse_adagrad_update_numpy(
var1_np,
accum1_np,
grads1_np_indices,
grads1_np[grads1_np_indices],
learning_rate,
)
self.assertAllCloseAccordingToType(var0_np,
self.evaluate(var0))
self.assertAllCloseAccordingToType(var1_np,
self.evaluate(var1))
def test_shared_embedding_column(self):
with tf.Graph().as_default():
vocabulary_size = 3
sparse_input_a = tf.compat.v1.SparseTensorValue(
# example 0, ids [2]
# example 1, ids [0, 1]
indices=((0, 0), (1, 0), (1, 1)),
values=(2, 0, 1),
dense_shape=(2, 2))
sparse_input_b = tf.compat.v1.SparseTensorValue(
# example 0, ids [1]
# example 1, ids [2, 0]
indices=((0, 0), (1, 0), (1, 1)),
values=(1, 2, 0),
dense_shape=(2, 2))
embedding_dimension = 2
embedding_values = (
(1., 2.), # id 0
(3., 4.), # id 1
(5., 6.) # id 2
)
def _get_initializer(embedding_dimension, embedding_values):
def _initializer(shape, dtype, partition_info=None):
self.assertAllEqual((vocabulary_size, embedding_dimension),
shape)
self.assertEqual(tf.float32, dtype)
self.assertIsNone(partition_info)
return embedding_values
return _initializer
expected_input_layer = [
# example 0, ids_a [2], ids_b [1]
[[5., 6., 3., 4.], [0., 0., 0., 0.]],
# example 1, ids_a [0, 1], ids_b [2, 0]
[[1., 2., 5., 6.], [3., 4., 1., 2.]],
]
expected_sequence_length = [1, 2]
categorical_column_a = tf.feature_column.sequence_categorical_column_with_identity(
key='aaa', num_buckets=vocabulary_size)
categorical_column_b = tf.feature_column.sequence_categorical_column_with_identity(
key='bbb', num_buckets=vocabulary_size)
# Test that columns are reordered alphabetically.
shared_embedding_columns = tf.feature_column.shared_embeddings(
[categorical_column_b, categorical_column_a],
dimension=embedding_dimension,
initializer=_get_initializer(embedding_dimension,
embedding_values))
sequence_input_layer = ksfc.SequenceFeatures(
shared_embedding_columns)
input_layer, sequence_length = sequence_input_layer({
'aaa':
sparse_input_a,
'bbb':
sparse_input_b
})
global_vars = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)
self.assertCountEqual(('aaa_bbb_shared_embedding:0', ),
tuple([v.name for v in global_vars]))
with _initialized_session() as sess:
self.assertAllEqual(embedding_values,
global_vars[0].eval(session=sess))
self.assertAllEqual(expected_input_layer,
input_layer.eval(session=sess))
self.assertAllEqual(expected_sequence_length,
sequence_length.eval(session=sess))
def test_TensorBoard_multi_input_output(self):
np.random.seed(1337)
tmpdir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
with tf.Graph().as_default(), self.cached_session():
filepath = os.path.join(tmpdir, 'logs')
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM, ),
num_classes=NUM_CLASSES)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
def data_generator(train):
if train:
max_batch_index = len(x_train) // BATCH_SIZE
else:
max_batch_index = len(x_test) // BATCH_SIZE
i = 0
while 1:
if train:
# simulate multi-input/output models
yield ([x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] *
2,
[y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] *
2)
else:
yield ([x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] *
2,
[y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] *
2)
i += 1
i %= max_batch_index
inp1 = input_layer.Input((INPUT_DIM, ))
inp2 = input_layer.Input((INPUT_DIM, ))
inp = layers.add([inp1, inp2])
hidden = layers.Dense(2, activation='relu')(inp)
hidden = layers.Dropout(0.1)(hidden)
output1 = layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
output2 = layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
model = training.Model([inp1, inp2], [output1, output2])
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# we must generate new callbacks for each test, as they aren't stateless
def callbacks_factory(histogram_freq):
return [
callbacks_v1.TensorBoard(log_dir=filepath,
histogram_freq=histogram_freq,
write_images=True,
write_grads=True,
batch_size=5)
]
# fit without validation data
model.fit([x_train] * 2, [y_train] * 2,
batch_size=BATCH_SIZE,
callbacks=callbacks_factory(histogram_freq=0),
epochs=3)
# fit with validation data and accuracy
model.fit([x_train] * 2, [y_train] * 2,
batch_size=BATCH_SIZE,
validation_data=([x_test] * 2, [y_test] * 2),
callbacks=callbacks_factory(histogram_freq=1),
epochs=2)
# fit generator without validation data
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
callbacks=callbacks_factory(histogram_freq=0))
# fit generator with validation data and accuracy
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
validation_data=([x_test] * 2, [y_test] * 2),
callbacks=callbacks_factory(histogram_freq=1))
assert os.path.isdir(filepath)
def test_with_graph(self):
with tf.Graph().as_default():
with tf.Graph().as_default() as g:
ds = _create_dataset(range(10))
np_ds = dataset_utils.as_numpy(ds, graph=g)
self.assertEqual(list(range(10)), [int(el) for el in list(np_ds)])
def test_Tensorboard_histogram_summaries_in_test_function(self):
class FileWriterStub(object):
def __init__(self, logdir, graph=None):
self.logdir = logdir
self.graph = graph
self.steps_seen = []
def add_summary(self, summary, global_step):
summary_obj = tf.compat.v1.Summary()
# ensure a valid Summary proto is being sent
if isinstance(summary, bytes):
summary_obj.ParseFromString(summary)
else:
assert isinstance(summary, tf.compat.v1.Summary)
summary_obj = summary
# keep track of steps seen for the merged_summary op,
# which contains the histogram summaries
if len(summary_obj.value) > 1:
self.steps_seen.append(global_step)
def flush(self):
pass
def close(self):
pass
def _init_writer(obj, _):
obj.writer = FileWriterStub(obj.log_dir)
np.random.seed(1337)
tmpdir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM, ),
num_classes=NUM_CLASSES)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
with tf.Graph().as_default(), self.cached_session():
model = sequential.Sequential()
model.add(
layers.Dense(NUM_HIDDEN,
input_dim=INPUT_DIM,
activation='relu'))
# non_trainable_weights: moving_variance, moving_mean
model.add(layers.BatchNormalization())
model.add(layers.Dense(NUM_CLASSES, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
callbacks_v1.TensorBoard._init_writer = _init_writer
tsb = callbacks_v1.TensorBoard(log_dir=tmpdir,
histogram_freq=1,
write_images=True,
write_grads=True,
batch_size=5)
cbks = [tsb]
# fit with validation data
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=3,
verbose=0)
self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
def test_with_1d_sparse_tensor(self):
embedding_values = (
(1.0, 2.0, 3.0, 4.0, 5.0), # id 0
(6.0, 7.0, 8.0, 9.0, 10.0), # id 1
(11.0, 12.0, 13.0, 14.0, 15.0), # id 2
)
def _initializer(shape, dtype, partition_info=None):
del shape, dtype, partition_info
return embedding_values
# price has 1 dimension in dense_features
price = tf.feature_column.numeric_column("price")
# one_hot_body_style has 3 dims in dense_features.
body_style = tf.feature_column.categorical_column_with_vocabulary_list(
"body-style", vocabulary_list=["hardtop", "wagon", "sedan"]
)
one_hot_body_style = tf.feature_column.indicator_column(body_style)
# embedded_body_style has 5 dims in dense_features.
country = tf.feature_column.categorical_column_with_vocabulary_list(
"country", vocabulary_list=["US", "JP", "CA"]
)
embedded_country = tf.feature_column.embedding_column(
country, dimension=5, initializer=_initializer
)
with tf.Graph().as_default():
# Provides 1-dim tensor and dense tensor.
features = {
"price": tf.constant(
[
11.0,
12.0,
]
),
"body-style": tf.SparseTensor(
indices=((0,), (1,)),
values=("sedan", "hardtop"),
dense_shape=(2,),
),
# This is dense tensor for the categorical_column.
"country": tf.constant(["CA", "US"]),
}
self.assertEqual(1, features["price"].shape.ndims)
self.assertEqual(
1, features["body-style"].dense_shape.get_shape()[0]
)
self.assertEqual(1, features["country"].shape.ndims)
net = df.DenseFeatures(
[price, one_hot_body_style, embedded_country]
)(features)
self.assertEqual(1 + 3 + 5, net.shape[1])
with _initialized_session() as sess:
# Each row is formed by concatenating `embedded_body_style`,
# `one_hot_body_style`, and `price` in order.
self.assertAllEqual(
[
[0.0, 0.0, 1.0, 11.0, 12.0, 13.0, 14.0, 15.0, 11.0],
[1.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 12.0],
],
sess.run(net),
)
def test_TensorBoard(self):
np.random.seed(1337)
temp_dir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM, ),
num_classes=NUM_CLASSES)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
def data_generator(train):
if train:
max_batch_index = len(x_train) // BATCH_SIZE
else:
max_batch_index = len(x_test) // BATCH_SIZE
i = 0
while 1:
if train:
yield (x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
else:
yield (x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
i += 1
i %= max_batch_index
# case: Sequential
with tf.Graph().as_default(), self.cached_session():
model = sequential.Sequential()
model.add(
layers.Dense(NUM_HIDDEN,
input_dim=INPUT_DIM,
activation='relu'))
# non_trainable_weights: moving_variance, moving_mean
model.add(layers.BatchNormalization())
model.add(layers.Dense(NUM_CLASSES, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
tsb = callbacks_v1.TensorBoard(log_dir=temp_dir,
histogram_freq=1,
write_images=True,
write_grads=True,
batch_size=5)
cbks = [tsb]
# fit with validation data
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=3,
verbose=0)
# fit with validation data and accuracy
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=2,
verbose=0)
# fit generator with validation data
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
validation_data=(x_test, y_test),
callbacks=cbks,
verbose=0)
# fit generator without validation data
# histogram_freq must be zero
tsb.histogram_freq = 0
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
callbacks=cbks,
verbose=0)
# fit generator with validation data and accuracy
tsb.histogram_freq = 1
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
validation_data=(x_test, y_test),
callbacks=cbks,
verbose=0)
# fit generator without validation data and accuracy
tsb.histogram_freq = 0
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
callbacks=cbks)
assert os.path.exists(temp_dir)
def testSparse(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
sparse_epsilon = 1e-7
for dtype in [tf.half, tf.float32, tf.float64]:
with tf.Graph().as_default(), self.cached_session():
# Initialize variables for numpy implementation.
m0, v0, m1, v1, mcache = 0.0, 0.0, 0.0, 0.0, 1.0
var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0, 0.01],
dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np)
var1 = tf.Variable(var1_np)
grads0_np_indices = np.array([0, 2], dtype=np.int32)
grads0 = tf.IndexedSlices(
tf.constant(grads0_np[grads0_np_indices]),
tf.constant(grads0_np_indices), tf.constant([3]))
grads1_np_indices = np.array([0, 2], dtype=np.int32)
grads1 = tf.IndexedSlices(
tf.constant(grads1_np[grads1_np_indices]),
tf.constant(grads1_np_indices), tf.constant([3]))
opt = nadam.Nadam(epsilon=sparse_epsilon)
update = opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
# Fetch params to validate initial values
self.assertAllClose([1.0, 1.0, 2.0], var0)
self.assertAllClose([3.0, 3.0, 4.0], var1)
beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
# Run 3 steps of Nadam
for t in range(3):
self.assertAllCloseAccordingToType(0.9**(t + 1),
beta1_power)
self.assertAllCloseAccordingToType(0.999**(t + 1),
beta2_power)
update.run()
mcache = update_m_cache(mcache, t)
var0_np, m0, v0 = nadam_update_numpy(
var0_np,
grads0_np,
t,
m0,
v0,
mcache,
epsilon=sparse_epsilon)
var1_np, m1, v1 = nadam_update_numpy(
var1_np,
grads1_np,
t,
m1,
v1,
mcache,
epsilon=sparse_epsilon)
# Validate updated params
self.assertAllCloseAccordingToType(var0_np, var0)
self.assertAllCloseAccordingToType(var1_np, var1)
def test_TensorBoard_update_freq(self):
class FileWriterStub(object):
def __init__(self, logdir, graph=None):
self.logdir = logdir
self.graph = graph
self.batch_summaries = []
self.epoch_summaries = []
def add_summary(self, summary, step):
if 'batch_' in summary.value[0].tag:
self.batch_summaries.append((step, summary))
elif 'epoch_' in summary.value[0].tag:
self.epoch_summaries.append((step, summary))
def flush(self):
pass
def close(self):
pass
with tf.Graph().as_default():
temp_dir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
# Epoch mode
tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='epoch')
tb_cbk.writer = FileWriterStub(temp_dir)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
self.assertEqual(tb_cbk.writer.batch_summaries, [])
tb_cbk.on_epoch_end(0, {'acc': 10.0, 'size': 1})
self.assertLen(tb_cbk.writer.epoch_summaries, 1)
tb_cbk.on_train_end()
# Batch mode
tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='batch')
tb_cbk.writer = FileWriterStub(temp_dir)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
self.assertLen(tb_cbk.writer.batch_summaries, 1)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1})
self.assertLen(tb_cbk.writer.batch_summaries, 2)
self.assertFalse(tb_cbk.writer.epoch_summaries)
tb_cbk.on_train_end()
# Integer mode
tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq=20)
tb_cbk.writer = FileWriterStub(temp_dir)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
self.assertFalse(tb_cbk.writer.batch_summaries)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
self.assertLen(tb_cbk.writer.batch_summaries, 1)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
self.assertLen(tb_cbk.writer.batch_summaries, 1)
tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10})
self.assertLen(tb_cbk.writer.batch_summaries, 2)
tb_cbk.on_batch_end(0, {'acc': 10.0, 'size': 10})
self.assertLen(tb_cbk.writer.batch_summaries, 2)
self.assertFalse(tb_cbk.writer.epoch_summaries)
tb_cbk.on_train_end()
def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(
self, ):
categorical_column_a = (
tf.feature_column.categorical_column_with_identity(key="aaa",
num_buckets=3))
categorical_column_b = (
tf.feature_column.categorical_column_with_identity(key="bbb",
num_buckets=3))
embedding_dimension = 2
(
embedding_column_b,
embedding_column_a,
) = tf.feature_column.shared_embeddings(
[categorical_column_b, categorical_column_a],
dimension=embedding_dimension,
)
all_cols = [embedding_column_a, embedding_column_b]
with tf.Graph().as_default():
features = {
"aaa":
tf.SparseTensor(
indices=((0, 0), (1, 0), (1, 1)),
values=(0, 1, 0),
dense_shape=(2, 2),
),
"bbb":
tf.SparseTensor(
indices=((0, 0), (1, 0), (1, 1)),
values=(1, 2, 1),
dense_shape=(2, 2),
),
}
df.DenseFeatures(all_cols)(features)
# Make sure that only 1 variable gets created in this case.
self.assertEqual(
1,
len(
tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)),
)
with tf.Graph().as_default():
features1 = {
"aaa":
tf.SparseTensor(
indices=((0, 0), (1, 0), (1, 1)),
values=(0, 1, 0),
dense_shape=(2, 2),
),
"bbb":
tf.SparseTensor(
indices=((0, 0), (1, 0), (1, 1)),
values=(1, 2, 1),
dense_shape=(2, 2),
),
}
df.DenseFeatures(all_cols)(features1)
# Make sure that only 1 variable gets created in this case.
self.assertEqual(
1,
len(
tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)),
)
self.assertCountEqual(
["aaa_bbb_shared_embedding:0"],
[
v.name for v in tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)
],
)
def testSharing(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
with tf.Graph().as_default():
for dtype in [tf.half, tf.float32, tf.float64]:
var0 = tf.Variable([1.0, 2.0], dtype=dtype)
var1 = tf.Variable([3.0, 4.0], dtype=dtype)
grads0 = tf.constant([0.1, 0.1], dtype=dtype)
grads1 = tf.constant([0.01, 0.01], dtype=dtype)
mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9)
mom_update1 = mom_opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
mom_update2 = mom_opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
slot0 = mom_opt.get_slot(var0, "momentum")
self.assertEqual(slot0.shape, var0.shape)
slot1 = mom_opt.get_slot(var1, "momentum")
self.assertEqual(slot1.shape, var1.shape)
# Fetch params to validate initial values
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Step 1: the momentum accumulators where 0. So we should see a normal
# update: v -= grad * learning_rate
self.evaluate(mom_update1)
# Check that the momentum accumulators have been updated.
self.assertAllCloseAccordingToType(np.array([-0.2, -0.2]),
self.evaluate(slot0))
self.assertAllCloseAccordingToType(np.array([-0.02, -0.02]),
self.evaluate(slot1))
# Check that the parameters have been updated.
self.assertAllCloseAccordingToType(
np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
self.evaluate(var0),
)
self.assertAllCloseAccordingToType(
np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
self.evaluate(var1),
)
# Step 2: the second momentum accumulators contain the previous update.
self.evaluate(mom_update2)
# Check that the momentum accumulators have been updated.
self.assertAllCloseAccordingToType(
np.array([(0.9 * (-0.2) - 2.0 * 0.1),
(0.9 * (-0.2) - 2.0 * 0.1)]),
self.evaluate(slot0),
)
self.assertAllCloseAccordingToType(
np.array([
(0.9 * (-0.02) - 2.0 * 0.01),
(0.9 * (-0.02) - 2.0 * 0.01),
]),
self.evaluate(slot1),
)
# Check that the parameters have been updated.
self.assertAllCloseAccordingToType(
np.array([
1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
]),
self.evaluate(var0),
)
self.assertAllCloseAccordingToType(
np.array([
2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
3.98 - ((0.9 * 0.01 + 0.01) * 2.0),
]),
self.evaluate(var1),
)
