def prep_input(inp):
inner_input_shape = self._get_shape_tuple((-1,), inp, 2)
# Shape: (num_samples * timesteps, ...). And track the
# transformation in self._input_map.
input_uid = object_list_uid(inp)
reshaped = K.reshape(inp, inner_input_shape)
self._input_map[input_uid] = reshaped
return reshaped
def test_TimeDistributed():
# first, test with Dense layer
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 2)),
epochs=1,
batch_size=10)
# test config
model.get_config()
# test when specifying a batch_input_shape
test_input = np.random.random((1, 3, 4))
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(
wrappers.TimeDistributed(layers.Dense(2), batch_input_shape=(1, 3, 4)))
reference.add(layers.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Embedding
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.Embedding(5, 6),
batch_input_shape=(10, 3, 4),
dtype='int32'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.randint(5, size=(10, 3, 4), dtype='int32'),
np.random.random((10, 3, 4, 6)),
epochs=1,
batch_size=10)
# compare to not using batch_input_shape
test_input = np.random.randint(5, size=(10, 3, 4), dtype='int32')
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(
wrappers.TimeDistributed(layers.Embedding(5, 6),
input_shape=(3, 4),
dtype='int32'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Conv2D
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.Conv2D(5, (2, 2), padding='same'),
input_shape=(2, 4, 4, 3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.random.random((1, 2, 4, 4, 3)),
np.random.random((1, 2, 4, 4, 5)))
model = model_from_json(model.to_json())
model.summary()
# test stacked layers
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(wrappers.TimeDistributed(layers.Dense(3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test wrapping Sequential model
model = Sequential()
model.add(layers.Dense(3, input_dim=2))
outer_model = Sequential()
outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test with functional API
x = Input(shape=(3, 2))
y = wrappers.TimeDistributed(model)(x)
outer_model = Model(x, y)
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test with BatchNormalization
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.BatchNormalization(center=True,
scale=True),
name='bn',
input_shape=(10, 2)))
model.compile(optimizer='rmsprop', loss='mse')
# Assert that mean and variance are 0 and 1.
td = model.layers[0]
assert np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Train
model.train_on_batch(np.random.normal(loc=2, scale=2, size=(1, 10, 2)),
np.broadcast_to(np.array([0, 1]), (1, 10, 2)))
# Assert that mean and variance changed.
assert not np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert not np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Verify input_map has one mapping from inputs to reshaped inputs.
uid = object_list_uid(model.inputs)
assert len(td._input_map.keys()) == 1
assert uid in td._input_map
assert K.int_shape(td._input_map[uid]) == (None, 2)
def run_internal_graph(model, inputs, mode, mask=None):
"""Computes output tensors for new inputs.
# Note:
- Expects `inputs` to be a list (potentially with 1 element).
- Can be run on non-Keras tensors.
# Arguments
inputs: List of tensors
masks: List of masks (tensors or None).
# Returns
Three lists: output_tensors, output_masks, output_shapes
"""
is_training = mode == tf.estimator.ModeKeys.TRAIN
inputs = to_list(inputs)
if mask is None:
masks = [None for _ in range(len(inputs))]
else:
masks = to_list(mask)
cache_key = object_list_uid(inputs)
cache_key += "_" + object_list_uid(masks)
if cache_key in model._output_tensor_cache:
return model._output_tensor_cache[cache_key]
# Dictionary mapping reference tensors to tuples
# (computed tensor, compute mask)
# we assume a 1:1 mapping from tensor to mask
# TODO: raise exception when a `.compute_mask()` call
# does not return a list the same size as `call`
tensor_map = {}
for x, y, mask in zip(model.inputs, inputs, masks):
tensor_map[str(id(x))] = (y, mask)
depth_keys = list(model._nodes_by_depth.keys())
depth_keys.sort(reverse=True)
for depth in depth_keys:
nodes = model._nodes_by_depth[depth]
for node in nodes:
# This is always a single layer, never a list.
layer = node.outbound_layer
reference_input_tensors = node.input_tensors
reference_output_tensors = node.output_tensors
# issue: https://github.com/tensorflow/tensorflow/issues/30208
if not isinstance(reference_input_tensors, list):
reference_input_tensors = [reference_input_tensors]
if not isinstance(reference_output_tensors, list):
reference_output_tensors = [reference_output_tensors]
# If all previous input tensors are available in tensor_map,
# then call node.inbound_layer on them.
computed_data = [] # List of tuples (input, mask).
for x in reference_input_tensors:
if str(id(x)) in tensor_map:
computed_data.append(tensor_map[str(id(x))])
if len(computed_data) == len(reference_input_tensors):
# call layer
with K.name_scope(layer.name):
if node.arguments:
kwargs = node.arguments
else:
kwargs = {}
if len(computed_data) == 1:
computed_tensor, computed_mask = computed_data[0]
if has_arg(layer.call, "mask"):
if "mask" not in kwargs:
kwargs["mask"] = computed_mask
if has_arg(layer.call, "training"):
kwargs["training"] = is_training
output_tensors = to_list(
layer.call(computed_tensor, **kwargs))
output_masks = layer.compute_mask(
computed_tensor, computed_mask)
if output_masks is None:
output_masks = [None for _ in output_tensors]
else:
output_masks = to_list(output_masks)
computed_tensors = [computed_tensor]
# computed_masks might be used in the future.
computed_masks = [computed_mask]
else:
computed_tensors = [x[0] for x in computed_data]
computed_masks = [x[1] for x in computed_data]
if has_arg(layer.call, "mask"):
if "mask" not in kwargs:
kwargs["mask"] = computed_masks
output_tensors = to_list(
layer.call(computed_tensors, **kwargs))
output_masks = layer.compute_mask(
computed_tensors, computed_masks)
if output_masks is None:
output_masks = [None for _ in output_tensors]
else:
output_masks = to_list(output_masks)
# Apply activity regularizer if any:
if (hasattr(layer, "activity_regularizer")
and layer.activity_regularizer is not None):
with K.name_scope("activity_regularizer"):
regularization_losses = [
layer.activity_regularizer(x)
for x in output_tensors
]
layer.add_loss(regularization_losses,
inputs=computed_tensors)
if len(output_masks) != len(output_tensors):
raise Exception(
"Layers should have equal number of output tensors "
"and output masks. Layer " + str(layer.name) +
" has"
" " + str(len(output_tensors)) + " output tensors "
"and " + str(len(output_masks)) + " output masks.")
# Update model updates and losses:
# Keep track of updates that depend on the inputs
# (e.g. BN updates).
model.add_update(layer.get_updates_for(computed_tensors),
inputs)
# Keep track of unconditional updates (e.g. a counter).
model.add_update(layer.get_updates_for(None), None)
# Keep track of losses that depend on the inputs
# (e.g. activity regularizers).
model.add_loss(layer.get_losses_for(computed_tensors), inputs)
# Keep track of unconditional losses
# (e.g. weight regularizers).
model.add_loss(layer.get_losses_for(None), None)
# Update _keras_shape.
if all([hasattr(x, "_keras_shape") for x in computed_tensors]):
input_shapes = unpack_singleton(
[x._keras_shape for x in computed_tensors])
shapes = to_list(layer.compute_output_shape(input_shapes))
uses_learning_phase = any(
[x._uses_learning_phase for x in computed_tensors])
for x, s in zip(output_tensors, shapes):
x._keras_shape = s
_u = getattr(x, "_uses_learning_phase", False)
x._uses_learning_phase = _u or uses_learning_phase
# Update tensor_map.
for x, y, mask in zip(reference_output_tensors, output_tensors,
output_masks):
tensor_map[str(id(x))] = (y, mask)
output_tensors = []
output_masks = []
output_shapes = []
for x in model.outputs:
assert str(id(x)) in tensor_map, "Could not compute output " + str(x)
tensor, mask = tensor_map[str(id(x))]
if hasattr(tensor, "_keras_shape") and output_shapes is not None:
shape = tensor._keras_shape
output_shapes.append(shape)
else:
output_shapes = None
output_tensors.append(tensor)
output_masks.append(mask)
# Update cache;
# keys are based on ids on input tensors and inputs masks.
cache_key = object_list_uid(inputs)
cache_key += "_" + object_list_uid(masks)
output_tensors = unpack_singleton(output_tensors)
model._output_tensor_cache[cache_key] = output_tensors
output_masks = unpack_singleton(output_masks)
model._output_mask_cache[cache_key] = output_masks
if output_shapes is not None:
input_shapes = [x._keras_shape for x in inputs]
cache_key = ", ".join([str(x) for x in input_shapes])
output_shapes = unpack_singleton(output_shapes)
model._output_shape_cache[cache_key] = output_shapes
return output_tensors
def test_TimeDistributed():
# first, test with Dense layer
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 2)),
epochs=1,
batch_size=10)
# test config
model.get_config()
# test when specifying a batch_input_shape
test_input = np.random.random((1, 3, 4))
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(wrappers.TimeDistributed(layers.Dense(2),
batch_input_shape=(1, 3, 4)))
reference.add(layers.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Embedding
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Embedding(5, 6),
batch_input_shape=(10, 3, 4),
dtype='int32'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.randint(5, size=(10, 3, 4), dtype='int32'),
np.random.random((10, 3, 4, 6)), epochs=1, batch_size=10)
# compare to not using batch_input_shape
test_input = np.random.randint(5, size=(10, 3, 4), dtype='int32')
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(wrappers.TimeDistributed(layers.Embedding(5, 6),
input_shape=(3, 4), dtype='int32'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Conv2D
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Conv2D(5, (2, 2),
padding='same'),
input_shape=(2, 4, 4, 3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.random.random((1, 2, 4, 4, 3)),
np.random.random((1, 2, 4, 4, 5)))
model = model_from_json(model.to_json())
model.summary()
# test stacked layers
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(wrappers.TimeDistributed(layers.Dense(3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 3)),
epochs=1, batch_size=10)
# test wrapping Sequential model
model = Sequential()
model.add(layers.Dense(3, input_dim=2))
outer_model = Sequential()
outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)),
epochs=1, batch_size=10)
# test with functional API
x = Input(shape=(3, 2))
y = wrappers.TimeDistributed(model)(x)
outer_model = Model(x, y)
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)),
epochs=1, batch_size=10)
# test with BatchNormalization
model = Sequential()
model.add(wrappers.TimeDistributed(
layers.BatchNormalization(center=True, scale=True),
name='bn', input_shape=(10, 2)))
model.compile(optimizer='rmsprop', loss='mse')
# Assert that mean and variance are 0 and 1.
td = model.layers[0]
assert np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Train
model.train_on_batch(np.random.normal(loc=2, scale=2, size=(1, 10, 2)),
np.broadcast_to(np.array([0, 1]), (1, 10, 2)))
# Assert that mean and variance changed.
assert not np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert not np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Verify input_map has one mapping from inputs to reshaped inputs.
uid = object_list_uid(model.inputs)
assert len(td._input_map.keys()) == 1
assert uid in td._input_map
assert K.int_shape(td._input_map[uid]) == (None, 2)