def testOverrideAndContToSameTensor(self):
stepper = NodeStepper(self.sess, self.e)
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
self.assertEqual(["c:0"], stepper.handle_names())
self.assertSetEqual({"c"}, stepper.handle_node_names())
self.assertAllClose(6.0, stepper.cont(self.c))
# The last cont() call should use the tensor handle directly.
self.assertEqual({"c:0": NodeStepper.FEED_TYPE_HANDLE},
stepper.last_feed_types())
# Override c:0.
stepper.override_tensor("c:0", 7.0)
# As a result of the override, the tensor handle should have been
# invalidated.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
result = stepper.cont(self.c)
self.assertAllClose(7.0, result)
self.assertEqual({"c:0": NodeStepper.FEED_TYPE_OVERRIDE},
stepper.last_feed_types())
def testUsingNodesNotUsingIntermediateTensors(self):
stepper = NodeStepper(self.sess, self.e)
# There should be no handles before any cont() calls.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
# Before the cont() call, the stepper should not have access to the value
# of c:0.
with self.assertRaisesRegexp(
ValueError,
"This stepper instance does not have access to the value of tensor "
"\"c:0\""):
stepper.get_tensor_value("c:0")
# Using the node/tensor itself, instead of the name str, should work on
# cont().
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
self.assertEqual(["c:0"], stepper.handle_names())
self.assertEqual({"c"}, stepper.handle_node_names())
# After the cont() call, the stepper should have access to the value of c:0
# via a tensor handle.
self.assertAllClose(6.0, stepper.get_tensor_value("c:0"))
result = stepper.cont(self.e)
self.assertAllClose(24.0, result)
self.assertEqual({"c:0": NodeStepper.FEED_TYPE_HANDLE},
stepper.last_feed_types())
def testOverrideValueTwice(self):
stepper = NodeStepper(self.sess, self.e)
# Override once.
stepper.override_tensor("c:0", 7.0)
self.assertAllClose(28.0, stepper.cont(self.e))
self.assertEqual({"c:0": NodeStepper.FEED_TYPE_OVERRIDE},
stepper.last_feed_types())
self.assertEqual(["e:0"], stepper.handle_names())
self.assertSetEqual({"e"}, stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
# Calling cont(self.e) again. This time the cached tensor handle of e
# should be used.
self.assertEqual(28.0, stepper.cont(self.e))
self.assertEqual({"e:0": NodeStepper.FEED_TYPE_HANDLE},
stepper.last_feed_types())
# Override c again. This should have invalidated the cache for e.
stepper.override_tensor("c:0", 8.0)
self.assertEqual([], stepper.handle_names())
self.assertEqual(set(), stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
self.assertAllClose(32.0, stepper.cont(self.e))
self.assertEqual({"c:0": NodeStepper.FEED_TYPE_OVERRIDE},
stepper.last_feed_types())
def testOverrideAndContToSameTensor(self):
stepper = NodeStepper(self.sess, self.e)
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
self.assertEqual(["c:0"], stepper.handle_names())
self.assertSetEqual({"c"}, stepper.handle_node_names())
self.assertAllClose(6.0, stepper.cont(self.c))
# The last cont() call should use the tensor handle directly.
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
# Override c:0.
stepper.override_tensor("c:0", 7.0)
# As a result of the override, the tensor handle should have been
# invalidated.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
result = stepper.cont(self.c)
self.assertAllClose(7.0, result)
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
def testOverrideValueTwice(self):
stepper = NodeStepper(self.sess, self.e)
# Override once.
stepper.override_tensor("c:0", 7.0)
self.assertAllClose(28.0, stepper.cont(self.e))
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
self.assertEqual(["e:0"], stepper.handle_names())
self.assertSetEqual({"e"}, stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
# Calling cont(self.e) again. This time the cached tensor handle of e
# should be used.
self.assertEqual(28.0, stepper.cont(self.e))
self.assertEqual({
"e:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
# Override c again. This should have invalidated the cache for e.
stepper.override_tensor("c:0", 8.0)
self.assertEqual([], stepper.handle_names())
self.assertEqual(set(), stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
self.assertAllClose(32.0, stepper.cont(self.e))
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
def testUsingNodesNotUsingIntermediateTensors(self):
stepper = NodeStepper(self.sess, self.e)
# There should be no handles before any cont() calls.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
# Before the cont() call, the stepper should not have access to the value
# of c:0.
with self.assertRaisesRegexp(
ValueError,
"This stepper instance does not have access to the value of tensor "
"\"c:0\""):
stepper.get_tensor_value("c:0")
# Using the node/tensor itself, instead of the name str, should work on
# cont().
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
self.assertEqual(["c:0"], stepper.handle_names())
self.assertEqual({"c"}, stepper.handle_node_names())
# After the cont() call, the stepper should have access to the value of c:0
# via a tensor handle.
self.assertAllClose(6.0, stepper.get_tensor_value("c:0"))
result = stepper.cont(self.e)
self.assertAllClose(24.0, result)
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
def testRemoveOverrideValue(self):
stepper = NodeStepper(self.sess, self.e)
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
# The previous cont() step should have generated a cached tensor handle.
self.assertEqual(["c:0"], stepper.handle_names())
self.assertSetEqual({"c"}, stepper.handle_node_names())
# Override c:0.
stepper.override_tensor("c:0", 7.0)
# The overriding should have invalidated the tensor handle.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
result = stepper.cont(self.e)
self.assertAllClose(28.0, result) # Should reflect the overriding value.
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
# The handle to tensor e:0 should have been cached, even though its
# transitive closure contains an override.
self.assertIn("e:0", stepper.handle_names())
self.assertSetEqual({"e"}, stepper.handle_node_names())
# Remove the override.
stepper.remove_override("c:0")
# c:0 should not be in the overrides anymore.
self.assertEqual([], stepper.override_names())
# Removing the override should have invalidated the tensor handle for c.
self.assertNotIn("e:0", stepper.handle_names())
self.assertNotIn("e", stepper.handle_node_names())
# Should reflect the non-overriding value.
self.assertAllClose(24.0, stepper.cont(self.e))
# This time, the handle to tensor e:0 should have been cached again, even
# thought its transitive closure contains an override.
self.assertIn("e:0", stepper.handle_names())
self.assertIn("e", stepper.handle_node_names())
# Calling cont(self.e) again should have used the tensor handle to e:0.
self.assertAllClose(24.0, stepper.cont(self.e))
self.assertEqual({
"e:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
def testRemoveOverrideValue(self):
stepper = NodeStepper(self.sess, self.e)
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
# The previous cont() step should have generated a cached tensor handle.
self.assertEqual(["c:0"], stepper.handle_names())
self.assertSetEqual({"c"}, stepper.handle_node_names())
# Override c:0.
stepper.override_tensor("c:0", 7.0)
# The overriding should have invalidated the tensor handle.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
result = stepper.cont(self.e)
self.assertAllClose(28.0, result) # Should reflect the overriding value.
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
# The handle to tensor e:0 should have been cached, even though its
# transitive closure contains an override.
self.assertIn("e:0", stepper.handle_names())
self.assertSetEqual({"e"}, stepper.handle_node_names())
# Remove the override.
stepper.remove_override("c:0")
# c:0 should not be in the overrides anymore.
self.assertEqual([], stepper.override_names())
# Removing the override should have invalidated the tensor handle for c.
self.assertNotIn("e:0", stepper.handle_names())
self.assertNotIn("e", stepper.handle_node_names())
# Should reflect the non-overriding value.
self.assertAllClose(24.0, stepper.cont(self.e))
# This time, the handle to tensor e:0 should have been cached again, even
# thought its transitive closure contains an override.
self.assertIn("e:0", stepper.handle_names())
self.assertIn("e", stepper.handle_node_names())
# Calling cont(self.e) again should have used the tensor handle to e:0.
self.assertAllClose(24.0, stepper.cont(self.e))
self.assertEqual({
"e:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
def testContWithPlaceholders(self):
stepper = NodeStepper(self.sess,
self.y,
feed_dict={
self.ph0: [[1.0, 2.0], [-3.0, 5.0]],
self.ph1: [[-1.0], [0.5]]
})
self.assertEqual(4, len(stepper.sorted_nodes()))
self.assertSetEqual({"ph0:0", "ph1:0", "x:0", "y:0"},
set(stepper.closure_elements()))
result = stepper.cont(self.x)
self.assertAllClose([[0.0], [5.5]], result)
self.assertEqual(
{
"ph0:0": NodeStepper.FEED_TYPE_CLIENT,
"ph1:0": NodeStepper.FEED_TYPE_CLIENT,
}, stepper.last_feed_types())
self.assertEqual(["x:0"], stepper.handle_names())
self.assertSetEqual({"x"}, stepper.handle_node_names())
result = stepper.cont(self.y)
self.assertAllClose([[-1.0], [6.0]], result)
self.assertEqual(
{
"x:0": NodeStepper.FEED_TYPE_HANDLE,
"ph1:0": NodeStepper.FEED_TYPE_CLIENT,
}, stepper.last_feed_types())
def testContWithPlaceholders(self):
stepper = NodeStepper(
self.sess,
self.y,
feed_dict={
self.ph0: [[1.0, 2.0], [-3.0, 5.0]],
self.ph1: [[-1.0], [0.5]]
})
self.assertEqual(4, len(stepper.sorted_nodes()))
self.assertSetEqual({"ph0:0", "ph1:0", "x:0", "y:0"},
set(stepper.closure_elements()))
result = stepper.cont(self.x)
self.assertAllClose([[0.0], [5.5]], result)
self.assertEqual({
"ph0:0": NodeStepper.FEED_TYPE_CLIENT,
"ph1:0": NodeStepper.FEED_TYPE_CLIENT,
}, stepper.last_feed_types())
self.assertEqual(["x:0"], stepper.handle_names())
self.assertSetEqual({"x"}, stepper.handle_node_names())
result = stepper.cont(self.y)
self.assertAllClose([[-1.0], [6.0]], result)
self.assertEqual({
"x:0": NodeStepper.FEED_TYPE_HANDLE,
"ph1:0": NodeStepper.FEED_TYPE_CLIENT,
}, stepper.last_feed_types())
def testContToNodeWithOutputTensors(self):
"""cont() to an op should cache its output tensors if appropriate."""
stepper = NodeStepper(self.sess, "optim")
# In the transitive closure of the stepper, look for an op of which the
# output tensor also is in the transitive closure.
# Do not assume a specific op, e.g., ""gradients/e_grad/Reshape_1",
# because it may vary between builds.
closure_elements = stepper.closure_elements()
op_with_output_in_closure = None
for element_name in closure_elements:
if element_name + ":0" in closure_elements:
op_with_output_in_closure = str(element_name)
break
self.assertEqual([0],
stepper.output_slots_in_closure(op_with_output_in_closure))
self.assertIsNotNone(op_with_output_in_closure)
output_tensor = op_with_output_in_closure + ":0"
# The op "gradients/?_grad/Reshape_1" is in the transitive closure of the
# stepper, because it is the control input to another o. However, its
# output tensor "gradients/?_grad/Reshape_1:0" is also in the transitive
# closure, because it is the (non-control) input of certain ops. Calling
# cont() on the op should lead to the caching of the tensor handle for
# the output tensor.
stepper.cont(op_with_output_in_closure)
self.assertEqual([output_tensor], stepper.handle_names())
self.assertSetEqual({op_with_output_in_closure},
stepper.handle_node_names())
# Do a cont() call that uses the cached tensor of
# "gradients/?_grad/Reshape_1:0".
stepper.cont(output_tensor)
self.assertEqual({
output_tensor: NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
def testContToNodeWithOutputTensors(self):
"""cont() to an op should cache its output tensors if appropriate."""
stepper = NodeStepper(self.sess, "optim")
# In the transitive closure of the stepper, look for an op of which the
# output tensor also is in the transitive closure.
# Do not assume a specific op, e.g., ""gradients/e_grad/Reshape_1",
# because it may vary between builds.
closure_elements = stepper.closure_elements()
op_with_output_in_closure = None
for element_name in closure_elements:
if element_name + ":0" in closure_elements:
op_with_output_in_closure = str(element_name)
break
self.assertEqual([0],
stepper.output_slots_in_closure(op_with_output_in_closure))
self.assertIsNotNone(op_with_output_in_closure)
output_tensor = op_with_output_in_closure + ":0"
# The op "gradients/?_grad/Reshape_1" is in the transitive closure of the
# stepper, because it is the control input to another o. However, its
# output tensor "gradients/?_grad/Reshape_1:0" is also in the transitive
# closure, because it is the (non-control) input of certain ops. Calling
# cont() on the op should lead to the caching of the tensor handle for
# the output tensor.
stepper.cont(op_with_output_in_closure)
self.assertEqual([output_tensor], stepper.handle_names())
self.assertSetEqual({op_with_output_in_closure},
stepper.handle_node_names())
# Do a cont() call that uses the cached tensor of
# "gradients/?_grad/Reshape_1:0".
stepper.cont(output_tensor)
self.assertEqual({
output_tensor: NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
def testOverrideValue(self):
stepper = NodeStepper(self.sess, self.e)
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
# There should be no overrides before any cont() calls.
self.assertEqual([], stepper.override_names())
# Calling cont() on c again should lead to use of the handle.
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
# Override c:0.
stepper.override_tensor("c:0", 7.0)
# After the overriding, calling get_tensor_value() on c:0 should yield the
# overriding value.
self.assertEqual(7.0, stepper.get_tensor_value("c:0"))
# Now c:0 should have only an override value, but no cached handle, because
# the handle should have been invalidated.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
# Run a downstream tensor after the value override.
result = stepper.cont(self.e)
self.assertAllClose(28.0, result) # Should reflect the overriding value.
# Should use override, instead of the handle.
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
def testOverrideValue(self):
stepper = NodeStepper(self.sess, self.e)
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({}, stepper.last_feed_types())
# There should be no overrides before any cont() calls.
self.assertEqual([], stepper.override_names())
# Calling cont() on c again should lead to use of the handle.
result = stepper.cont(self.c)
self.assertAllClose(6.0, result)
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
# Override c:0.
stepper.override_tensor("c:0", 7.0)
# After the overriding, calling get_tensor_value() on c:0 should yield the
# overriding value.
self.assertEqual(7.0, stepper.get_tensor_value("c:0"))
# Now c:0 should have only an override value, but no cached handle, because
# the handle should have been invalidated.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
self.assertEqual(["c:0"], stepper.override_names())
# Run a downstream tensor after the value override.
result = stepper.cont(self.e)
self.assertAllClose(28.0, result) # Should reflect the overriding value.
# Should use override, instead of the handle.
self.assertEqual({
"c:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
def testOverrideThenContToUpdate(self):
"""Test cont() to update nodes after overriding tensor values."""
stepper = NodeStepper(self.sess, "optim")
result = stepper.cont("d:0")
self.assertAllClose(2.0, result)
self.assertEqual({}, stepper.last_feed_types())
self.assertEqual(set(), stepper.dirty_variables())
self.assertEqual(["d:0"], stepper.handle_names())
self.assertSetEqual({"d"}, stepper.handle_node_names())
# Override the value from 1.0 to 10.0.
stepper.override_tensor("a/read:0", 10.0)
self.assertEqual(["a/read:0"], stepper.override_names())
result = stepper.cont("optim/update_c/ApplyGradientDescent",
restore_variable_values=True)
self.assertIsNone(result)
# The last cont() call should have not used the tensor handle to d:0,
# because the transitive closure of d:0 contains an override tensor.
self.assertEqual({"a/read:0": NodeStepper.FEED_TYPE_OVERRIDE},
stepper.last_feed_types())
# The tensor handle to d:0 should have been removed due to the dirty
# transitive closure.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
# For this backprop on c, the overriding value of a/read:0 should have been
# used:
# 4.0 - learning_rate * a * b * b
# = 4.0 - 0.01 * 10.0 * 2.0 * 2.0 = 3.6.
self.assertAllClose(3.6, self.sess.run(self.c))
# Now remove the overriding value of a/read:0.
stepper.remove_override("a/read:0")
self.assertEqual([], stepper.override_names())
# Obtain the tensor handle to d:0 again.
result = stepper.cont("d:0")
self.assertAllClose(2.0, result)
self.assertEqual(["d:0"], stepper.handle_names())
self.assertSetEqual({"d"}, stepper.handle_node_names())
# Then call update_c again, without restoring c.
result = stepper.cont("optim/update_c/ApplyGradientDescent",
restore_variable_values=False)
self.assertIsNone(result)
# This time, the d:0 tensor handle should have been used, because its
# transitive closure is clean.
self.assertEqual({"d:0": NodeStepper.FEED_TYPE_HANDLE},
stepper.last_feed_types())
# For this backprop on c, the overriding value of a/read:0 should have been
# used:
# 3.6 - learning_rate * a * b * b
# = 3.6 - 0.01 * 1.0 * 2.0 * 2.0 = 3.56.
self.assertAllClose(3.56, self.sess.run(self.c))
def testOverrideThenContToUpdate(self):
"""Test cont() to update nodes after overriding tensor values."""
stepper = NodeStepper(self.sess, "optim")
result = stepper.cont("d:0")
self.assertAllClose(2.0, result)
self.assertEqual({}, stepper.last_feed_types())
self.assertEqual(set(), stepper.dirty_variables())
self.assertEqual(["d:0"], stepper.handle_names())
self.assertSetEqual({"d"}, stepper.handle_node_names())
# Override the value from 1.0 to 10.0.
stepper.override_tensor("a/read:0", 10.0)
self.assertEqual(["a/read:0"], stepper.override_names())
result = stepper.cont(
"optim/update_c/ApplyGradientDescent", restore_variable_values=True)
self.assertIsNone(result)
# The last cont() call should have not used the tensor handle to d:0,
# because the transitive closure of d:0 contains an override tensor.
self.assertEqual({
"a/read:0": NodeStepper.FEED_TYPE_OVERRIDE
}, stepper.last_feed_types())
# The tensor handle to d:0 should have been removed due to the dirty
# transitive closure.
self.assertEqual([], stepper.handle_names())
self.assertSetEqual(set(), stepper.handle_node_names())
# For this backprop on c, the overriding value of a/read:0 should have been
# used:
# 4.0 - learning_rate * a * b * b
# = 4.0 - 0.01 * 10.0 * 2.0 * 2.0 = 3.6.
self.assertAllClose(3.6, self.sess.run(self.c))
# Now remove the overriding value of a/read:0.
stepper.remove_override("a/read:0")
self.assertEqual([], stepper.override_names())
# Obtain the tensor handle to d:0 again.
result = stepper.cont("d:0")
self.assertAllClose(2.0, result)
self.assertEqual(["d:0"], stepper.handle_names())
self.assertSetEqual({"d"}, stepper.handle_node_names())
# Then call update_c again, without restoring c.
result = stepper.cont(
"optim/update_c/ApplyGradientDescent", restore_variable_values=False)
self.assertIsNone(result)
# This time, the d:0 tensor handle should have been used, because its
# transitive closure is clean.
self.assertEqual({
"d:0": NodeStepper.FEED_TYPE_HANDLE
}, stepper.last_feed_types())
# For this backprop on c, the overriding value of a/read:0 should have been
# used:
# 3.6 - learning_rate * a * b * b
# = 3.6 - 0.01 * 1.0 * 2.0 * 2.0 = 3.56.
self.assertAllClose(3.56, self.sess.run(self.c))
