def testAssignNoShape(self):
with self.cached_session():
value = self._NewShapelessTensor()
var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), var.get_shape())
self.assertEqual(tensor_shape.unknown_shape(),
state_ops.assign(var, value).get_shape())
def testAssignNoValueShape(self): value = self._NewShapelessTensor() shape = [1, 2] var = state_ops.variable_op(shape, dtypes.float32) assigned = state_ops.assign(var, value) self.assertEqual(shape, var.get_shape()) self.assertEqual(shape, assigned.get_shape())
def testIsVariableInitialized(self):
for use_gpu in [True, False]:
with self.test_session(use_gpu=use_gpu):
v0 = state_ops.variable_op([1, 2], dtypes.float32)
self.assertEqual(False, variables.is_variable_initialized(v0).eval())
state_ops.assign(v0, [[2.0, 3.0]]).eval()
self.assertEqual(True, variables.is_variable_initialized(v0).eval())
def testAssignNoShapeNoValidateShape(self):
with self.test_session():
value = self._NewShapelessTensor()
var = state_ops.variable_op([1, 2], tf.float32, set_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), var.get_shape())
self.assertEqual(tensor_shape.unknown_shape(),
tf.assign(var, value, validate_shape=False).get_shape())
def testAssignNoValueShapeNoValidateShape(self):
value = self._NewShapelessTensor()
shape = [1, 2]
var = state_ops.variable_op(shape, tf.float32)
self.assertEqual(shape, var.get_shape())
assigned = tf.assign(var, value, validate_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), assigned.get_shape())
def testAssignNoValueShapeNoValidateShape(self): value = self._NewShapelessTensor() shape = [1, 2] var = state_ops.variable_op(shape, dtypes.float32) self.assertEqual(shape, var.get_shape()) assigned = state_ops.assign(var, value, validate_shape=False) self.assertEqual(tensor_shape.unknown_shape(), assigned.get_shape())
def testAssignNoValueShape(self):
value = self._NewShapelessTensor()
shape = [1, 2]
var = state_ops.variable_op(shape, tf.float32)
assigned = tf.assign(var, value)
self.assertEqual(shape, var.get_shape())
self.assertEqual(shape, assigned.get_shape())
def testAssignNoShape(self):
with self.test_session():
value = self._NewShapelessTensor()
var = state_ops.variable_op([1, 2], tf.float32, set_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), var.get_shape())
self.assertEqual(tensor_shape.unknown_shape(),
tf.assign(var, value).get_shape())
def testIsVariableInitialized(self):
for use_gpu in [True, False]:
with self.test_session(use_gpu=use_gpu):
v0 = state_ops.variable_op([1, 2], dtypes.float32)
self.assertEqual(False, variables.is_variable_initialized(v0).eval())
state_ops.assign(v0, [[2.0, 3.0]]).eval()
self.assertEqual(True, variables.is_variable_initialized(v0).eval())
def __init__(self, initial_value, trainable=True, collections=None,
validate_shape=True, name=None):
"""Creates a new variable with value `initial_value`.
The new variable is added to the graph collections listed in `collections`,
which defaults to `[GraphKeys.VARIABLES]`.
If `trainable` is `True` the variable is also added to the graph collection
`GraphKeys.TRAINABLE_VARIABLES`.
This constructor creates both a `variable` Op and an `assign` Op to set the
variable to its initial value.
Args:
initial_value: A `Tensor`, or Python object convertible to a `Tensor`.
The initial value for the Variable. Must have a shape specified unless
`validate_shape` is set to False.
trainable: If `True`, the default, also adds the variable to the graph
collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
the default list of variables to use by the `Optimizer` classes.
collections: List of graph collections keys. The new variable is added to
these collections. Defaults to `[GraphKeys.VARIABLES]`.
validate_shape: If `False`, allows the variable to be initialized with a
value of unknown shape. If `True`, the default, the shape of
`initial_value` must be known.
name: Optional name for the variable. Defaults to `'Variable'` and gets
uniquified automatically.
Returns:
A Variable.
Raises:
ValueError: If the initial value does not have a shape and
`validate_shape` is `True`.
"""
if collections is None:
collections = [ops.GraphKeys.VARIABLES]
if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
with ops.control_dependencies(None):
with ops.op_scope([initial_value], name, "Variable") as name:
self._initial_value = ops.convert_to_tensor(initial_value,
name="initial_value")
initial_value_shape = self._initial_value.get_shape()
if validate_shape and not initial_value_shape.is_fully_defined():
raise ValueError("initial_value must have a shape specified: %s"
% self._initial_value)
shape_to_set = initial_value_shape if validate_shape else []
self._variable = state_ops.variable_op(
shape_to_set, self._initial_value.dtype.base_dtype,
set_shape=validate_shape, name=name)
with ops.device(self._variable.device):
self._initializer_op = state_ops.assign(
self._variable, self._initial_value,
validate_shape=validate_shape).op
self._snapshot = array_ops.identity(self._variable, name="read")
ops.add_to_collections(collections, self)
self._save_slice_info = None
def testAssignNoVarShape(self):
value = np.array([[42.0, 43.0]])
var = state_ops.variable_op(value.shape,
dtypes.float32,
set_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), var.get_shape())
assigned = state_ops.assign(var, value)
self.assertShapeEqual(value, assigned)
def _init_from_args(self, initial_value=None, trainable=True,
collections=None, validate_shape=True,
caching_device=None, name=None):
"""Creates a new variable from arguments.
Args:
initial_value: A `Tensor`, or Python object convertible to a `Tensor`.
The initial value for the Variable. Must have a shape specified unless
`validate_shape` is set to False.
trainable: If `True`, the default, also adds the variable to the graph
collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
the default list of variables to use by the `Optimizer` classes.
collections: List of graph collections keys. The new variable is added to
these collections. Defaults to `[GraphKeys.VARIABLES]`.
validate_shape: If `False`, allows the variable to be initialized with a
value of unknown shape. If `True`, the default, the shape of
`initial_value` must be known.
caching_device: Optional device string or function describing where the
Variable should be cached for reading. Defaults to the Variable's
device. If not `None`, caches on another device. Typical use is to
cache on the device where the Ops using the Variable reside, to
deduplicate copying through `Switch` and other conditional statements.
name: Optional name for the variable. Defaults to `'Variable'` and gets
uniquified automatically.
Raises:
ValueError: If the initial value is not specified, or does not have a
shape and `validate_shape` is `True`.
"""
if initial_value is None:
raise ValueError("initial_value must be specified.")
if collections is None:
collections = [ops.GraphKeys.VARIABLES]
if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
with ops.control_dependencies(None):
with ops.op_scope([initial_value], name, "Variable") as name:
self._initial_value = ops.convert_to_tensor(initial_value,
name="initial_value")
initial_value_shape = self._initial_value.get_shape()
if validate_shape and not initial_value_shape.is_fully_defined():
raise ValueError("initial_value must have a shape specified: %s"
% self._initial_value)
shape_to_set = initial_value_shape if validate_shape else []
self._variable = state_ops.variable_op(
shape_to_set, self._initial_value.dtype.base_dtype,
set_shape=validate_shape, name=name)
with ops.device(self._variable.device):
self._initializer_op = state_ops.assign(
self._variable, self._initial_value,
validate_shape=validate_shape).op
with ops.device(caching_device if caching_device is not None
else self._variable.device):
self._snapshot = array_ops.identity(self._variable, name="read")
ops.add_to_collections(collections, self)
self._caching_device = caching_device
self._save_slice_info = None
def testAssignUpdate(self):
for dtype in [
dtypes.float32, dtypes.int64, dtypes.uint32, dtypes.uint8
]:
var = state_ops.variable_op([1, 2], dtype)
added = state_ops.assign_add(var, [[2, 3]])
self.assertEqual([1, 2], added.get_shape())
subbed = state_ops.assign_sub(var, [[12, 13]])
self.assertEqual([1, 2], subbed.get_shape())
def testAssignNoShapeNoValidateShape(self):
with self.cached_session():
value = self._NewShapelessTensor()
var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), var.get_shape())
self.assertEqual(
tensor_shape.unknown_shape(),
state_ops.assign(
var, value, validate_shape=False).get_shape())
def testAssign(self):
for dtype in [
dtypes.float32, dtypes.int64, dtypes.uint32, dtypes.uint8
]:
value = np.array([[42, 43]])
var = state_ops.variable_op(value.shape, dtype)
self.assertShapeEqual(value, var)
assigned = state_ops.assign(var, value)
self.assertShapeEqual(value, assigned)
def testAssignDependencyAcrossDevices(self):
with self.test_session(use_gpu=True):
# The variable and an op to increment it are on the GPU.
var = state_ops.variable_op([1], tf.float32)
tf.assign(var, [1.0]).eval()
increment = tf.assign_add(var, [1.0])
with tf.control_dependencies([increment]):
with tf.device("/cpu:0"):
# This mul op is pinned to the CPU, but reads the variable from the
# GPU. The te
def testAverageVariablesDeviceAssignment(self):
with ops.device("dev_v0"):
v0 = variables.Variable(10.0, name="v0")
with ops.device("dev_v1"):
v1 = state_ops.variable_op(shape=[1], dtype=types.float32, name="v1")
tensor2 = v0 + v1
ema = moving_averages.ExponentialMovingAverage(0.25, name="foo_avg")
with ops.device("default"):
ema.apply([v0, v1, tensor2])
self.assertEqual("dev_v0", ema.average(v0).device)
self.assertEqual("dev_v1", ema.average(v1).device)
self.assertEqual("default", ema.average(tensor2).device)
def testAverageVariablesDeviceAssignment(self):
with tf.device("dev_v0"):
v0 = tf.Variable(10.0, name="v0")
with tf.device("dev_v1"):
v1 = state_ops.variable_op(shape=[1], dtype=tf.float32, name="v1")
tensor2 = v0 + v1
ema = tf.train.ExponentialMovingAverage(0.25, name="foo_avg")
with tf.device("default"):
ema.apply([v0, v1, tensor2])
self.assertEqual("dev_v0", ema.average(v0).device)
self.assertEqual("dev_v1", ema.average(v1).device)
self.assertEqual("default", ema.average(tensor2).device)
def testAssignDependencyAcrossDevices(self):
with self.test_session(use_gpu=True):
# The variable and an op to increment it are on the GPU.
var = state_ops.variable_op([1], tf.float32)
tf.assign(var, [1.0]).eval()
increment = tf.assign_add(var, [1.0])
with tf.control_dependencies([increment]):
with tf.device("/cpu:0"):
# This mul op is pinned to the CPU, but reads the variable from the
# GPU. The test ensures that the dependency on 'increment' is still
# honored, i.e., the Send and Recv from GPU to CPU should take place
# only after the increment.
result = tf.mul(var, var)
self.assertAllClose([4.0], result.eval())
def testObtainNext(self):
with self.test_session():
var = state_ops.variable_op([], dtypes.int64)
state_ops.assign(var, -1).op.run()
c = constant_op.constant(["a", "b"])
sample1 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample1.eval())
self.assertEqual(0, var.eval())
sample2 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"b", sample2.eval())
self.assertEqual(1, var.eval())
sample3 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample3.eval())
self.assertEqual(0, var.eval())
def _buildInitialVars(self, shape, dev_list):
values = []
num_devices = len(dev_list)
dim = np.prod(shape) if shape else 1
for d in range(0, num_devices):
with ops.device(dev_list[d]):
npt = np.zeros(shape).astype(np.float32)
alias = np.frombuffer(npt.data, dtype=np.float32)
for i in range(0, dim):
alias[i] = i + 0.01 * d
var = state_ops.variable_op(shape, types_pb2.DT_FLOAT)
state_ops.init_variable(var, npt).op.run()
values.append(var)
return values
def testObtainNext(self):
with self.test_session():
var = state_ops.variable_op([1], tf.int64)
tf.assign(var, [-1]).op.run()
c = tf.constant(["a", "b"])
sample1 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample1.eval())
self.assertEqual([0], var.eval())
sample2 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"b", sample2.eval())
self.assertEqual([1], var.eval())
sample3 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample3.eval())
self.assertEqual([0], var.eval())
def _buildInitialVars(self, shape, dev_list):
values = []
num_devices = len(dev_list)
dim = np.prod(shape, dtype=int) if shape else 1
for d in range(0, num_devices):
with ops.device(dev_list[d]):
npt = np.zeros(shape).astype(np.float32)
alias = np.frombuffer(npt.data, dtype=np.float32)
for i in range(0, dim):
alias[i] = i + 0.01 * d
var = state_ops.variable_op(shape, types_pb2.DT_FLOAT)
state_ops.init_variable(var, npt).op.run()
values.append(var)
return values
def testObtainNext(self):
with self.test_session():
var = state_ops.variable_op([1], tf.int64)
tf.assign(var, [-1]).op.run()
c = tf.constant(["a", "b"])
sample1 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample1.eval())
self.assertEqual([0], var.eval())
sample2 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"b", sample2.eval())
self.assertEqual([1], var.eval())
sample3 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample3.eval())
self.assertEqual([0], var.eval())
def testObtainNext(self):
with self.test_session():
var = state_ops.variable_op([], dtypes.int64)
state_ops.assign(var, -1).op.run()
c = constant_op.constant(["a", "b"])
sample1 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample1.eval())
self.assertEqual(0, var.eval())
sample2 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"b", sample2.eval())
self.assertEqual(1, var.eval())
sample3 = input_pipeline_ops.obtain_next(c, var)
self.assertEqual(b"a", sample3.eval())
self.assertEqual(0, var.eval())
def testAssignDependencyAcrossDevices(self):
with test_util.use_gpu():
# The variable and an op to increment it are on the GPU.
var = state_ops.variable_op([1], dtypes.float32)
self.evaluate(state_ops.assign(var, [1.0]))
increment = state_ops.assign_add(var, [1.0])
with ops.control_dependencies([increment]):
with test_util.force_cpu():
# This mul op is pinned to the CPU, but reads the variable from the
# GPU. The test ensures that the dependency on 'increment' is still
# honored, i.e., the Send and Recv from GPU to CPU should take place
# only after the increment.
result = math_ops.multiply(var, var)
self.assertAllClose([4.0], self.evaluate(result))
def testAverageVariablesDeviceAssignment(self):
with tf.device("/job:dev_v0"):
v0 = tf.Variable(10.0, name="v0")
with tf.device("/job:dev_v1"):
v1 = state_ops.variable_op(shape=[1], dtype=tf.float32, name="v1")
tensor2 = v0 + v1
ema = tf.train.ExponentialMovingAverage(0.25, name="foo_avg")
with tf.device("/job:default"):
ema.apply([v0, v1, tensor2])
self.assertDeviceEqual("/job:dev_v0", ema.average(v0).device)
self.assertDeviceEqual("/job:dev_v1", ema.average(v1).device)
# However, the colocation property is maintained.
self.assertEqual([b"loc:@v1"], ema.average(v1).op.colocation_groups())
self.assertDeviceEqual("/job:default", ema.average(tensor2).device)
def testAssignDependencyAcrossDevices(self):
with test_util.use_gpu():
# The variable and an op to increment it are on the GPU.
var = state_ops.variable_op([1], dtypes.float32)
self.evaluate(state_ops.assign(var, [1.0]))
increment = state_ops.assign_add(var, [1.0])
with ops.control_dependencies([increment]):
with test_util.force_cpu():
# This mul op is pinned to the CPU, but reads the variable from the
# GPU. The test ensures that the dependency on 'increment' is still
# honored, i.e., the Send and Recv from GPU to CPU should take place
# only after the increment.
result = math_ops.multiply(var, var)
self.assertAllClose([4.0], self.evaluate(result))
def testAssignDependencyAcrossDevices(self):
with self.test_session(use_gpu=True):
# The variable and an op to increment it are on the GPU.
var = state_ops.variable_op([1], tf.float32)
tf.assign(var, [1.0]).eval()
increment = tf.assign_add(var, [1.0])
with tf.control_dependencies([increment]):
with tf.device("/cpu:0"):
# This mul op is pinned to the CPU, but reads the variable from the
# GPU. The test ensures that the dependency on 'increment' is still
# honored, i.e., the Send and Recv from GPU to CPU should take place
# only after the increment.
result = tf.mul(var, var)
self.assertAllClose([4.0], result.eval())
def testAverageVariablesDeviceAssignment(self):
with tf.device("/job:dev_v0"):
v0 = tf.Variable(10.0, name="v0")
with tf.device("/job:dev_v1"):
v1 = state_ops.variable_op(shape=[1], dtype=tf.float32, name="v1")
tensor2 = v0 + v1
ema = tf.train.ExponentialMovingAverage(0.25, name="foo_avg")
with tf.device("/job:default"):
ema.apply([v0, v1, tensor2])
self.assertDeviceEqual("/job:dev_v0", ema.average(v0).device)
self.assertDeviceEqual("/job:dev_v1", ema.average(v1).device)
# However, the colocation property is maintained.
self.assertEqual([b"loc:@v1"],
ema.average(v1).op.colocation_groups())
self.assertDeviceEqual("/job:default", ema.average(tensor2).device)
def testDecay(self):
initial_lr = 0.1
k = 10
decay_rate = 0.96
step = state_ops.variable_op([], dtypes.int32)
assign_step = state_ops.assign(step, 0)
increment_step = state_ops.assign_add(step, 1)
decayed_lr = learning_rate_decay.inverse_time_decay(
initial_lr, step, k, decay_rate)
with self.test_session():
assign_step.op.run()
for i in range(k + 1):
expected = initial_lr / (1 + i / k * decay_rate)
self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
increment_step.op.run()
def testDecay(self):
initial_lr = 0.1
k = 10
decay_rate = 0.96
step = state_ops.variable_op([], dtypes.int32)
assign_step = state_ops.assign(step, 0)
increment_step = state_ops.assign_add(step, 1)
decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr, step,
k, decay_rate)
with self.test_session():
assign_step.op.run()
for i in range(k+1):
expected = initial_lr * math.exp(-i / k * decay_rate)
self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
increment_step.op.run()
def testStaircase(self):
with self.test_session():
step = state_ops.variable_op([], dtypes.int32)
assign_100 = state_ops.assign(step, 100)
assign_1 = state_ops.assign(step, 1)
assign_2 = state_ops.assign(step, 2)
decayed_lr = learning_rate_decay.exponential_decay(0.1, step, 3, 0.96, staircase=True)
# No change to learning rate
assign_1.op.run()
self.assertAllClose(decayed_lr.eval(), 0.1, 1e-6)
assign_2.op.run()
self.assertAllClose(decayed_lr.eval(), 0.1, 1e-6)
# Decayed learning rate
assign_100.op.run()
expected = 0.1 * 0.96 ** (100 // 3)
self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
def testStaircase(self):
with self.test_session():
step = state_ops.variable_op([], dtypes.int32)
assign_100 = state_ops.assign(step, 100)
assign_1 = state_ops.assign(step, 1)
assign_2 = state_ops.assign(step, 2)
decayed_lr = learning_rate_decay.exponential_decay(.1, step, 3, 0.96,
staircase=True)
# No change to learning rate
assign_1.op.run()
self.assertAllClose(decayed_lr.eval(), .1, 1e-6)
assign_2.op.run()
self.assertAllClose(decayed_lr.eval(), .1, 1e-6)
# Decayed learning rate
assign_100.op.run()
expected = .1 * 0.96**(100 // 3)
self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
def testContainer(self):
with tf.Graph().as_default():
v0 = tf.Variable([0])
with tf.container("l1"):
v1 = tf.Variable([1])
with tf.container("l2"):
v2 = tf.Variable([2])
special_v = state_ops.variable_op([1], tf.float32, container="l3")
v3 = tf.Variable([3])
v4 = tf.Variable([4])
self.assertEqual(tf.compat.as_bytes(""), v0.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l1"), v1.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l2"), v2.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l3"),
special_v.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l1"), v3.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes(""), v4.op.get_attr("container"))
def testStaircase(self):
initial_lr = 0.1
k = 10
decay_rate = 0.96
step = state_ops.variable_op([], dtypes.int32)
assign_step = state_ops.assign(step, 0)
increment_step = state_ops.assign_add(step, 1)
decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr,
step,
k,
decay_rate,
staircase=True)
with self.test_session():
assign_step.op.run()
for i in range(k + 1):
expected = initial_lr * math.exp(-decay_rate * (i // k))
self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
increment_step.op.run()
def testPinRequiredOpsOnCPU(self):
with ops.Graph().as_default() as g, g.device(graph_util.pin_variables_on_cpu):
const_a = constant_op.constant(5.0)
const_b = constant_op.constant(10.0)
add_c = const_a + const_b
var_v = state_ops.variable_op([], dtype=types.float32)
assign_c_to_v = state_ops.assign(var_v, add_c)
dynamic_stitch_int_result = data_flow_ops.dynamic_stitch([[0, 1, 2], [2, 3]], [[12, 23, 34], [1, 2]])
dynamic_stitch_float_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12.0, 23.0, 34.0], [1.0, 2.0]]
)
# Non-variable ops shuld not specify a device
self.assertEqual(const_a.device, None)
self.assertEqual(const_b.device, None)
self.assertEqual(add_c.device, None)
# Variable ops specify a device
self.assertEqual(var_v.device, "/device:CPU:0")
self.assertEqual(assign_c_to_v.device, "/device:CPU:0")
def testStaircase(self):
initial_lr = 0.1
k = 10
decay_rate = 0.96
step = state_ops.variable_op([], dtypes.int32)
assign_step = state_ops.assign(step, 0)
increment_step = state_ops.assign_add(step, 1)
decayed_lr = learning_rate_decay.inverse_time_decay(initial_lr,
step,
k,
decay_rate,
staircase=True)
with self.test_session():
assign_step.op.run()
for i in range(k+1):
expected = initial_lr / (1 + decay_rate * (i // k))
self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
increment_step.op.run()
def testContainer(self):
with tf.Graph().as_default():
v0 = tf.Variable([0])
with tf.container("l1"):
v1 = tf.Variable([1])
with tf.container("l2"):
v2 = tf.Variable([2])
special_v = state_ops.variable_op([1],
tf.float32,
container="l3")
v3 = tf.Variable([3])
v4 = tf.Variable([4])
self.assertEqual(tf.compat.as_bytes(""), v0.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l1"), v1.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l2"), v2.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l3"),
special_v.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes("l1"), v3.op.get_attr("container"))
self.assertEqual(tf.compat.as_bytes(""), v4.op.get_attr("container"))
def testPinRequiredOpsOnCPU(self):
with ops.Graph().as_default() as g, g.device(
graph_util.pin_variables_on_cpu):
const_a = constant_op.constant(5.0)
const_b = constant_op.constant(10.0)
add_c = const_a + const_b
var_v = state_ops.variable_op([], dtype=dtypes.float32)
assign_c_to_v = state_ops.assign(var_v, add_c)
dynamic_stitch_int_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12, 23, 34], [1, 2]])
dynamic_stitch_float_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12.0, 23.0, 34.0], [1.0, 2.0]])
# Non-variable ops shuld not specify a device
self.assertDeviceEqual(const_a.device, None)
self.assertDeviceEqual(const_b.device, None)
self.assertDeviceEqual(add_c.device, None)
# Variable ops specify a device
self.assertDeviceEqual(var_v.device, "/device:CPU:0")
self.assertDeviceEqual(assign_c_to_v.device, "/device:CPU:0")
def testPinToCpu(self):
with ops.Graph().as_default() as g, g.device(graph_util.pin_to_cpu):
const_a = constant_op.constant(5.0)
const_b = constant_op.constant(10.0)
add_c = const_a + const_b
var_v = state_ops.variable_op([], dtype=types.float32)
assign_c_to_v = state_ops.assign(var_v, add_c)
const_string = constant_op.constant("on a cpu")
dynamic_stitch_int_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12, 23, 34], [1, 2]])
dynamic_stitch_float_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12.0, 23.0, 34.0], [1.0, 2.0]])
self.assertEqual(const_a.device, "/device:CPU:0")
self.assertEqual(const_b.device, "/device:CPU:0")
self.assertEqual(add_c.device, "/device:CPU:0")
self.assertEqual(var_v.device, "/device:CPU:0")
self.assertEqual(assign_c_to_v.device, "/device:CPU:0")
self.assertEqual(const_string.device, "/device:CPU:0")
self.assertEqual(dynamic_stitch_int_result.device, "/device:CPU:0")
self.assertEqual(dynamic_stitch_float_result.device, "/device:CPU:0")
def testPinToCpu(self):
with ops.Graph().as_default() as g, g.device(graph_util.pin_to_cpu):
const_a = constant_op.constant(5.0)
const_b = constant_op.constant(10.0)
add_c = const_a + const_b
var_v = state_ops.variable_op([], dtype=dtypes.float32)
assign_c_to_v = state_ops.assign(var_v, add_c)
const_string = constant_op.constant("on a cpu")
dynamic_stitch_int_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12, 23, 34], [1, 2]])
dynamic_stitch_float_result = data_flow_ops.dynamic_stitch(
[[0, 1, 2], [2, 3]], [[12.0, 23.0, 34.0], [1.0, 2.0]])
self.assertDeviceEqual(const_a.device, "/device:CPU:0")
self.assertDeviceEqual(const_b.device, "/device:CPU:0")
self.assertDeviceEqual(add_c.device, "/device:CPU:0")
self.assertDeviceEqual(var_v.device, "/device:CPU:0")
self.assertDeviceEqual(assign_c_to_v.device, "/device:CPU:0")
self.assertDeviceEqual(const_string.device, "/device:CPU:0")
self.assertDeviceEqual(dynamic_stitch_int_result.device, "/device:CPU:0")
self.assertDeviceEqual(dynamic_stitch_float_result.device, "/device:CPU:0")
def testTwoDeviceFunctions(self):
with ops.Graph().as_default() as g:
var_0 = state_ops.variable_op([1], dtype=dtypes.float32)
with g.device(graph_util.pin_variables_on_cpu):
var_1 = state_ops.variable_op([1], dtype=dtypes.float32)
var_2 = state_ops.variable_op([1], dtype=dtypes.float32)
var_3 = state_ops.variable_op([1], dtype=dtypes.float32)
with g.device(graph_util.pin_variables_on_cpu):
var_4 = state_ops.variable_op([1], dtype=dtypes.float32)
with g.device("/device:GPU:0"):
var_5 = state_ops.variable_op([1], dtype=dtypes.float32)
var_6 = state_ops.variable_op([1], dtype=dtypes.float32)
self.assertDeviceEqual(var_0.device, None)
self.assertDeviceEqual(var_1.device, "/device:CPU:0")
self.assertDeviceEqual(var_2.device, None)
self.assertDeviceEqual(var_3.device, None)
self.assertDeviceEqual(var_4.device, "/device:CPU:0")
self.assertDeviceEqual(var_5.device, "/device:GPU:0")
self.assertDeviceEqual(var_6.device, "/device:CPU:0")
def testTwoDeviceFunctions(self):
with ops.Graph().as_default() as g:
var_0 = state_ops.variable_op([1], dtype=dtypes.float32)
with g.device(graph_util.pin_variables_on_cpu):
var_1 = state_ops.variable_op([1], dtype=dtypes.float32)
var_2 = state_ops.variable_op([1], dtype=dtypes.float32)
var_3 = state_ops.variable_op([1], dtype=dtypes.float32)
with g.device(graph_util.pin_variables_on_cpu):
var_4 = state_ops.variable_op([1], dtype=dtypes.float32)
with g.device("/device:GPU:0"):
var_5 = state_ops.variable_op([1], dtype=dtypes.float32)
var_6 = state_ops.variable_op([1], dtype=dtypes.float32)
self.assertDeviceEqual(var_0.device, None)
self.assertDeviceEqual(var_1.device, "/device:CPU:0")
self.assertDeviceEqual(var_2.device, None)
self.assertDeviceEqual(var_3.device, None)
self.assertDeviceEqual(var_4.device, "/device:CPU:0")
self.assertDeviceEqual(var_5.device, "/device:GPU:0")
self.assertDeviceEqual(var_6.device, "/device:CPU:0")
def testUnknown(self): tf_val = state_ops.variable_op(shape=[3, 4, 7], dtype=tf.float32) self.assertIs(None, tf.contrib.util.constant_value(tf_val))
def testUnknown(self): tf_val = state_ops.variable_op(shape=[3, 4, 7], dtype=dtypes.float32) self.assertIs(None, tensor_util.ConstantValue(tf_val))
def _init_from_args(
self,
initial_value=None,
trainable=True,
collections=None,
validate_shape=True,
caching_device=None,
name=None,
dtype=None,
expected_shape=None,
):
"""Creates a new variable from arguments.
Args:
initial_value: A `Tensor`, or Python object convertible to a `Tensor`,
which is the initial value for the Variable. The initial value must have
a shape specified unless `validate_shape` is set to False. Can also be a
callable with no argument that returns the initial value when called. In
that case, `dtype` must be specified. (Note that initializer functions
from init_ops.py must first be bound to a shape before being used here.)
trainable: If `True`, the default, also adds the variable to the graph
collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
the default list of variables to use by the `Optimizer` classes.
collections: List of graph collections keys. The new variable is added to
these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`.
validate_shape: If `False`, allows the variable to be initialized with a
value of unknown shape. If `True`, the default, the shape of
`initial_value` must be known.
caching_device: Optional device string or function describing where the
Variable should be cached for reading. Defaults to the Variable's
device. If not `None`, caches on another device. Typical use is to
cache on the device where the Ops using the Variable reside, to
deduplicate copying through `Switch` and other conditional statements.
name: Optional name for the variable. Defaults to `'Variable'` and gets
uniquified automatically.
dtype: If set, initial_value will be converted to the given type.
If None, either the datatype will be kept (if initial_value is
a Tensor) or float32 will be used (if it is a Python object convertible
to a Tensor).
expected_shape: A TensorShape. If set, initial_value is expected
to have this shape.
Raises:
ValueError: If the initial value is not specified, or does not have a
shape and `validate_shape` is `True`.
"""
if initial_value is None:
raise ValueError("initial_value must be specified.")
init_from_fn = callable(initial_value)
if init_from_fn and dtype is None:
raise ValueError("dtype must also be specified when initial_value is callable.")
if collections is None:
collections = [ops.GraphKeys.GLOBAL_VARIABLES]
if not isinstance(collections, (list, tuple, set)):
raise ValueError(
"collections argument to Variable constructor must be a list, tuple, "
"or set. Got %s of type %s" % (collections, type(collections))
)
if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
expected_shape = tensor_shape.as_shape(expected_shape)
with ops.control_dependencies(None):
with ops.name_scope(name, "Variable", [] if init_from_fn else [initial_value]) as name:
# Get the initial value from a callable function. The real shape of the
# variable will be set later, since under the init_from_fn case, the
# shape won't be known until after the function is invoked.
#
# NOTE: The current Variable OpKernel does not support
# partially defined shapes, so we only set the shape if it is
# fully defined. For historical reasons, we use the scalar
# shape (`[]`) to represent an unknown or partially known
# shape. A future version of the Variable ops will remove this
# limitation.
def full_shape_to_list(shape):
"""Returns shape as a list if shape is fully defined."""
if shape and shape.is_fully_defined():
return shape.as_list()
else:
return []
def assert_expected_shape():
"""Asserts that the initial value has the expected shape."""
if expected_shape:
expected_shape.assert_is_compatible_with(self._initial_value.get_shape())
if init_from_fn:
expected_shape_list = full_shape_to_list(expected_shape)
set_shape = validate_shape and expected_shape.is_fully_defined()
self._variable = state_ops.variable_op(
expected_shape_list, dtype.base_dtype, set_shape=set_shape, name=name
)
with ops.colocate_with(self._variable.op):
with ops.name_scope("Initializer"):
# Colocate the tensors created by the initial_value() function
# with the variable itself.
self._initial_value = ops.convert_to_tensor(
initial_value(), name="initial_value", dtype=dtype
)
assert_expected_shape()
# Or get the initial value from a Tensor or Python object.
else:
self._initial_value = ops.convert_to_tensor(initial_value, name="initial_value", dtype=dtype)
assert_expected_shape()
set_shape = validate_shape and self._initial_value.get_shape().is_fully_defined()
# In this case, the variable op can't be created until after the
# initial_value has been converted to a Tensor with a known type.
self._variable = state_ops.variable_op(
full_shape_to_list(self._initial_value.get_shape()),
self._initial_value.dtype.base_dtype,
set_shape=set_shape,
name=name,
)
# Manually overrides the variable's shape with the initial value's.
if validate_shape:
initial_value_shape = self._initial_value.get_shape()
if not initial_value_shape.is_fully_defined():
raise ValueError("initial_value must have a shape specified: %s" % self._initial_value)
self._variable.set_shape(initial_value_shape)
# TODO(b/28152992): Remove the below hack modifying the node_def shape
# directly once set_shape() handles it.
self._variable.op.node_def.attr["shape"].shape.CopyFrom(initial_value_shape.as_proto())
# Assigns initial value.
self._initializer_op = state_ops.assign(
self._variable, self._initial_value, validate_shape=validate_shape
).op
# TODO(vrv): Change this class to not take caching_device, but
# to take the op to colocate the snapshot with, so we can use
# colocation rather than devices.
if caching_device is not None:
with ops.device(caching_device):
self._snapshot = array_ops.identity(self._variable, name="read")
else:
with ops.colocate_with(self._variable.op):
self._snapshot = array_ops.identity(self._variable, name="read")
ops.add_to_collections(collections, self)
self._caching_device = caching_device
self._save_slice_info = None
def testAssignNoValidateShape(self): value = np.array([[42.0, 43.0]]) var = state_ops.variable_op(value.shape, dtypes.float32) self.assertShapeEqual(value, var) assigned = state_ops.assign(var, value, validate_shape=False) self.assertShapeEqual(value, assigned)
def _init_from_args(self,
initial_value=None,
trainable=True,
collections=None,
validate_shape=True,
caching_device=None,
name=None):
"""Creates a new variable from arguments.
Args:
initial_value: A `Tensor`, or Python object convertible to a `Tensor`.
The initial value for the Variable. Must have a shape specified unless
`validate_shape` is set to False.
trainable: If `True`, the default, also adds the variable to the graph
collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
the default list of variables to use by the `Optimizer` classes.
collections: List of graph collections keys. The new variable is added to
these collections. Defaults to `[GraphKeys.VARIABLES]`.
validate_shape: If `False`, allows the variable to be initialized with a
value of unknown shape. If `True`, the default, the shape of
`initial_value` must be known.
caching_device: Optional device string or function describing where the
Variable should be cached for reading. Defaults to the Variable's
device. If not `None`, caches on another device. Typical use is to
cache on the device where the Ops using the Variable reside, to
deduplicate copying through `Switch` and other conditional statements.
name: Optional name for the variable. Defaults to `'Variable'` and gets
uniquified automatically.
Raises:
ValueError: If the initial value is not specified, or does not have a
shape and `validate_shape` is `True`.
"""
if initial_value is None:
raise ValueError("initial_value must be specified.")
if collections is None:
collections = [ops.GraphKeys.VARIABLES]
if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
collections = list(collections) + [
ops.GraphKeys.TRAINABLE_VARIABLES
]
with ops.control_dependencies(None):
with ops.op_scope([initial_value], name, "Variable") as name:
self._initial_value = ops.convert_to_tensor(
initial_value, name="initial_value")
initial_value_shape = self._initial_value.get_shape()
if validate_shape and not initial_value_shape.is_fully_defined(
):
raise ValueError(
"initial_value must have a shape specified: %s" %
self._initial_value)
shape_to_set = initial_value_shape if validate_shape else []
self._variable = state_ops.variable_op(
shape_to_set,
self._initial_value.dtype.base_dtype,
set_shape=validate_shape,
name=name)
with ops.device(self._variable.device):
self._initializer_op = state_ops.assign(
self._variable,
self._initial_value,
validate_shape=validate_shape).op
with ops.device(caching_device if caching_device is not None
else self._variable.device):
self._snapshot = array_ops.identity(self._variable,
name="read")
ops.add_to_collections(collections, self)
self._caching_device = caching_device
self._save_slice_info = None
def testAssignUpdateNoShape(self): var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False) added = state_ops.assign_add(var, self._NewShapelessTensor()) self.assertEqual(tensor_shape.unknown_shape(), added.get_shape()) subbed = state_ops.assign_sub(var, self._NewShapelessTensor()) self.assertEqual(tensor_shape.unknown_shape(), subbed.get_shape())
def _initFetch(self, x, tftype, use_gpu=None):
with self.test_session(use_gpu=use_gpu):
p = state_ops.variable_op(x.shape, tftype)
op = state_ops.assign(p, x)
op.op.run()
return self.evaluate(p)
def testAssignNoVarShapeNoValidateShape(self): value = np.array([[42.0, 43.0]]) var = state_ops.variable_op(value.shape, dtypes.float32, set_shape=False) self.assertEqual(tensor_shape.unknown_shape(), var.get_shape()) assigned = state_ops.assign(var, value, validate_shape=False) self.assertShapeEqual(value, assigned)
def _initFetch(self, x, tftype, use_gpu=None):
with self.test_session(use_gpu=use_gpu):
p = state_ops.variable_op(x.shape, tftype)
op = tf.assign(p, x)
op.op.run()
return p.eval()
def testset_shape(self):
p = state_ops.variable_op([1, 2], tf.float32)
self.assertEqual([1, 2], p.get_shape())
p = state_ops.variable_op([1, 2], tf.float32, set_shape=False)
self.assertEqual(tensor_shape.unknown_shape(), p.get_shape())
def __init__(self,
initial_value,
trainable=True,
collections=None,
validate_shape=True,
name=None):
"""Creates a new variable with value `initial_value`.
The new variable is added to the graph collections listed in `collections`,
which defaults to `[GraphKeys.VARIABLES]`.
If `trainable` is `True` the variable is also added to the graph collection
`GraphKeys.TRAINABLE_VARIABLES`.
This constructor creates both a `variable` Op and an `assign` Op to set the
variable to its initial value.
Args:
initial_value: A `Tensor`, or Python object convertible to a `Tensor`.
The initial value for the Variable. Must have a shape specified unless
`validate_shape` is set to False.
trainable: If `True`, the default, also adds the variable to the graph
collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
the default list of variables to use by the `Optimizer` classes.
collections: List of graph collections keys. The new variable is added to
these collections. Defaults to `[GraphKeys.VARIABLES]`.
validate_shape: If `False`, allows the variable to be initialized with a
value of unknown shape. If `True`, the default, the shape of
`initial_value` must be known.
name: Optional name for the variable. Defaults to `'Variable'` and gets
uniquified automatically.
Returns:
A Variable.
Raises:
ValueError: If the initial value does not have a shape and
`validate_shape` is `True`.
"""
if collections is None:
collections = [ops.GraphKeys.VARIABLES]
if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
# pylint: disable=g-no-augmented-assignment
#
# Pylint wants us to write collections += [...TRAINABLE_VARIABLES] which
# is not the same (it modifies the list in place.) Here, we only want to
# modify the value of the variable, not the list.
collections = collections + [ops.GraphKeys.TRAINABLE_VARIABLES]
# pylint: enable=g-no-augmented-assignment
with ops.op_scope([initial_value], name, "Variable") as name:
self._initial_value = ops.convert_to_tensor(initial_value,
name="initial_value")
if not self._initial_value.get_shape().is_fully_defined():
if validate_shape:
raise ValueError(
"initial_value must have a shape specified: %s" %
self._initial_value)
self._variable = state_ops.variable_op(
[],
self._initial_value.dtype.base_dtype,
set_shape=False,
name=name)
with ops.device(self._variable.device):
self._initializer_op = state_ops.assign(
self._variable,
self._initial_value,
validate_shape=False).op
self._snapshot = array_ops.identity(self._variable,
name="read")
else:
self._variable = state_ops.variable_op(
self._initial_value.get_shape(),
self._initial_value.dtype.base_dtype,
name=name)
with ops.device(self._variable.device):
self._initializer_op = state_ops.assign(
self._variable, self._initial_value).op
self._snapshot = array_ops.identity(self._variable,
name="read")
for key in collections:
ops.add_to_collection(key, self)
self._save_slice_info = None
def testAssignNoValidateShape(self):
value = np.array([[42.0, 43.0]])
var = state_ops.variable_op(value.shape, tf.float32)
self.assertShapeEqual(value, var)
assigned = tf.assign(var, value, validate_shape=False)
self.assertShapeEqual(value, assigned)
def testUnknown(self):
tf_val = state_ops.variable_op(shape=[3, 4, 7], dtype=tf.float32)
self.assertIs(None, tf.contrib.util.constant_value(tf_val))
def testset_shape(self): p = state_ops.variable_op([1, 2], dtypes.float32) self.assertEqual([1, 2], p.get_shape()) p = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False) self.assertEqual(tensor_shape.unknown_shape(), p.get_shape())
