def testReturnShape(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=[2, None])
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[[1, 2], [3, 4]],
[[5, 6], [7, 8]]]
]).astype(np.float32)).run()
val = sess.run(q.take_indexed_slices_grad(1))
self.assertAllEqual(val.dense_shape, [2, 2, 2, 2])
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=[None, 2])
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[[1, 2, 3], [4, 5, 6]],
[[7, 8, 9], [10, 11, 12]]]
]).astype(np.float32)).run()
val = sess.run(q.take_indexed_slices_grad(1))
self.assertAllEqual(val.dense_shape, [-1, 2, 2, 3])
def testAccumulatorMultipleAccumulators(self):
with self.test_session() as sess:
q_f32_0 = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([2, 2]))
q_f32_1 = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([2, 2]))
q_f16_0 = tf.SparseConditionalAccumulator(
tf.float16, name="Q", shape=tf.TensorShape([2, 2]))
q_f16_1 = tf.SparseConditionalAccumulator(
tf.float16, name="Q", shape=tf.TensorShape([2, 2]))
accums = [q_f16_0, q_f16_1, q_f32_0, q_f32_1]
elems = [[[1, 0], [0, 0]], [[0, 1], [0, 0]], [[0, 0], [1, 0]], [[0, 0],
[0, 1]]]
expected_tensors = []
for i in range(len(accums)):
tensor_to_add = np.array(elems[i]).astype(accums[
i].dtype.as_numpy_dtype)
expected_tensor = _indexedslice(tensor_to_add)
expected_tensors.append(expected_tensor)
st = _indexedslice(tensor_to_add)
accums[i].apply_indexed_slices_grad(st).run()
for i in range(len(accums)):
result = sess.run(accums[i].take_indexed_slices_grad(1))
self._assertEqual_indexedslices(expected_tensors[i], result)
def testEmptyShapeApply(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=tf.TensorShape([]))
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"Input indices should be vector"):
q.apply_grad(grad_indices=0, grad_values=[1.0],
grad_shape=[]).run()
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"Input indices should be vector"):
q.apply_grad(grad_indices=0, grad_values=[1.0]).run()
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"Values cannot be 0-dimensional."):
q.apply_grad(grad_indices=[0], grad_values=1.0,
grad_shape=[]).run()
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"Values cannot be 0-dimensional."):
q.apply_grad(grad_indices=[0], grad_values=1.0).run()
# The right way to apply a scalar
q.apply_grad(grad_indices=[0], grad_values=[1.0],
grad_shape=[]).run()
q.apply_grad(grad_indices=[0], grad_values=[1.0]).run()
def testParallelApplyGrad(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=tf.TensorShape([2, 2]))
elems = [
10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0
]
accum_ops = []
for x in elems:
x = _indexedslice(
np.array([[x, 0], [0, x]]).astype(np.float32))
accum_ops.append(q.apply_indexed_slices_grad(x, local_step=0))
takeg_t = q.take_indexed_slices_grad(1)
def apply_indexed_slices_grad(accum_op):
sess.run(accum_op)
threads = [
self.checkedThread(target=apply_indexed_slices_grad,
args=(o, )) for o in accum_ops
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
val = sess.run(takeg_t)
expected_val = sum(elems) / len(elems)
self._assertEqual_nparray(
np.array([[expected_val, 0],
[0, expected_val]]).astype(np.float32), val, sess)
def testApplyGradtInt32IndicesAndShape(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=tf.TensorShape([3, 3]))
accum_op = q.apply_grad(grad_indices=tf.constant([0, 2],
dtype=tf.int32),
grad_values=tf.constant(
[[0, 0, 1], [3, 0, 4]],
dtype=tf.float32),
grad_shape=tf.constant([3, 3],
dtype=tf.int32))
accum_op.run()
accum_op = q.apply_indexed_slices_grad(
tf.IndexedSlices(indices=tf.constant([0, 2], dtype=tf.int32),
values=tf.constant([[0, 0, 1], [3, 0, 4]],
dtype=tf.float32),
dense_shape=tf.constant([3, 3],
dtype=tf.int32)))
accum_op.run()
self.assertEqual(q.num_accumulated().eval(), 2)
val = sess.run(q.take_indexed_slices_grad(1))
self.assertAllEqual(val.indices, [0, 2])
self.assertAllEqual(val.values, [[0, 0, 1], [3, 0, 4]])
self.assertAllEqual(val.dense_shape, [3, 3])
def testAccumulatorApplyAndBlockingTake(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=tf.TensorShape([2, 2]))
elems = [10.0, 20.0, 30.0]
elems_ave = sum(elems) / len(elems)
accum_ops = []
for x in elems:
x = _indexedslice(
np.array([[0, x], [0, 0]]).astype(np.float32))
accum_ops.append(q.apply_indexed_slices_grad(x, local_step=0))
takeg_t = q.take_indexed_slices_grad(3)
results = []
def apply_indexed_slices_grad():
for accum_op in accum_ops:
sess.run(accum_op)
def take_grad():
results.append(sess.run(takeg_t))
accum_thread = self.checkedThread(target=apply_indexed_slices_grad)
takeg_thread = self.checkedThread(target=take_grad)
accum_thread.start()
takeg_thread.start()
accum_thread.join()
takeg_thread.join()
self._assertEqual_nparray([[0, elems_ave], [0, 0]], results[0],
sess)
def testAccumulatorSetGlobalStep(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=tf.TensorShape([1]))
set_global_step_op = q.set_global_step(1)
set_global_step_op.run()
def testZeroDimensionValues(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([3, 3]))
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"Values cannot be 0-dimensional."):
q.apply_grad(
grad_indices=[0], grad_values=np.array(1).astype(np.float32)).run()
def testAccumulatorApplyGradFloat32(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([3, 3]))
accum_op = q.apply_indexed_slices_grad(
tf.IndexedSlices(
indices=[0, 2],
values=np.array([[0, 0, 1], [3, 0, 4]]).astype(np.float32)))
accum_op.run()
self.assertEqual(q.num_accumulated().eval(), 1)
def testWrongNonEmptyInputValues(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([3, 3]))
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
" non-empty input values, got "):
q.apply_grad(
grad_indices=[0, 1],
grad_values=np.array([[0, 1, 1]]).astype(np.float32)).run()
def testConstructor(self):
with tf.Graph().as_default():
q = tf.SparseConditionalAccumulator(tf.float32, name="Q")
self.assertTrue(isinstance(q.accumulator_ref, tf.Tensor))
self.assertProtoEquals("""
name:'Q' op:'SparseConditionalAccumulator'
attr { key: 'dtype' value { type: DT_FLOAT } }
attr { key: 'shape' value { shape { unknown_rank: true} } }
attr { key: 'container' value { s: '' } }
attr { key: 'shared_name' value { s: '' } }
""", q.accumulator_ref.op.node_def)
def testNonVectorIndices(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([3, 3]))
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Input indices should be vector but received shape:"):
q.apply_grad(
grad_indices=[[0, 1], [1, 0]],
grad_values=np.array([1, 2]).astype(np.float32)).run()
def testDynamicWrongNonEmptyInputValues(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([3, 3]))
x_indices = tf.placeholder(tf.int64)
x_values = tf.placeholder(tf.float32)
accum_op = q.apply_grad(grad_indices=x_indices, grad_values=x_values)
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
" non-empty input values, got "):
sess.run(accum_op,
feed_dict={x_indices: [0, 1],
x_values: np.array([[0, 1, 1]]).astype(np.float32)})
def testDynamicNonVectorIndices(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([3, 3]))
x_indices = tf.placeholder(tf.int64)
x_values = tf.placeholder(tf.float32)
accum_op = q.apply_grad(grad_indices=x_indices, grad_values=x_values)
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Input indices should be vector but received shape:"):
sess.run(accum_op,
feed_dict={x_indices: [[0, 1], [1, 0]],
x_values: np.array([1, 2]).astype(np.float32)})
def testAccumulatorCancel(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([1, 2, 3]))
takeg_t = q.take_indexed_slices_grad(1)
takeg_thread = self.checkedThread(
self._blocking_takeg, args=(sess, takeg_t))
takeg_thread.start()
time.sleep(1.0)
sess.close() # Will cancel blocked operation
takeg_thread.join()
def testConstructorWithShape(self):
with tf.Graph().as_default():
q = tf.SparseConditionalAccumulator(
tf.float32, name="Q", shape=tf.TensorShape([1, 5, 2, 8]))
self.assertTrue(isinstance(q.accumulator_ref, tf.Tensor))
self.assertProtoEquals("""
name:'Q' op:'SparseConditionalAccumulator'
attr { key: 'dtype' value { type: DT_FLOAT } }
attr { key: 'shape' value { shape { dim {size: 1 }
dim {size: 5 }
dim {size: 2 }
dim {size: 8 }
} } }
attr { key: 'container' value { s: '' } }
attr { key: 'shared_name' value { s: '' } }
""", q.accumulator_ref.op.node_def)
def testAccumulatorTakeGrad(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32, name="Q", shape=())
grad_indexed_slices = tf.IndexedSlices(
indices=[0, 1], values=np.array([[1, 0], [0, 2]]).astype(np.float32))
accum_op = q.apply_indexed_slices_grad(grad_indexed_slices)
accum_op.run()
accum_op = q.apply_grad([0, 2],
np.array([[0, 1], [3, 0]]).astype(np.float32),
[3, 2])
accum_op.run()
takeg_t = q.take_indexed_slices_grad(1)
val = sess.run(takeg_t)
self.assertAllEqual(val.indices, [0, 1, 2])
self.assertAllEqual(val.values, [[0.5, 0.5], [0, 2], [3, 0]])
self.assertAllEqual(val.dense_shape, [-1, 2])
def testParallelTakeGrad(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=tf.TensorShape([2, 2]))
elems = [e + 1 for e in range(10)]
accum_ops = []
for e in elems:
v = _indexedslice(
np.array([[0, 0], [e, 0]]).astype(np.float32))
accum_ops.append(
q.apply_indexed_slices_grad(v, local_step=e - 1))
takeg_t = q.take_indexed_slices_grad(1)
results = []
def apply_indexed_slices_grad():
for accum_op in accum_ops:
time.sleep(1.0)
sess.run(accum_op)
apply_indexed_slices_grad_thread = self.checkedThread(
target=apply_indexed_slices_grad)
def take_grad():
t = sess.run(takeg_t)
results.append(t)
threads = [self.checkedThread(target=take_grad) for _ in range(10)]
for thread in threads:
thread.start()
apply_indexed_slices_grad_thread.start()
for thread in threads:
thread.join()
apply_indexed_slices_grad_thread.join()
for i in range(len(accum_ops)):
self._assertEqual_nparray(np.array([[0, 0], [elems[i], 0]]),
results[i], sess)
def testDtypes(self):
with self.test_session() as sess:
dtypes = [tf.float16, tf.float32, tf.float64]
for i in range(len(dtypes)):
dtype = dtypes[i]
q = tf.SparseConditionalAccumulator(
dtype, shape=tf.TensorShape([3, 3, 3]))
elems = np.arange(2)
sum_elems = np.zeros([3, 3, 3]).astype(dtype.as_numpy_dtype)
for e in elems:
mat_to_add = np.zeros([3, 3, 3]).astype(dtype.as_numpy_dtype)
mat_to_add[i, i, i] = e + 1
sum_elems += mat_to_add
t = _indexedslice(mat_to_add)
q.apply_indexed_slices_grad(t).run()
result = sess.run(q.take_indexed_slices_grad(1))
self._assertEqual_nparray(sum_elems / len(elems), result, sess)
barrier.insert_many(0, keys=["k1", "k2"], values=["a", "b"]).run() barrier.insert_many(1, keys=["k1"], values=[1]).run() barrier.insert_many(0, keys=["k3"], values=["c"]).run() barrier.insert_many(1, keys=["k3"], values=[3]).run() barrier.insert_many(1, keys=["k2"], values=[2]).run() r(barrier.take_many(2)) #acc1 = tf.SparseConditionalAccumulator(dtype=tf.float32) acc1 = tft.SparseSum() r(acc1.apply_grad([42, 69], [42.0, 420.69])) r(acc1.apply_grad([42, 69, 128], [42.0, 420.69, 4.0])) r(acc1.take()) acc2 = tf.SparseConditionalAccumulator(dtype=tf.float32, reduction_type='SUM') r(acc2.apply_grad([42, 69], [42.0, 420.69])) r(acc2.apply_grad([42, 69, 128], [42.0, 420.69, 4.0])) r(acc2.take_grad(1)) >>> fetch_function = lambda squared_tensor:([squared_tensor.sq], lambda val: val[0]) >>> feed_function = lambda feed, feed_val: [(feed.sq, feed_val)] >>> feed_function_for_partial_run = lambda feed: [feed.sq] >>> tf_session_lib.register_session_run_conversion_functions(SquaredTensor, fetch_function=fetch_function, feed_function=feed_function, feed_function_for_partial_run=feed_function_for_partial_run) >>> sq1 = SquaredTensor(tf.identity(42.0)) >>> r(tf.add(1,2)) 3 >>> r(sq1) 1764.0 >>> sq1
def testAccumulatorSizeEmpty(self):
with self.test_session():
q = tf.SparseConditionalAccumulator(tf.float32, name="Q")
self.assertEqual(q.num_accumulated().eval(), 0)
def testValidateShape(self):
with self.test_session() as sess:
q = tf.SparseConditionalAccumulator(tf.float32,
name="Q",
shape=[2, 2, None])
# Provided shape has wrong rank
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected shape rank at least 3, got 2"):
q.apply_grad(grad_indices=[0],
grad_values=np.array([[1, 2]]).astype(np.float32),
grad_shape=[2, 2]).run()
# Provided shape has wrong dim
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected shape dim 1 to be 2, got 3"):
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[1, 2], [3, 4],
[5,
6]]]).astype(np.float32),
grad_shape=[2, 3, 2]).run()
# Indices exceeded accumulator's shape's limits
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: index of slice 0 exceeded limits of shape;"
" index is 3 exceeded 2"):
q.apply_grad(grad_indices=[3],
grad_values=np.array([[[1, 2], [3, 4]]]).astype(
np.float32)).run()
# Values' rank does not match shape
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected values rank at least 3, got 2"):
q.apply_grad(grad_indices=[0, 1],
grad_values=np.array([[1, 2], [3, 4]]).astype(
np.float32)).run()
# Values' dim does not match shape
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected values dim 1 to be 2, got 3"):
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[1, 2], [3, 4],
[5, 6]]]).astype(
np.float32)).run()
# First successful gradient creates additional constraints
# Shape will be additionally be constrained to [None,2,2,2] hereafter.
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[[1, 2], [3, 4]],
[[5, 6], [7, 8]]]
]).astype(np.float32)).run()
# Values' rank does not match accumulated gradient
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected values rank 4, got 3"):
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[1, 2], [3, 4]]]).astype(
np.float32)).run()
# Values' dim does not match accumulated gradient
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected values dim 3 to be 2, got 3"):
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[[1, 2, 3], [4, 5, 6]],
[[7, 8, 9],
[10, 11, 12]]]]).astype(
np.float32)).run()
# After take grad, constraints on accumulated gradient are removed
sess.run(q.take_grad(1))
# First successful gradient imposes new constraints.
# Hereafter, shape will additionally constrained to [None,2,2,3]
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[[1, 2, 3], [4, 5, 6]],
[[7, 8, 9],
[10, 11,
12]]]]).astype(np.float32),
local_step=1).run()
with self.assertRaisesRegexp(
tf.errors.InvalidArgumentError,
"Shape mismatch: expected values dim 3 to be 3, got 2"):
q.apply_grad(grad_indices=[0],
grad_values=np.array([[[[1, 2], [3, 4]],
[[5, 6], [7, 8]]]
]).astype(np.float32),
local_step=1).run()
