def cpu_sum(s):
c = lambda i, s: tf.less(i, 10)
def b(i, s):
i1 = tf.add(i, 1)
with tf.device("/cpu:0"):
s1 = tf.add(i, s)
return i1, s1
_, r_s = control_flow_ops.While(c, b, [n, s])
return r_s
def _testWhile_Gpu_1(self, use_gpu):
with self.test_session(use_gpu=use_gpu):
n = tf.constant(1.0)
c = lambda x: tf.less(x, 10.0)
b = lambda x: tf.add(x, 1.0)
r = control_flow_ops.While(c, b, [n])
result = r.eval()
self.assertEqual(10.0, result)
def testWhile_1(self):
with self.test_session():
n = tf.constant(0)
c = lambda x: tf.less(x, 10000)
b = lambda x: tf.add(x, 1)
r = control_flow_ops.While(c, b, [n], parallel_iterations=20)
result = r.eval()
self.assertTrue(check_op_order(n.graph))
self.assertEqual(10000, result)
def testWhileGrad_1(self):
with self.test_session():
v = tf.constant(2.0, name="v")
c = lambda v: tf.less(v, 100.0)
b = tf.square
r = control_flow_ops.While(c, b, [v], parallel_iterations=1)
r = tf.gradients(r, v)
result = r[0].eval()
self.assertEqual(1024.0, result)
def testWhileCond_2(self):
with self.test_session():
n = tf.convert_to_tensor(0, name="n")
c = lambda x: tf.less(x, 10)
b = lambda x: control_flow_ops.cond(tf.constant(True), lambda: tf.
add(x, 1), lambda: n)
r = control_flow_ops.While(c, b, [n])
result = r.eval()
self.assertTrue(check_op_order(n.graph))
self.assertAllEqual(10, result)
def testWhileGrad_2(self):
with self.test_session():
a = tf.constant(3.0, name="a")
v = tf.constant(2.0, name="v")
c = lambda v: tf.less(v, 100.0)
b = lambda v: tf.mul(v, a)
r = control_flow_ops.While(c, b, [v], parallel_iterations=1)
r = tf.gradients(r, a)
result = r[0].eval()
self.assertEqual(216.0, result)
def testCondWhile_2(self):
with self.test_session():
n = tf.convert_to_tensor(0)
c = lambda x: tf.less(x, 10)
b = lambda x: tf.add(x, 1)
r = control_flow_ops.cond(tf.less(1, 0), lambda: tf.add(n, 1),
lambda: control_flow_ops.While(c, b, [n]))
result = r.eval()
self.assertTrue(check_op_order(n.graph))
self.assertAllEqual(10, result)
def _testWhile_Gpu_2(self, use_gpu):
with self.test_session(use_gpu=use_gpu):
n = tf.constant(1.0)
c = lambda x: tf.less(x, 10.0)
def b(x):
with tf.device("/cpu:0"):
return tf.add(x, 1.0)
r = control_flow_ops.While(c, b, [n])
result = r.eval()
self.assertEqual(10.0, result)
def make_heavy_op(myqueue, name):
"""Make an op that loops many times. This op reads off an int32 from "myqueue"
and increments counter that many times. The return value is a Print op that
final value of the counter."""
looplimit = myqueue.dequeue()
startval = tf.constant(0)
condition = lambda i: tf.less(i, looplimit)
increment = lambda i: tf.add(i, 1)
result = control_flow_ops.While(condition, increment, [startval])
resultprint = tf.Print(result, [name])
return resultprint
def gibbs_sample(k):
def gibbs_step(count, k, xk):
hk = sample(tf.sigmoid(tf.matmul(xk, W) + bh))
xk = sample(tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv))
return count+1, k, xk
ct = tf.constant(0)
[_, _, x_sample] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter,
gibbs_step, [ct, tf.constant(k), x], 1, False)
x_sample = tf.stop_gradient(x_sample)
return x_sample
def testWhileCond_1(self):
with self.test_session():
i = tf.convert_to_tensor(0, name="i")
n = tf.convert_to_tensor(10, name="n")
one = tf.convert_to_tensor(1, name="one")
c = lambda x: tf.less(x, n)
b = lambda x: control_flow_ops.cond(tf.constant(
True), lambda: tf.add(x, one), lambda: tf.sub(x, one))
r = control_flow_ops.While(c, b, [i])
result = r.eval()
self.assertTrue(check_op_order(n.graph))
self.assertAllEqual(10, result)
def generate(num, x=x, size_bt=size_bt, u0=u0, prime_with_x=False, n_visible=n_visible, prime_length=100):
m = tf.zeros([1, n_visible], tf.float32)
ct = tf.constant(1, tf.int32) #counter
if prime_with_x:
Uarr = tf.scan(rnn_recurrence, x, initializer=u0)
U = Uarr[prime_length, :, :]
else:
U = u0
# x = tf.slice(x, [100, 0], [num, n_visible])
[_, _, _, _, _, music] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter,
generate_recurrence,
[ct, tf.constant(num), U, tf.zeros([1, n_visible], tf.float32), x, m])
return music
def testWhileStack_1(self):
with self.test_session():
s = gen_data_flow_ops._stack(tf.int32, stack_name="foo")
i = tf.constant(0)
def c(i):
return tf.less(i, 10)
def b(i):
ni = tf.add(i, 1)
ni = control_flow_ops.with_dependencies(
[gen_data_flow_ops._stack_push(s, i)], ni)
return ni
r = control_flow_ops.While(c, b, [i], parallel_iterations=1)
x = tf.constant(0)
def c1(i, _):
return tf.greater(i, 0)
def b1(i, x):
ni = tf.sub(i, 1)
nx = x + gen_data_flow_ops._stack_pop(s, tf.int32)
return [ni, nx]
_, rx = control_flow_ops.While(c1, b1, [r, x], parallel_iterations=1)
self.assertEqual(45, rx.eval())
def testWhileWithControl_2(self):
with self.test_session():
r = tf.constant(0)
condition = lambda r_: tf.less(r_, 10)
def body(r_):
with r_.graph.control_dependencies([r_]):
r_ = tf.constant(12)
return [r_]
res = control_flow_ops.While(condition, body, [r], parallel_iterations=1)
result = res.eval()
self.assertTrue(check_op_order(r.graph))
self.assertAllEqual(12, result)
def gibbs_sample(k):
#Runs a k-step gibbs chain to sample from the probability distribution of the RBM defined by W, bh, bv
def gibbs_step(count, k, xk):
#Runs a single gibbs step. The visible values are initialized to xk
hk = sample(tf.sigmoid(tf.matmul(xk, W) + bh)) #Propagate the visible values to sample the hidden values
xk = sample(tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv)) #Propagate the hidden values to sample the visible values
return count+1, k, xk
#Run gibbs steps for k iterations
ct = tf.constant(0) #counter
[_, _, x_sample] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter,
gibbs_step, [ct, tf.constant(k), x], 1, False)
#This is not strictly necessary in this implementation, but if you want to adapt this code to use one of TensorFlow's
#optimizers, you need this in order to stop tensorflow from propagating gradients back through the gibbs step
x_sample = tf.stop_gradient(x_sample)
return x_sample
def testWhileGrad_6(self):
with self.test_session():
i = tf.constant(0, name="i")
x = tf.constant(2.0, name="x")
c = lambda i, x: tf.less(i, 10)
def b(i, x):
x = tf.mul(x, 2.0)
i = tf.add(i, 1)
return i, x
r = control_flow_ops.While(c, b, [i, x], parallel_iterations=1)
# Must use the complete r.
r = tf.gradients(r, x)
r = r[0].eval()
self.assertEqual(1024.0, r)
def testWhileGrad_5(self):
with self.test_session():
x = tf.constant(3.0, name="x")
y = tf.constant(2.0, name="y")
c = lambda x, y: tf.less(x, 100.0)
def b(x, y):
y1 = tf.add(x, y)
x1 = tf.mul(x, y1)
return x1, y1
r = control_flow_ops.While(c, b, [x, y], parallel_iterations=1)
# Must use the complete r.
r = tf.gradients(r, x)
result = r[0].eval()
self.assertEqual(304.0, result)
def testWhileQueue_1(self):
with self.test_session():
q = tf.FIFOQueue(-1, tf.int32)
i = tf.constant(0)
def c(i):
return tf.less(i, 10)
def b(i):
ni = tf.add(i, 1)
ni = control_flow_ops.with_dependencies([q.enqueue((i,))], ni)
return ni
r = control_flow_ops.While(c, b, [i], parallel_iterations=1)
self.assertEqual([10], r.eval())
for i in xrange(10):
self.assertEqual([i], q.dequeue().eval())
def _testWhileNested_1(self, use_gpu):
with self.test_session(use_gpu=use_gpu):
n = tf.constant(0)
def cpu_sum(s):
c = lambda i, s: tf.less(i, 10)
def b(i, s):
i1 = tf.add(i, 1)
with tf.device("/cpu:0"):
s1 = tf.add(i, s)
return i1, s1
_, r_s = control_flow_ops.While(c, b, [n, s])
return r_s
c = lambda x: tf.less(x, 200)
b = lambda x: tf.add(x, cpu_sum(n))
r = control_flow_ops.While(c, b, [n])
result = r.eval()
self.assertEqual(225, result)
def testIndexedSlicesGradient(self):
with ops.Graph().as_default():
embedding_matrix = tf.get_variable(
"embedding_matrix", [5, 5],
initializer=tf.random_normal_initializer())
def Cond(it, _):
return it < 5
def Body(it, cost):
embedding = embedding_ops.embedding_lookup(embedding_matrix + 0.0, [0])
cost += tf.reduce_sum(embedding)
return it + 1, cost
_, cost = control_flow_ops.While(
Cond, Body, [tf.constant(0), tf.constant(0.0)])
optimizer = momentum.MomentumOptimizer(0.1, 0.9)
train_op = optimizer.minimize(cost)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
for _ in range(10):
sess.run([train_op])
def testWhile_5(self):
with self.test_session():
def compute(i, c, o):
c = tf.slice(x, tf.expand_dims(i, 0), [1])
o = tf.concat(0, [o, c])
i = tf.add(i, 1)
return [i, c, o]
i = tf.convert_to_tensor(0)
c = tf.convert_to_tensor(0)
o = tf.convert_to_tensor([0])
x = tf.convert_to_tensor([1, 2, 3, 4, 5, 6])
s = tf.size(x)
r = control_flow_ops.While(
lambda i, c, o: tf.less(i, s), compute, [i, c, o])
result = r[2].eval()
self.assertTrue(check_op_order(i.graph))
self.assertAllEqual(np.array([0, 1, 2, 3, 4, 5, 6]), result)
def testWhileUpdateVariable_3(self):
with self.test_session():
select = tf.Variable([3.0, 4.0, 5.0])
n = tf.constant(0)
def loop_iterator(j, _):
return tf.less(j, 3)
def loop_body(j, _):
ns = tf.scatter_update(select, j, 10.0)
nj = tf.add(j, 1)
return [nj, ns]
r = control_flow_ops.While(loop_iterator,
loop_body, [n, tf.identity(select)],
parallel_iterations=1)
tf.initialize_all_variables().run()
result = r[1].eval()
self.assertTrue(check_op_order(n.graph))
self.assertAllEqual(np.array([10.0, 10.0, 10.0]), result)
def testWhile_3(self):
with self.test_session():
def compute(i, m, c, o):
m, c = [tf.add(m, 1), tf.add(c, 1)]
o = tf.add(o, m)
o = tf.add(o, c)
i = tf.add(i, 1)
return [i, m, c, o]
i = tf.convert_to_tensor(0)
m = tf.convert_to_tensor(0)
c = tf.convert_to_tensor(0)
o = tf.convert_to_tensor(0)
d = tf.convert_to_tensor(100)
r = control_flow_ops.While(
lambda i, m, c, o: tf.less(i, d), compute, [i, m, c, o])
result = r[3].eval()
self.assertTrue(check_op_order(i.graph))
self.assertAllEqual(10100, result)
def testParsingReaderOpWhileLoop(self):
feature_size = 3
batch_size = 5
def ParserEndpoints():
return gen_parser_ops.gold_parse_reader(
self._task_context,
feature_size,
batch_size,
corpus_name='training-corpus')
with self.test_session() as sess:
# The 'condition' and 'body' functions expect as many arguments as there
# are loop variables. 'condition' depends on the 'epoch' loop variable
# only, so we disregard the remaining unused function arguments. 'body'
# returns a list of updated loop variables.
def Condition(epoch, *unused_args):
return tf.less(epoch, 2)
def Body(epoch, num_actions, *feature_args):
# By adding one of the outputs of the reader op ('epoch') as a control
# dependency to the reader op we force the repeated evaluation of the
# reader op.
with epoch.graph.control_dependencies([epoch]):
features, epoch, gold_actions = ParserEndpoints()
num_actions = tf.maximum(
num_actions,
tf.reduce_max(gold_actions, [0], False) + 1)
feature_ids = []
for i in range(len(feature_args)):
feature_ids.append(features[i])
return [epoch, num_actions] + feature_ids
epoch = ParserEndpoints()[-2]
num_actions = tf.constant(0)
loop_vars = [epoch, num_actions]
res = sess.run(
cf.While(Condition, Body, loop_vars, parallel_iterations=1))
logging.info('Result: %s', res)
self.assertEqual(res[0], 2)
def generate(num, x=x, size_bt=size_bt, u0=u0, n_visible=n_visible, prime_length=100):
"""
This function handles generating music. This function is one of the outputs of the build_rnnrbm function
Args:
num (int): The number of timesteps to generate
x (tf.placeholder): The data vector. We can use feed_dict to set this to the music primer.
size_bt (tf.float32): The batch size
u0 (tf.Variable): The initial state of the RNN
n_visible (int): The size of the data vectors
prime_length (int): The number of timesteps into the primer song that we use befoe beginning to generate music
Returns:
The generated music, as a tf.Tensor
"""
Uarr = tf.scan(rnn_recurrence, x, initializer=u0)
U = Uarr[np.floor(prime_length/midi_manipulation.num_timesteps), :, :]
[_, _, _, _, _, music] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter,
generate_recurrence, [tf.constant(1, tf.int32), tf.constant(num), U,
tf.zeros([1, n_visible], tf.float32), x,
tf.zeros([1, n_visible], tf.float32)])
return music
def testWhile_4(self):
with self.test_session():
def compute(i, m, c, o):
m, c = [tf.gather(x, i), tf.gather(x, i)]
o = tf.add(o, m)
o = tf.add(o, c)
i = tf.add(i, 1)
return [i, m, c, o]
i = tf.convert_to_tensor(0)
m = tf.convert_to_tensor(0)
c = tf.convert_to_tensor(0)
o = tf.convert_to_tensor(0)
x = tf.convert_to_tensor([1, 2, 3, 4, 5, 6])
s = tf.size(x)
r = control_flow_ops.While(
lambda i, m, c, o: tf.less(i, s), compute, [i, m, c, o])
result = r[3].eval()
self.assertTrue(check_op_order(i.graph))
self.assertAllEqual(42, result)
def gibbs_sample_generate(k):
"""
Runs k-step gibbs chain to sample from the probability distribution of the RBM defined by W, bh, bv.
:params k: number of gibbs step iterations to run
:type k: int
:returns: matrix of music sampled
:rtype: tensor
"""
def gibbs_step_generate(count, k, xk):
"""
Runs a single gibbs step
:param count: number of iterations done
:param k: total number of gibbs step iterations to run
:param xk: the visible values are initialized to xk.
:type count: int
:type k: int
:type xk: tensor
"""
hk = sample(tf.sigmoid(
tf.matmul(xk, W) +
bh)) # Propagate the visible values to sample the hidden values
tv = tf.matmul(hk, tf.transpose(
W)) + bv # Propagate the hidden values to the visible values
xk = sample_correct(
tf.sigmoid(tv),
emotions) # sample the visible values and clamp emotion values
return count + 1, k, xk
x_sample = x
# run gibbs steps for k iterations
ct = tf.constant(0) # initialize counter to 0
[_, _, x_sample] = control_flow_ops.While(
lambda count, num_iter, *args: count < num_iter, gibbs_step_generate,
[ct, tf.constant(k), x_sample], 1, False)
# This is not strictly necessary in this implementation, but if you want to adapt this code to use one of TensorFlow's
# optimizers, you need this in order to stop tensorflow from propagating gradients back through the gibbs step
x_sample = tf.stop_gradient(x_sample)
return x_sample
def testWhileUpdateVariable_1(self):
with self.test_session():
select = tf.Variable([3.0, 4.0, 5.0])
n = tf.constant(0)
def loop_iterator(j):
return tf.less(j, 3)
def loop_body(j):
ns = tf.scatter_update(select, j, 10.0)
nj = tf.add(j, 1)
op = control_flow_ops.group(ns)
nj = control_flow_ops.with_dependencies([op], nj)
return [nj]
r = control_flow_ops.While(loop_iterator,
loop_body, [n],
parallel_iterations=1)
self.assertTrue(check_op_order(n.graph))
tf.initialize_all_variables().run()
self.assertEqual(3, r.eval())
result = select.eval()
self.assertAllEqual(np.array([10.0, 10.0, 10.0]), result)
def gibbs_sample(x, W, bv, bh, k, binary=True, c_1=0.5, c_2=0.5):
size_bt = tf.shape(x)[0]
# CD-k
# we use tf.while_loop to achieve the multiple (k - 1) gibbs sampling
def gibbs_step(count, k, xk, raw_xk, W=W):
hk = gibbs_forward(xk, W, bh, binary)
C1 = tf.cond(count + 1 >= k, lambda: tf.constant(c_1),
lambda: tf.constant(0.5))
C2 = tf.cond(count + 1 >= k, lambda: tf.constant(c_2),
lambda: tf.constant(0.5))
raw_xk = tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv)
xk = sample(raw_xk, binary=binary, c_1=c_1, c_2=c_2)
return count + 1, k, xk, raw_xk
ct = tf.constant(0) #counter
h = sample(tf.sigmoid(tf.matmul(x, W) + bh), binary=binary)
[_, _, xk1, raw_xk] = control_flow_ops.While(
lambda count, num_iter, *args: count < num_iter, gibbs_step,
[ct, tf.constant(k), x, x], 1, False)
xk1 = tf.stop_gradient(xk1)
raw_xk = tf.stop_gradient(raw_xk)
return xk1, raw_xk
def testWhileUpdateVariable_4(self):
with self.test_session():
var_a = tf.Variable(0, name="a")
var_b = tf.Variable(0, name="b")
tf.initialize_all_variables().run()
c = tf.constant(0, name="c")
asn1 = tf.assign_add(var_a, 1, name="a_add")
# Loop condition
def pred(i):
return tf.less(i, 10)
# Loop body
def loop_body(i):
asn2 = tf.assign_add(var_b, asn1, name="b_add")
with tf.control_dependencies([asn2]):
ni = tf.add(i, 1, name="i_add")
return ni
lpa = control_flow_ops.While(pred, loop_body, [c],
parallel_iterations=1)
self.assertEqual(0, var_b.eval())
lpa.eval() # Run the loop
self.assertEqual(10, var_b.eval())
