def cpu_sum(s): c = lambda i, s: tf.less(i, 10) def b(i, s): i1 = tf.add(i, 1) with tf.device("/cpu:0"): s1 = tf.add(i, s) return i1, s1 _, r_s = control_flow_ops.While(c, b, [n, s]) return r_s
def _testWhile_Gpu_1(self, use_gpu): with self.test_session(use_gpu=use_gpu): n = tf.constant(1.0) c = lambda x: tf.less(x, 10.0) b = lambda x: tf.add(x, 1.0) r = control_flow_ops.While(c, b, [n]) result = r.eval() self.assertEqual(10.0, result)
def testWhile_1(self): with self.test_session(): n = tf.constant(0) c = lambda x: tf.less(x, 10000) b = lambda x: tf.add(x, 1) r = control_flow_ops.While(c, b, [n], parallel_iterations=20) result = r.eval() self.assertTrue(check_op_order(n.graph)) self.assertEqual(10000, result)
def testWhileGrad_1(self): with self.test_session(): v = tf.constant(2.0, name="v") c = lambda v: tf.less(v, 100.0) b = tf.square r = control_flow_ops.While(c, b, [v], parallel_iterations=1) r = tf.gradients(r, v) result = r[0].eval() self.assertEqual(1024.0, result)
def testWhileCond_2(self): with self.test_session(): n = tf.convert_to_tensor(0, name="n") c = lambda x: tf.less(x, 10) b = lambda x: control_flow_ops.cond(tf.constant(True), lambda: tf. add(x, 1), lambda: n) r = control_flow_ops.While(c, b, [n]) result = r.eval() self.assertTrue(check_op_order(n.graph)) self.assertAllEqual(10, result)
def testWhileGrad_2(self): with self.test_session(): a = tf.constant(3.0, name="a") v = tf.constant(2.0, name="v") c = lambda v: tf.less(v, 100.0) b = lambda v: tf.mul(v, a) r = control_flow_ops.While(c, b, [v], parallel_iterations=1) r = tf.gradients(r, a) result = r[0].eval() self.assertEqual(216.0, result)
def testCondWhile_2(self): with self.test_session(): n = tf.convert_to_tensor(0) c = lambda x: tf.less(x, 10) b = lambda x: tf.add(x, 1) r = control_flow_ops.cond(tf.less(1, 0), lambda: tf.add(n, 1), lambda: control_flow_ops.While(c, b, [n])) result = r.eval() self.assertTrue(check_op_order(n.graph)) self.assertAllEqual(10, result)
def _testWhile_Gpu_2(self, use_gpu): with self.test_session(use_gpu=use_gpu): n = tf.constant(1.0) c = lambda x: tf.less(x, 10.0) def b(x): with tf.device("/cpu:0"): return tf.add(x, 1.0) r = control_flow_ops.While(c, b, [n]) result = r.eval() self.assertEqual(10.0, result)
def make_heavy_op(myqueue, name): """Make an op that loops many times. This op reads off an int32 from "myqueue" and increments counter that many times. The return value is a Print op that final value of the counter.""" looplimit = myqueue.dequeue() startval = tf.constant(0) condition = lambda i: tf.less(i, looplimit) increment = lambda i: tf.add(i, 1) result = control_flow_ops.While(condition, increment, [startval]) resultprint = tf.Print(result, [name]) return resultprint
def gibbs_sample(k): def gibbs_step(count, k, xk): hk = sample(tf.sigmoid(tf.matmul(xk, W) + bh)) xk = sample(tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv)) return count+1, k, xk ct = tf.constant(0) [_, _, x_sample] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter, gibbs_step, [ct, tf.constant(k), x], 1, False) x_sample = tf.stop_gradient(x_sample) return x_sample
def testWhileCond_1(self): with self.test_session(): i = tf.convert_to_tensor(0, name="i") n = tf.convert_to_tensor(10, name="n") one = tf.convert_to_tensor(1, name="one") c = lambda x: tf.less(x, n) b = lambda x: control_flow_ops.cond(tf.constant( True), lambda: tf.add(x, one), lambda: tf.sub(x, one)) r = control_flow_ops.While(c, b, [i]) result = r.eval() self.assertTrue(check_op_order(n.graph)) self.assertAllEqual(10, result)
def generate(num, x=x, size_bt=size_bt, u0=u0, prime_with_x=False, n_visible=n_visible, prime_length=100): m = tf.zeros([1, n_visible], tf.float32) ct = tf.constant(1, tf.int32) #counter if prime_with_x: Uarr = tf.scan(rnn_recurrence, x, initializer=u0) U = Uarr[prime_length, :, :] else: U = u0 # x = tf.slice(x, [100, 0], [num, n_visible]) [_, _, _, _, _, music] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter, generate_recurrence, [ct, tf.constant(num), U, tf.zeros([1, n_visible], tf.float32), x, m]) return music
def testWhileStack_1(self): with self.test_session(): s = gen_data_flow_ops._stack(tf.int32, stack_name="foo") i = tf.constant(0) def c(i): return tf.less(i, 10) def b(i): ni = tf.add(i, 1) ni = control_flow_ops.with_dependencies( [gen_data_flow_ops._stack_push(s, i)], ni) return ni r = control_flow_ops.While(c, b, [i], parallel_iterations=1) x = tf.constant(0) def c1(i, _): return tf.greater(i, 0) def b1(i, x): ni = tf.sub(i, 1) nx = x + gen_data_flow_ops._stack_pop(s, tf.int32) return [ni, nx] _, rx = control_flow_ops.While(c1, b1, [r, x], parallel_iterations=1) self.assertEqual(45, rx.eval())
def testWhileWithControl_2(self): with self.test_session(): r = tf.constant(0) condition = lambda r_: tf.less(r_, 10) def body(r_): with r_.graph.control_dependencies([r_]): r_ = tf.constant(12) return [r_] res = control_flow_ops.While(condition, body, [r], parallel_iterations=1) result = res.eval() self.assertTrue(check_op_order(r.graph)) self.assertAllEqual(12, result)
def gibbs_sample(k): #Runs a k-step gibbs chain to sample from the probability distribution of the RBM defined by W, bh, bv def gibbs_step(count, k, xk): #Runs a single gibbs step. The visible values are initialized to xk hk = sample(tf.sigmoid(tf.matmul(xk, W) + bh)) #Propagate the visible values to sample the hidden values xk = sample(tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv)) #Propagate the hidden values to sample the visible values return count+1, k, xk #Run gibbs steps for k iterations ct = tf.constant(0) #counter [_, _, x_sample] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter, gibbs_step, [ct, tf.constant(k), x], 1, False) #This is not strictly necessary in this implementation, but if you want to adapt this code to use one of TensorFlow's #optimizers, you need this in order to stop tensorflow from propagating gradients back through the gibbs step x_sample = tf.stop_gradient(x_sample) return x_sample
def testWhileGrad_6(self): with self.test_session(): i = tf.constant(0, name="i") x = tf.constant(2.0, name="x") c = lambda i, x: tf.less(i, 10) def b(i, x): x = tf.mul(x, 2.0) i = tf.add(i, 1) return i, x r = control_flow_ops.While(c, b, [i, x], parallel_iterations=1) # Must use the complete r. r = tf.gradients(r, x) r = r[0].eval() self.assertEqual(1024.0, r)
def testWhileGrad_5(self): with self.test_session(): x = tf.constant(3.0, name="x") y = tf.constant(2.0, name="y") c = lambda x, y: tf.less(x, 100.0) def b(x, y): y1 = tf.add(x, y) x1 = tf.mul(x, y1) return x1, y1 r = control_flow_ops.While(c, b, [x, y], parallel_iterations=1) # Must use the complete r. r = tf.gradients(r, x) result = r[0].eval() self.assertEqual(304.0, result)
def testWhileQueue_1(self): with self.test_session(): q = tf.FIFOQueue(-1, tf.int32) i = tf.constant(0) def c(i): return tf.less(i, 10) def b(i): ni = tf.add(i, 1) ni = control_flow_ops.with_dependencies([q.enqueue((i,))], ni) return ni r = control_flow_ops.While(c, b, [i], parallel_iterations=1) self.assertEqual([10], r.eval()) for i in xrange(10): self.assertEqual([i], q.dequeue().eval())
def _testWhileNested_1(self, use_gpu): with self.test_session(use_gpu=use_gpu): n = tf.constant(0) def cpu_sum(s): c = lambda i, s: tf.less(i, 10) def b(i, s): i1 = tf.add(i, 1) with tf.device("/cpu:0"): s1 = tf.add(i, s) return i1, s1 _, r_s = control_flow_ops.While(c, b, [n, s]) return r_s c = lambda x: tf.less(x, 200) b = lambda x: tf.add(x, cpu_sum(n)) r = control_flow_ops.While(c, b, [n]) result = r.eval() self.assertEqual(225, result)
def testIndexedSlicesGradient(self): with ops.Graph().as_default(): embedding_matrix = tf.get_variable( "embedding_matrix", [5, 5], initializer=tf.random_normal_initializer()) def Cond(it, _): return it < 5 def Body(it, cost): embedding = embedding_ops.embedding_lookup(embedding_matrix + 0.0, [0]) cost += tf.reduce_sum(embedding) return it + 1, cost _, cost = control_flow_ops.While( Cond, Body, [tf.constant(0), tf.constant(0.0)]) optimizer = momentum.MomentumOptimizer(0.1, 0.9) train_op = optimizer.minimize(cost) with self.test_session() as sess: sess.run(tf.initialize_all_variables()) for _ in range(10): sess.run([train_op])
def testWhile_5(self): with self.test_session(): def compute(i, c, o): c = tf.slice(x, tf.expand_dims(i, 0), [1]) o = tf.concat(0, [o, c]) i = tf.add(i, 1) return [i, c, o] i = tf.convert_to_tensor(0) c = tf.convert_to_tensor(0) o = tf.convert_to_tensor([0]) x = tf.convert_to_tensor([1, 2, 3, 4, 5, 6]) s = tf.size(x) r = control_flow_ops.While( lambda i, c, o: tf.less(i, s), compute, [i, c, o]) result = r[2].eval() self.assertTrue(check_op_order(i.graph)) self.assertAllEqual(np.array([0, 1, 2, 3, 4, 5, 6]), result)
def testWhileUpdateVariable_3(self): with self.test_session(): select = tf.Variable([3.0, 4.0, 5.0]) n = tf.constant(0) def loop_iterator(j, _): return tf.less(j, 3) def loop_body(j, _): ns = tf.scatter_update(select, j, 10.0) nj = tf.add(j, 1) return [nj, ns] r = control_flow_ops.While(loop_iterator, loop_body, [n, tf.identity(select)], parallel_iterations=1) tf.initialize_all_variables().run() result = r[1].eval() self.assertTrue(check_op_order(n.graph)) self.assertAllEqual(np.array([10.0, 10.0, 10.0]), result)
def testWhile_3(self): with self.test_session(): def compute(i, m, c, o): m, c = [tf.add(m, 1), tf.add(c, 1)] o = tf.add(o, m) o = tf.add(o, c) i = tf.add(i, 1) return [i, m, c, o] i = tf.convert_to_tensor(0) m = tf.convert_to_tensor(0) c = tf.convert_to_tensor(0) o = tf.convert_to_tensor(0) d = tf.convert_to_tensor(100) r = control_flow_ops.While( lambda i, m, c, o: tf.less(i, d), compute, [i, m, c, o]) result = r[3].eval() self.assertTrue(check_op_order(i.graph)) self.assertAllEqual(10100, result)
def testParsingReaderOpWhileLoop(self): feature_size = 3 batch_size = 5 def ParserEndpoints(): return gen_parser_ops.gold_parse_reader( self._task_context, feature_size, batch_size, corpus_name='training-corpus') with self.test_session() as sess: # The 'condition' and 'body' functions expect as many arguments as there # are loop variables. 'condition' depends on the 'epoch' loop variable # only, so we disregard the remaining unused function arguments. 'body' # returns a list of updated loop variables. def Condition(epoch, *unused_args): return tf.less(epoch, 2) def Body(epoch, num_actions, *feature_args): # By adding one of the outputs of the reader op ('epoch') as a control # dependency to the reader op we force the repeated evaluation of the # reader op. with epoch.graph.control_dependencies([epoch]): features, epoch, gold_actions = ParserEndpoints() num_actions = tf.maximum( num_actions, tf.reduce_max(gold_actions, [0], False) + 1) feature_ids = [] for i in range(len(feature_args)): feature_ids.append(features[i]) return [epoch, num_actions] + feature_ids epoch = ParserEndpoints()[-2] num_actions = tf.constant(0) loop_vars = [epoch, num_actions] res = sess.run( cf.While(Condition, Body, loop_vars, parallel_iterations=1)) logging.info('Result: %s', res) self.assertEqual(res[0], 2)
def generate(num, x=x, size_bt=size_bt, u0=u0, n_visible=n_visible, prime_length=100): """ This function handles generating music. This function is one of the outputs of the build_rnnrbm function Args: num (int): The number of timesteps to generate x (tf.placeholder): The data vector. We can use feed_dict to set this to the music primer. size_bt (tf.float32): The batch size u0 (tf.Variable): The initial state of the RNN n_visible (int): The size of the data vectors prime_length (int): The number of timesteps into the primer song that we use befoe beginning to generate music Returns: The generated music, as a tf.Tensor """ Uarr = tf.scan(rnn_recurrence, x, initializer=u0) U = Uarr[np.floor(prime_length/midi_manipulation.num_timesteps), :, :] [_, _, _, _, _, music] = control_flow_ops.While(lambda count, num_iter, *args: count < num_iter, generate_recurrence, [tf.constant(1, tf.int32), tf.constant(num), U, tf.zeros([1, n_visible], tf.float32), x, tf.zeros([1, n_visible], tf.float32)]) return music
def testWhile_4(self): with self.test_session(): def compute(i, m, c, o): m, c = [tf.gather(x, i), tf.gather(x, i)] o = tf.add(o, m) o = tf.add(o, c) i = tf.add(i, 1) return [i, m, c, o] i = tf.convert_to_tensor(0) m = tf.convert_to_tensor(0) c = tf.convert_to_tensor(0) o = tf.convert_to_tensor(0) x = tf.convert_to_tensor([1, 2, 3, 4, 5, 6]) s = tf.size(x) r = control_flow_ops.While( lambda i, m, c, o: tf.less(i, s), compute, [i, m, c, o]) result = r[3].eval() self.assertTrue(check_op_order(i.graph)) self.assertAllEqual(42, result)
def gibbs_sample_generate(k): """ Runs k-step gibbs chain to sample from the probability distribution of the RBM defined by W, bh, bv. :params k: number of gibbs step iterations to run :type k: int :returns: matrix of music sampled :rtype: tensor """ def gibbs_step_generate(count, k, xk): """ Runs a single gibbs step :param count: number of iterations done :param k: total number of gibbs step iterations to run :param xk: the visible values are initialized to xk. :type count: int :type k: int :type xk: tensor """ hk = sample(tf.sigmoid( tf.matmul(xk, W) + bh)) # Propagate the visible values to sample the hidden values tv = tf.matmul(hk, tf.transpose( W)) + bv # Propagate the hidden values to the visible values xk = sample_correct( tf.sigmoid(tv), emotions) # sample the visible values and clamp emotion values return count + 1, k, xk x_sample = x # run gibbs steps for k iterations ct = tf.constant(0) # initialize counter to 0 [_, _, x_sample] = control_flow_ops.While( lambda count, num_iter, *args: count < num_iter, gibbs_step_generate, [ct, tf.constant(k), x_sample], 1, False) # This is not strictly necessary in this implementation, but if you want to adapt this code to use one of TensorFlow's # optimizers, you need this in order to stop tensorflow from propagating gradients back through the gibbs step x_sample = tf.stop_gradient(x_sample) return x_sample
def testWhileUpdateVariable_1(self): with self.test_session(): select = tf.Variable([3.0, 4.0, 5.0]) n = tf.constant(0) def loop_iterator(j): return tf.less(j, 3) def loop_body(j): ns = tf.scatter_update(select, j, 10.0) nj = tf.add(j, 1) op = control_flow_ops.group(ns) nj = control_flow_ops.with_dependencies([op], nj) return [nj] r = control_flow_ops.While(loop_iterator, loop_body, [n], parallel_iterations=1) self.assertTrue(check_op_order(n.graph)) tf.initialize_all_variables().run() self.assertEqual(3, r.eval()) result = select.eval() self.assertAllEqual(np.array([10.0, 10.0, 10.0]), result)
def gibbs_sample(x, W, bv, bh, k, binary=True, c_1=0.5, c_2=0.5): size_bt = tf.shape(x)[0] # CD-k # we use tf.while_loop to achieve the multiple (k - 1) gibbs sampling def gibbs_step(count, k, xk, raw_xk, W=W): hk = gibbs_forward(xk, W, bh, binary) C1 = tf.cond(count + 1 >= k, lambda: tf.constant(c_1), lambda: tf.constant(0.5)) C2 = tf.cond(count + 1 >= k, lambda: tf.constant(c_2), lambda: tf.constant(0.5)) raw_xk = tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv) xk = sample(raw_xk, binary=binary, c_1=c_1, c_2=c_2) return count + 1, k, xk, raw_xk ct = tf.constant(0) #counter h = sample(tf.sigmoid(tf.matmul(x, W) + bh), binary=binary) [_, _, xk1, raw_xk] = control_flow_ops.While( lambda count, num_iter, *args: count < num_iter, gibbs_step, [ct, tf.constant(k), x, x], 1, False) xk1 = tf.stop_gradient(xk1) raw_xk = tf.stop_gradient(raw_xk) return xk1, raw_xk
def testWhileUpdateVariable_4(self): with self.test_session(): var_a = tf.Variable(0, name="a") var_b = tf.Variable(0, name="b") tf.initialize_all_variables().run() c = tf.constant(0, name="c") asn1 = tf.assign_add(var_a, 1, name="a_add") # Loop condition def pred(i): return tf.less(i, 10) # Loop body def loop_body(i): asn2 = tf.assign_add(var_b, asn1, name="b_add") with tf.control_dependencies([asn2]): ni = tf.add(i, 1, name="i_add") return ni lpa = control_flow_ops.While(pred, loop_body, [c], parallel_iterations=1) self.assertEqual(0, var_b.eval()) lpa.eval() # Run the loop self.assertEqual(10, var_b.eval())