def testDeclare(self): foo = function.Declare("Foo", [("x", dtypes.float32)], [("y", dtypes.float32)]) @function.Defun(dtypes.float32, func_name="Foo", out_names=["y"]) def FooImpl(x): return x * x + 1 x = array_ops.placeholder(dtypes.float32) y = foo(x) g = ops.get_default_graph() FooImpl.add_to_graph(g) with self.test_session(): rand = np.random.uniform(size=(3, 3)) expected = rand * rand + 1.0 self.assertAllClose(expected, y.eval(feed_dict={x: rand}))
def testDeclareTypeMistake(self): foo = function.Declare("Foo", [("x", dtypes.float32)], [("y", dtypes.float32)]) @function.Defun(dtypes.float32, func_name="Foo", out_names=["y"]) def Foo(x): return x * x + 1 g = ops.Graph() with g.as_default(): y = foo(2.0) with self.test_session(graph=g): with self.assertRaisesRegexp(errors_impl.NotFoundError, "not registered"): _ = y.eval() g = ops.Graph() with g.as_default(): Foo.add_to_graph(g) y = foo(2) with self.test_session(graph=g): with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, "int32.*float"): _ = y.eval() g = ops.Graph() with g.as_default(): Foo.add_to_graph(g) with self.assertRaisesRegexp( ValueError, "Expected number of arguments: 1, received: 2"): _ = foo(2.0, 2.0) g = ops.Graph() with g.as_default(): Foo.add_to_graph(g) y = foo(2.0) with self.test_session(graph=g): self.assertAllEqual(y.eval(), 5.0)
def testDeclareTypeMistake(self): foo = function.Declare("Foo", [tf.float32], [tf.float32]) @function.Defun(tf.float32) def Foo(x): return x * x + 1 g = tf.Graph() with g.as_default(): y = foo(2.0) with self.test_session(graph=g): with self.assertRaisesRegexp(tf.errors.NotFoundError, "not registered"): _ = y.eval() g = tf.Graph() with g.as_default(): Foo.add_to_graph(g) y = foo(2) with self.test_session(graph=g): with self.assertRaisesRegexp(tf.errors.InvalidArgumentError, "int32.*float"): _ = y.eval() g = tf.Graph() with g.as_default(): Foo.add_to_graph(g) with self.assertRaisesRegexp( ValueError, "Expected number of arguments: 1, received: 2"): _ = foo(2.0, 2.0) g = tf.Graph() with g.as_default(): Foo.add_to_graph(g) y = foo(2.0) with self.test_session(graph=g): self.assertAllEqual(y.eval(), 5.0)
def testDeclareTypeMistake(self): foo = function.Declare("Foo", [tf.float32], [tf.float32]) @function.Defun(tf.float32) def Foo(x): return x * x + 1 g = tf.Graph() with g.as_default(): y = foo(tf.constant(2, tf.float32)) with self.test_session(graph=g): with self.assertRaisesRegexp(tf.errors.NotFoundError, "not registered"): _ = y.eval() g = tf.Graph() with g.as_default(): Foo.add_to_graph(g) y = foo(tf.constant(2, tf.int32)) with self.test_session(graph=g): with self.assertRaisesRegexp(tf.errors.InvalidArgumentError, "int32.*float"): _ = y.eval() g = tf.Graph() with g.as_default(): Foo.add_to_graph(g) with self.assertRaisesRegexp(ValueError, "Mismatch number of args: 2 vs. 1"): _ = foo(tf.constant(2, tf.float32), tf.constant(2, tf.float32)) g = tf.Graph() with g.as_default(): Foo.add_to_graph(g) y = foo(tf.constant(2, tf.float32)) with self.test_session(graph=g): self.assertAllEqual(y.eval(), 5.0)
def __init__(self, config): self.config = config # Load train data and build vocabulary self.train_data, self.dev_data, self.test_data = tree.simplified_data( 700, 100, 200) # print("data ",self.train_data)) self.vocab = utils.Vocab() train_sents = [t.get_words() for t in self.train_data] self.vocab.construct(list(itertools.chain.from_iterable(train_sents))) # add input placeholders self.is_leaf_placeholder = tf.placeholder(tf.int32, (None), name='is_leaf_placeholder') self.node_word_indices_placeholder = tf.placeholder( tf.int32, (None), name='node_word_indices_placeholder') self.labels_placeholder = tf.placeholder(tf.int32, (None), name='labels_placeholder') self.cons_placeholder = tf.placeholder(tf.int32, (None), name='cons') # add model variables # making initialization deterministic for now # initializer = tf.random_normal_initializer(seed=1) with tf.variable_scope('Embeddings'): self.embeddings = tf.get_variable( 'embeddings', [len(self.vocab), self.config.embed_size]) with tf.variable_scope('Composition'): W1 = tf.get_variable( 'W1', [2 * self.config.embed_size, self.config.embed_size]) b1 = tf.get_variable('b1', [1, self.config.embed_size]) with tf.variable_scope('Projection'): U = tf.get_variable( 'U', [self.config.embed_size, self.config.label_size]) bs = tf.get_variable('bs', [1, self.config.label_size]) # Build recursive graph def embed_word(word_index, embeddings): return tf.expand_dims(tf.gather(embeddings, word_index), 0) def combine_children(left_tensor, right_tensor, W, b): return tf.nn.relu( tf.matmul(tf.concat([left_tensor, right_tensor], 1), W) + b) def find_loss(node_tensor, i, labels, U, bs): # add projection layer node_logits = tf.matmul(node_tensor, U) + bs loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=node_logits, labels=labels[i:i + 1]) return loss def base_case(node_word_indices, i, embeddings, labels, U, bs): word_index = tf.gather(node_word_indices, i) node_tensor = embed_word(word_index, embeddings) loss = find_loss(node_tensor, i, labels, U, bs) return [node_tensor, loss] def rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs): left_node, left_loss = rec(i * 2, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs) right_node, right_loss = rec(i * 2 + 1, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs) node_tensor = combine_children(left_node, right_node, W, b) node_loss = find_loss(node_tensor, i, labels, U, bs) loss = tf.concat([left_loss, node_loss, right_loss], 0) return [node_tensor, loss] # Function Declaration rec = function.Declare("Rec", [("i", tf.int32), ("is_leaf", tf.int32), ("node_word_indices", tf.int32), ("embeddings", tf.float32), ("W", tf.float32), ("b", tf.float32), ("labels", tf.int32), ("U", tf.float32), ("bs", tf.float32)], [("ret", tf.float32), ("ret1", tf.float32)]) # Function Definition @function.Defun(tf.int32, tf.int32, tf.int32, tf.float32, tf.float32, tf.float32, tf.int32, tf.float32, tf.float32, func_name="Rec", grad_func="GradFac", create_grad_func=True, out_names=["ret", "ret1"]) def RecImpl(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs): node_tensor, loss = \ tf.cond(tf.equal(tf.gather(is_leaf, i), tf.constant(1)), lambda: base_case(node_word_indices, i, embeddings, labels, U, bs), lambda: rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs)) return [node_tensor, loss] RecImpl.add_to_graph(tf.get_default_graph()) self.node_tensor, self.full_loss = rec( self.cons_placeholder, self.is_leaf_placeholder, self.node_word_indices_placeholder, self.embeddings, W1, b1, self.labels_placeholder, U, bs) # add projection layer self.root_logits = tf.matmul(self.node_tensor, U) + bs self.root_prediction = tf.squeeze(tf.argmax(self.root_logits, 1)) # add loss layer self.root_loss = tf.reduce_sum( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.root_logits, labels=self.labels_placeholder[1:2])) regularization_loss = self.config.l2 * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(U)) self.full_loss = regularization_loss + tf.reduce_sum(self.full_loss) # # add training op self.train_op = tf.train.AdamOptimizer(self.config.lr).minimize( self.full_loss)
def __init__(self, config): self.config = config # Load train data and build vocabulary self.train_data, self.dev_data, self.test_data = tree.simplified_data(700, 100, 200) # print("data ",self.train_data)) self.vocab = utils.Vocab() train_sents = [t.get_words() for t in self.train_data] self.vocab.construct(list(itertools.chain.from_iterable(train_sents))) # add input placeholders self.is_leaf_placeholder = tf.placeholder( tf.int32, (None), name='is_leaf_placeholder') self.node_word_indices_placeholder = tf.placeholder( tf.int32, (None), name='node_word_indices_placeholder') self.labels_placeholder = tf.placeholder( tf.int32, (None), name='labels_placeholder') self.cons_placeholder = tf.placeholder( tf.int32, (None), name='cons') # add model variables # making initialization deterministic for now initializer = tf.random_normal_initializer(seed=1) with tf.variable_scope('Embeddings'): self.embeddings = tf.get_variable('embeddings', [len(self.vocab), self.config.embed_size]) # , # initializer=initializer) #tf.constant_initializer(2.0)) with tf.variable_scope('Composition'): W1 = tf.get_variable('W1', [2 * self.config.embed_size, self.config.embed_size]) # , # initializer=initializer) #tf.constant_initializer(0.0)) b1 = tf.get_variable('b1', [1, self.config.embed_size]) # , # initializer=initializer) #tf.constant_initializer(0.0)) with tf.variable_scope('Projection'): U = tf.get_variable('U', [self.config.embed_size, self.config.label_size]) # , # initializer=initializer) #tf.constant_initializer(0.0)) bs = tf.get_variable('bs', [1, self.config.label_size]) # , # initializer=initializer) #tf.constant_initializer(0.0)) # Build recursive graph # tensor_array = tf.TensorArray( # tf.float32, # size=0, # dynamic_size=True, # clear_after_read=False, # infer_shape=False) # Build recursive graph def embed_word(word_index, embeddings): # with tf.device('/cpu:0'): return tf.expand_dims(tf.gather(embeddings, word_index), 0) def combine_children(left_tensor, right_tensor, W, b): return tf.nn.relu(tf.matmul(tf.concat([left_tensor, right_tensor], 1), W) + b) # Function Declaration rec = function.Declare("Rec", [("i", tf.int32), ("is_leaf", tf.int32), ("node_word_indices", tf.int32), ("embeddings", tf.float32), ("W", tf.float32),("b", tf.float32)], [("ret", tf.float32)]) # Function Definition @function.Defun(tf.int32, tf.int32, tf.int32, tf.float32, tf.float32, tf.float32, func_name="Rec", grad_func="GradFac", create_grad_func=True, out_names=["ret"]) def RecImpl(i, is_leaf, node_word_indices, embeddings, W, b): node_word_index = tf.gather(node_word_indices, i) node_tensor = \ tf.cond(tf.equal(tf.gather(is_leaf, i), tf.constant(1)), lambda: embed_word(node_word_index, embeddings), lambda: combine_children(rec(i*2, is_leaf, node_word_indices, embeddings, W, b), rec(i*2+1, is_leaf, node_word_indices, embeddings, W, b), W, b)) return node_tensor RecImpl.add_to_graph(tf.get_default_graph()) self.node_tensor = rec(self.cons_placeholder, self.is_leaf_placeholder, self.node_word_indices_placeholder, self.embeddings, W1, b1) # add projection layer # self.logits = tf.matmul(self.tensor_array.concat(), U) + bs # 1x35 * 35x35 + 1x35 -> 1x35 projection self.root_logits = tf.matmul(self.node_tensor, U) + bs self.root_prediction = tf.squeeze(tf.argmax(self.root_logits, 1)) # add loss layer # regularization_loss = self.config.l2 * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(U)) # included_indices = tf.where(tf.less(self.labels_placeholder, 2)) # self.full_loss = regularization_loss + tf.reduce_sum( # tf.nn.sparse_softmax_cross_entropy_with_logits( # logits=tf.gather(self.logits, included_indices),labels=tf.gather(self.labels_placeholder, included_indices))) self.root_loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.root_logits,labels=self.labels_placeholder[1:2])) # # add training op self.train_op = tf.train.GradientDescentOptimizer(self.config.lr).minimize(self.root_loss)
def __init__(self, config): self.config = config # Load train data and build vocabulary self.train_data, self.dev_data, self.test_data = tree.simplified_data(700, 100, 200) max_height = tree.get_max_tree_height(self.train_data + self.dev_data + self.test_data) self.config.max_tree_height = pow(2, max_height + 1) print(self.config.max_tree_height) # print("data ",self.train_data)) self.vocab = utils.Vocab() train_sents = [t.get_words() for t in self.train_data] self.vocab.construct(list(itertools.chain.from_iterable(train_sents))) # add input placeholders dim1 = self.config.batch_size dim2 = self.config.max_tree_height self.is_leaf_placeholder = tf.placeholder( tf.int32, [dim1, dim2], name='is_leaf_placeholder') self.node_word_indices_placeholder = tf.placeholder( tf.int32, [dim1, dim2], name='node_word_indices_placeholder') self.labels_placeholder = tf.placeholder( tf.int32, [dim1, dim2], name='labels_placeholder') self.cons_placeholder = tf.placeholder( tf.int32, (None), name='cons') # add model variables with tf.variable_scope('Embeddings'): self.embeddings = tf.get_variable('embeddings', [len(self.vocab), self.config.embed_size]) with tf.variable_scope('Composition'): self.W1 = tf.get_variable('W1', [2 * self.config.embed_size, self.config.embed_size]) self.b1 = tf.get_variable('b1', [1, self.config.embed_size]) with tf.variable_scope('Projection'): self.U = tf.get_variable('U', [self.config.embed_size, self.config.label_size]) self.bs = tf.get_variable('bs', [1, self.config.label_size]) # Build recursive graph def embed_word(word_index, embeddings): return tf.expand_dims(tf.gather(embeddings, word_index), 0) def combine_children(left_tensor, right_tensor, W, b): return tf.nn.relu(tf.matmul(tf.concat([left_tensor, right_tensor], 1), W) + b) def find_loss(node_tensor, i, labels, U, bs): # add projection layer node_logits = tf.matmul(node_tensor, U) + bs loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=node_logits, labels=labels[i:i+1]) return loss def base_case(node_word_indices, i, embeddings, labels, U, bs): word_index = tf.gather(node_word_indices, i) node_tensor = embed_word(word_index, embeddings) loss = find_loss(node_tensor, i, labels, U, bs) return [node_tensor, loss] def rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs): left_node, left_loss = self.rec(i*2, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs) right_node, right_loss = self.rec(i*2+1, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs) node_tensor = combine_children(left_node, right_node, W, b) node_loss = find_loss(node_tensor, i, labels, U, bs) loss = tf.concat([left_loss, node_loss, right_loss], 0) return [node_tensor, loss] # Function Declaration self.rec = function.Declare("Rec", [("i", tf.int32), ("is_leaf", tf.int32), ("node_word_indices", tf.int32), ("embeddings", tf.float32), ("W", tf.float32), ("b", tf.float32), ("labels", tf.int32), ("U", tf.float32), ("bs", tf.float32)], [("ret", tf.float32), ("ret1", tf.float32)]) # Function Definition @function.Defun(tf.int32, tf.int32, tf.int32, tf.float32, tf.float32, tf.float32, tf.int32, tf.float32, tf.float32, func_name="Rec", grad_func="GradFac", create_grad_func=True, out_names=["ret", "ret1"]) def RecImpl(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs): node_tensor, loss = \ tf.cond(tf.equal(tf.gather(is_leaf, i), tf.constant(1)), lambda: base_case(node_word_indices, i, embeddings, labels, U, bs), lambda: rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs)) return [node_tensor, loss] RecImpl.add_to_graph(tf.get_default_graph()) outloss = [] prediction = [] root_loss = [] for idx_batch in range(self.config.batch_size): self.root_prediction, self.full_loss, self.root_loss = self.compute_tree(idx_batch) prediction.append(self.root_prediction) outloss.append(self.full_loss) root_loss.append(self.root_loss) batch_loss = tf.stack(outloss) self.pred = tf.stack(prediction) self.rloss = tf.stack(root_loss) # Compute batch loss # reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) # regpart = tf.add_n(reg_losses) # loss = tf.reduce_mean(batch_loss) # self.total_loss = loss + 0.5*regpart self.total_loss = tf.reduce_mean(batch_loss) # Add training op self.train_op = tf.train.AdamOptimizer(self.config.lr).minimize(self.total_loss)