def testDeclare(self):
foo = function.Declare("Foo", [("x", dtypes.float32)],
[("y", dtypes.float32)])
@function.Defun(dtypes.float32, func_name="Foo", out_names=["y"])
def FooImpl(x):
return x * x + 1
x = array_ops.placeholder(dtypes.float32)
y = foo(x)
g = ops.get_default_graph()
FooImpl.add_to_graph(g)
with self.test_session():
rand = np.random.uniform(size=(3, 3))
expected = rand * rand + 1.0
self.assertAllClose(expected, y.eval(feed_dict={x: rand}))
def testDeclareTypeMistake(self):
foo = function.Declare("Foo", [("x", dtypes.float32)],
[("y", dtypes.float32)])
@function.Defun(dtypes.float32, func_name="Foo", out_names=["y"])
def Foo(x):
return x * x + 1
g = ops.Graph()
with g.as_default():
y = foo(2.0)
with self.test_session(graph=g):
with self.assertRaisesRegexp(errors_impl.NotFoundError,
"not registered"):
_ = y.eval()
g = ops.Graph()
with g.as_default():
Foo.add_to_graph(g)
y = foo(2)
with self.test_session(graph=g):
with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
"int32.*float"):
_ = y.eval()
g = ops.Graph()
with g.as_default():
Foo.add_to_graph(g)
with self.assertRaisesRegexp(
ValueError,
"Expected number of arguments: 1, received: 2"):
_ = foo(2.0, 2.0)
g = ops.Graph()
with g.as_default():
Foo.add_to_graph(g)
y = foo(2.0)
with self.test_session(graph=g):
self.assertAllEqual(y.eval(), 5.0)
def testDeclareTypeMistake(self):
foo = function.Declare("Foo", [tf.float32], [tf.float32])
@function.Defun(tf.float32)
def Foo(x):
return x * x + 1
g = tf.Graph()
with g.as_default():
y = foo(2.0)
with self.test_session(graph=g):
with self.assertRaisesRegexp(tf.errors.NotFoundError,
"not registered"):
_ = y.eval()
g = tf.Graph()
with g.as_default():
Foo.add_to_graph(g)
y = foo(2)
with self.test_session(graph=g):
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"int32.*float"):
_ = y.eval()
g = tf.Graph()
with g.as_default():
Foo.add_to_graph(g)
with self.assertRaisesRegexp(
ValueError,
"Expected number of arguments: 1, received: 2"):
_ = foo(2.0, 2.0)
g = tf.Graph()
with g.as_default():
Foo.add_to_graph(g)
y = foo(2.0)
with self.test_session(graph=g):
self.assertAllEqual(y.eval(), 5.0)
def testDeclareTypeMistake(self):
foo = function.Declare("Foo", [tf.float32], [tf.float32])
@function.Defun(tf.float32)
def Foo(x):
return x * x + 1
g = tf.Graph()
with g.as_default():
y = foo(tf.constant(2, tf.float32))
with self.test_session(graph=g):
with self.assertRaisesRegexp(tf.errors.NotFoundError, "not registered"):
_ = y.eval()
g = tf.Graph()
with g.as_default():
Foo.add_to_graph(g)
y = foo(tf.constant(2, tf.int32))
with self.test_session(graph=g):
with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
"int32.*float"):
_ = y.eval()
g = tf.Graph()
with g.as_default():
Foo.add_to_graph(g)
with self.assertRaisesRegexp(ValueError,
"Mismatch number of args: 2 vs. 1"):
_ = foo(tf.constant(2, tf.float32), tf.constant(2, tf.float32))
g = tf.Graph()
with g.as_default():
Foo.add_to_graph(g)
y = foo(tf.constant(2, tf.float32))
with self.test_session(graph=g):
self.assertAllEqual(y.eval(), 5.0)
def __init__(self, config):
self.config = config
# Load train data and build vocabulary
self.train_data, self.dev_data, self.test_data = tree.simplified_data(
700, 100, 200)
# print("data ",self.train_data))
self.vocab = utils.Vocab()
train_sents = [t.get_words() for t in self.train_data]
self.vocab.construct(list(itertools.chain.from_iterable(train_sents)))
# add input placeholders
self.is_leaf_placeholder = tf.placeholder(tf.int32, (None),
name='is_leaf_placeholder')
self.node_word_indices_placeholder = tf.placeholder(
tf.int32, (None), name='node_word_indices_placeholder')
self.labels_placeholder = tf.placeholder(tf.int32, (None),
name='labels_placeholder')
self.cons_placeholder = tf.placeholder(tf.int32, (None), name='cons')
# add model variables
# making initialization deterministic for now
# initializer = tf.random_normal_initializer(seed=1)
with tf.variable_scope('Embeddings'):
self.embeddings = tf.get_variable(
'embeddings', [len(self.vocab), self.config.embed_size])
with tf.variable_scope('Composition'):
W1 = tf.get_variable(
'W1', [2 * self.config.embed_size, self.config.embed_size])
b1 = tf.get_variable('b1', [1, self.config.embed_size])
with tf.variable_scope('Projection'):
U = tf.get_variable(
'U', [self.config.embed_size, self.config.label_size])
bs = tf.get_variable('bs', [1, self.config.label_size])
# Build recursive graph
def embed_word(word_index, embeddings):
return tf.expand_dims(tf.gather(embeddings, word_index), 0)
def combine_children(left_tensor, right_tensor, W, b):
return tf.nn.relu(
tf.matmul(tf.concat([left_tensor, right_tensor], 1), W) + b)
def find_loss(node_tensor, i, labels, U, bs):
# add projection layer
node_logits = tf.matmul(node_tensor, U) + bs
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=node_logits, labels=labels[i:i + 1])
return loss
def base_case(node_word_indices, i, embeddings, labels, U, bs):
word_index = tf.gather(node_word_indices, i)
node_tensor = embed_word(word_index, embeddings)
loss = find_loss(node_tensor, i, labels, U, bs)
return [node_tensor, loss]
def rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels,
U, bs):
left_node, left_loss = rec(i * 2, is_leaf, node_word_indices,
embeddings, W, b, labels, U, bs)
right_node, right_loss = rec(i * 2 + 1, is_leaf, node_word_indices,
embeddings, W, b, labels, U, bs)
node_tensor = combine_children(left_node, right_node, W, b)
node_loss = find_loss(node_tensor, i, labels, U, bs)
loss = tf.concat([left_loss, node_loss, right_loss], 0)
return [node_tensor, loss]
# Function Declaration
rec = function.Declare("Rec", [("i", tf.int32), ("is_leaf", tf.int32),
("node_word_indices", tf.int32),
("embeddings", tf.float32),
("W", tf.float32), ("b", tf.float32),
("labels", tf.int32), ("U", tf.float32),
("bs", tf.float32)],
[("ret", tf.float32), ("ret1", tf.float32)])
# Function Definition
@function.Defun(tf.int32,
tf.int32,
tf.int32,
tf.float32,
tf.float32,
tf.float32,
tf.int32,
tf.float32,
tf.float32,
func_name="Rec",
grad_func="GradFac",
create_grad_func=True,
out_names=["ret", "ret1"])
def RecImpl(i, is_leaf, node_word_indices, embeddings, W, b, labels, U,
bs):
node_tensor, loss = \
tf.cond(tf.equal(tf.gather(is_leaf, i), tf.constant(1)),
lambda: base_case(node_word_indices, i, embeddings, labels, U, bs),
lambda: rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs))
return [node_tensor, loss]
RecImpl.add_to_graph(tf.get_default_graph())
self.node_tensor, self.full_loss = rec(
self.cons_placeholder, self.is_leaf_placeholder,
self.node_word_indices_placeholder, self.embeddings, W1, b1,
self.labels_placeholder, U, bs)
# add projection layer
self.root_logits = tf.matmul(self.node_tensor, U) + bs
self.root_prediction = tf.squeeze(tf.argmax(self.root_logits, 1))
# add loss layer
self.root_loss = tf.reduce_sum(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.root_logits, labels=self.labels_placeholder[1:2]))
regularization_loss = self.config.l2 * (tf.nn.l2_loss(W1) +
tf.nn.l2_loss(U))
self.full_loss = regularization_loss + tf.reduce_sum(self.full_loss)
# # add training op
self.train_op = tf.train.AdamOptimizer(self.config.lr).minimize(
self.full_loss)
def __init__(self, config):
self.config = config
# Load train data and build vocabulary
self.train_data, self.dev_data, self.test_data = tree.simplified_data(700,
100,
200)
# print("data ",self.train_data))
self.vocab = utils.Vocab()
train_sents = [t.get_words() for t in self.train_data]
self.vocab.construct(list(itertools.chain.from_iterable(train_sents)))
# add input placeholders
self.is_leaf_placeholder = tf.placeholder(
tf.int32, (None), name='is_leaf_placeholder')
self.node_word_indices_placeholder = tf.placeholder(
tf.int32, (None), name='node_word_indices_placeholder')
self.labels_placeholder = tf.placeholder(
tf.int32, (None), name='labels_placeholder')
self.cons_placeholder = tf.placeholder(
tf.int32, (None), name='cons')
# add model variables
# making initialization deterministic for now
initializer = tf.random_normal_initializer(seed=1)
with tf.variable_scope('Embeddings'):
self.embeddings = tf.get_variable('embeddings',
[len(self.vocab),
self.config.embed_size])
# ,
# initializer=initializer) #tf.constant_initializer(2.0))
with tf.variable_scope('Composition'):
W1 = tf.get_variable('W1',
[2 * self.config.embed_size,
self.config.embed_size])
# ,
# initializer=initializer) #tf.constant_initializer(0.0))
b1 = tf.get_variable('b1', [1, self.config.embed_size])
# ,
# initializer=initializer) #tf.constant_initializer(0.0))
with tf.variable_scope('Projection'):
U = tf.get_variable('U',
[self.config.embed_size,
self.config.label_size])
# ,
# initializer=initializer) #tf.constant_initializer(0.0))
bs = tf.get_variable('bs', [1, self.config.label_size])
# ,
# initializer=initializer) #tf.constant_initializer(0.0))
# Build recursive graph
# tensor_array = tf.TensorArray(
# tf.float32,
# size=0,
# dynamic_size=True,
# clear_after_read=False,
# infer_shape=False)
# Build recursive graph
def embed_word(word_index, embeddings):
# with tf.device('/cpu:0'):
return tf.expand_dims(tf.gather(embeddings, word_index), 0)
def combine_children(left_tensor, right_tensor, W, b):
return tf.nn.relu(tf.matmul(tf.concat([left_tensor, right_tensor], 1), W) + b)
# Function Declaration
rec = function.Declare("Rec", [("i", tf.int32), ("is_leaf", tf.int32),
("node_word_indices", tf.int32), ("embeddings", tf.float32), ("W", tf.float32),("b", tf.float32)],
[("ret", tf.float32)])
# Function Definition
@function.Defun(tf.int32, tf.int32, tf.int32, tf.float32, tf.float32, tf.float32, func_name="Rec", grad_func="GradFac", create_grad_func=True, out_names=["ret"])
def RecImpl(i, is_leaf, node_word_indices, embeddings, W, b):
node_word_index = tf.gather(node_word_indices, i)
node_tensor = \
tf.cond(tf.equal(tf.gather(is_leaf, i), tf.constant(1)),
lambda: embed_word(node_word_index, embeddings),
lambda: combine_children(rec(i*2, is_leaf, node_word_indices, embeddings, W, b),
rec(i*2+1, is_leaf, node_word_indices, embeddings, W, b), W, b))
return node_tensor
RecImpl.add_to_graph(tf.get_default_graph())
self.node_tensor = rec(self.cons_placeholder, self.is_leaf_placeholder,
self.node_word_indices_placeholder, self.embeddings, W1, b1)
# add projection layer
# self.logits = tf.matmul(self.tensor_array.concat(), U) + bs
# 1x35 * 35x35 + 1x35 -> 1x35 projection
self.root_logits = tf.matmul(self.node_tensor, U) + bs
self.root_prediction = tf.squeeze(tf.argmax(self.root_logits, 1))
# add loss layer
# regularization_loss = self.config.l2 * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(U))
# included_indices = tf.where(tf.less(self.labels_placeholder, 2))
# self.full_loss = regularization_loss + tf.reduce_sum(
# tf.nn.sparse_softmax_cross_entropy_with_logits(
# logits=tf.gather(self.logits, included_indices),labels=tf.gather(self.labels_placeholder, included_indices)))
self.root_loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.root_logits,labels=self.labels_placeholder[1:2]))
# # add training op
self.train_op = tf.train.GradientDescentOptimizer(self.config.lr).minimize(self.root_loss)
def __init__(self, config):
self.config = config
# Load train data and build vocabulary
self.train_data, self.dev_data, self.test_data = tree.simplified_data(700,
100,
200)
max_height = tree.get_max_tree_height(self.train_data + self.dev_data + self.test_data)
self.config.max_tree_height = pow(2, max_height + 1)
print(self.config.max_tree_height)
# print("data ",self.train_data))
self.vocab = utils.Vocab()
train_sents = [t.get_words() for t in self.train_data]
self.vocab.construct(list(itertools.chain.from_iterable(train_sents)))
# add input placeholders
dim1 = self.config.batch_size
dim2 = self.config.max_tree_height
self.is_leaf_placeholder = tf.placeholder(
tf.int32, [dim1, dim2], name='is_leaf_placeholder')
self.node_word_indices_placeholder = tf.placeholder(
tf.int32, [dim1, dim2], name='node_word_indices_placeholder')
self.labels_placeholder = tf.placeholder(
tf.int32, [dim1, dim2], name='labels_placeholder')
self.cons_placeholder = tf.placeholder(
tf.int32, (None), name='cons')
# add model variables
with tf.variable_scope('Embeddings'):
self.embeddings = tf.get_variable('embeddings',
[len(self.vocab),
self.config.embed_size])
with tf.variable_scope('Composition'):
self.W1 = tf.get_variable('W1',
[2 * self.config.embed_size,
self.config.embed_size])
self.b1 = tf.get_variable('b1', [1, self.config.embed_size])
with tf.variable_scope('Projection'):
self.U = tf.get_variable('U',
[self.config.embed_size,
self.config.label_size])
self.bs = tf.get_variable('bs', [1, self.config.label_size])
# Build recursive graph
def embed_word(word_index, embeddings):
return tf.expand_dims(tf.gather(embeddings, word_index), 0)
def combine_children(left_tensor, right_tensor, W, b):
return tf.nn.relu(tf.matmul(tf.concat([left_tensor, right_tensor], 1), W) + b)
def find_loss(node_tensor, i, labels, U, bs):
# add projection layer
node_logits = tf.matmul(node_tensor, U) + bs
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=node_logits, labels=labels[i:i+1])
return loss
def base_case(node_word_indices, i, embeddings, labels, U, bs):
word_index = tf.gather(node_word_indices, i)
node_tensor = embed_word(word_index, embeddings)
loss = find_loss(node_tensor, i, labels, U, bs)
return [node_tensor, loss]
def rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs):
left_node, left_loss = self.rec(i*2, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs)
right_node, right_loss = self.rec(i*2+1, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs)
node_tensor = combine_children(left_node, right_node, W, b)
node_loss = find_loss(node_tensor, i, labels, U, bs)
loss = tf.concat([left_loss, node_loss, right_loss], 0)
return [node_tensor, loss]
# Function Declaration
self.rec = function.Declare("Rec", [("i", tf.int32), ("is_leaf", tf.int32), ("node_word_indices", tf.int32),
("embeddings", tf.float32), ("W", tf.float32), ("b", tf.float32), ("labels", tf.int32), ("U", tf.float32), ("bs", tf.float32)],
[("ret", tf.float32), ("ret1", tf.float32)])
# Function Definition
@function.Defun(tf.int32, tf.int32, tf.int32, tf.float32, tf.float32, tf.float32, tf.int32, tf.float32, tf.float32, func_name="Rec", grad_func="GradFac", create_grad_func=True, out_names=["ret", "ret1"])
def RecImpl(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs):
node_tensor, loss = \
tf.cond(tf.equal(tf.gather(is_leaf, i), tf.constant(1)),
lambda: base_case(node_word_indices, i, embeddings, labels, U, bs),
lambda: rec_case(i, is_leaf, node_word_indices, embeddings, W, b, labels, U, bs))
return [node_tensor, loss]
RecImpl.add_to_graph(tf.get_default_graph())
outloss = []
prediction = []
root_loss = []
for idx_batch in range(self.config.batch_size):
self.root_prediction, self.full_loss, self.root_loss = self.compute_tree(idx_batch)
prediction.append(self.root_prediction)
outloss.append(self.full_loss)
root_loss.append(self.root_loss)
batch_loss = tf.stack(outloss)
self.pred = tf.stack(prediction)
self.rloss = tf.stack(root_loss)
# Compute batch loss
# reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
# regpart = tf.add_n(reg_losses)
# loss = tf.reduce_mean(batch_loss)
# self.total_loss = loss + 0.5*regpart
self.total_loss = tf.reduce_mean(batch_loss)
# Add training op
self.train_op = tf.train.AdamOptimizer(self.config.lr).minimize(self.total_loss)
