def embed_tree(logits_and_state, is_root):
"""Creates a block that embeds trees; output is tree LSTM state."""
return td.InputTransform(tokenize) >> td.OneOf(
key_fn=lambda pair: pair[0] == '2', # label 2 means neutral
case_blocks=(add_metrics(
is_root, is_neutral=False), add_metrics(is_root, is_neutral=True)),
pre_block=(td.Scalar('int32'), logits_and_state))
def setup_plan(plan):
"""Sets up a TensorFlow Fold plan for language identification."""
if plan.mode != 'train': raise ValueError('only train mode is supported')
embed_char = (td.Scalar(tf.int32) >> td.Embedding(
NUM_LETTERS, FLAGS.char_embed_vector_length))
process_word = (td.InputTransform(word_to_char_index) >> rnn_block(
embed_char, FLAGS.char_state_vector_length))
sentence = (td.InputTransform(sentence_to_words) >> rnn_block(
process_word, FLAGS.word_state_vector_length) >> td.FC(
NUM_LABELS, activation=None))
label = td.Scalar('int64')
# This is the final model. It takes a dictionary (i.e, a record) of the
# form
#
# {
# 'sentence': sentence-string,
# 'label': label-integer
# }
#
# as input and produces a tuple of (unscaled_logits, label) as output.
root_block = td.Record([('sentence', sentence), ('label', label)])
# Compile root_block to get a tensorflow model that we can run.
plan.compiler = td.Compiler.create(root_block)
# Get the tensorflow tensors that correspond to the outputs of root_block.
# These are TF tensors, and we can use them to compute loss in the usual way.
logits, labels = plan.compiler.output_tensors
plan.losses[
'cross_entropy'] = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels)
predictions = tf.argmax(logits, 1)
plan.metrics['accuracy'] = tf.reduce_mean(
tf.cast(tf.equal(predictions, labels), dtype=tf.float32))
plan.examples = sentence_rows(FLAGS.train_file)
plan.dev_examples = sentence_rows(FLAGS.dev_file)
def build_sequence_transcoder(vocab_filepath, word_embedding_size):
vocab_size = 5
# From words to list of integers
vocab = load_vocab(vocab_filepath)
words2integers = td.InputTransform(
lambda s: [word2index(vocab, w) for w in s])
# From interger to word embedding
word2embedding = td.Scalar('int32') >> td.Function(
td.Embedding(vocab_size, word_embedding_size))
# From word to array of embeddings
sequence_transcoder = words2integers >> td.Map(word2embedding)
return sequence_transcoder
def create_model(word_embedding, word_idx, lstm_num_units, mlp_size, keep_prob=1):
"""Creates a sentiment model. Returns (compiler, mean metrics)."""
tree_lstm = td.ScopedLayer(
tf.contrib.rnn.DropoutWrapper(
BinaryTreeLSTMCell(lstm_num_units, keep_prob=keep_prob),
input_keep_prob=keep_prob, output_keep_prob=keep_prob),
name_or_scope='tree_lstm')
embed_subtree = td.ForwardDeclaration(output_type=tree_lstm.state_size)
unknown_idx = len(word_idx)
def lookup_word(word):
return word_idx.get(word, unknown_idx)
word2vec = (td.GetItem(0) >> td.InputTransform(lookup_word) >>
td.Scalar('int32') >> word_embedding)
child2vec = td.GetItem(1) >> td.TupleType(td.Map(embed_subtree()))
tree2vec = td.AllOf(word2vec, child2vec)
tree = tree2vec >> tree_lstm
embed_subtree.resolve_to(tree)
expression_logits = (td.GetItem(0) >> td.Map(tree) >> td.Concat()
>> td.FC(mlp_size, activation='relu', name='mlp1')
>> td.FC(mlp_size, activation='relu', name='mlp2'))
expression_label = td.GetItem(1) >> td.Scalar('int32')
model = td.AllOf(expression_logits, expression_label)
compiler = td.Compiler.create(model)
logits, labels = compiler.output_tensors
_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=labels))
_accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(labels, 1),
tf.argmax(logits, 1)),
dtype=tf.float32))
return compiler, _loss, _accuracy
def logits_and_state():
"""Creates a block that goes from tokens to (logits, state) tuples."""
word2vec = (td.GetItem(0) >> td.InputTransform(lookup_word) >>
td.Scalar('int32') >> word_embedding)
pair2vec = (embed_subtree(), embed_subtree())
# Trees are binary, so the tree layer takes two states as its input_state.
zero_state = td.Zeros((tree_lstm.state_size, ) * 2)
# Input is a word vector.
zero_inp = td.Zeros(word_embedding.output_type.shape[0])
word_case = td.AllOf(word2vec, zero_state)
pair_case = td.AllOf(zero_inp, pair2vec)
tree2vec = td.OneOf(len, [(1, word_case), (2, pair_case)])
return tree2vec >> tree_lstm >> (output_layer, td.Identity())
def __init__(self, state_size):
# Expressions are either constants, or calculator ops that take other
# expressions as their arguments. Since an Expression is a recursive type,
# the model must likewise be recursive. A ForwardDeclaration declares the
# type of expression, so it can be used before it before it is defined.
expr_decl = td.ForwardDeclaration(td.PyObjectType(), state_size)
# Create a block for each type of expression.
# The terminals are the digits 0-9, which we map to vectors using
# an embedding table.
digit = (
td.GetItem('number') >> td.Scalar(dtype='int32') >> td.Function(
td.Embedding(10, state_size, name='terminal_embed')))
# For non terminals, recursively apply expression to the left/right sides,
# concatenate the results, and pass them through a fully-connected layer.
# Each operation uses different weights in the FC layer.
def bin_op(name):
return (td.Record([('left', expr_decl()), ('right', expr_decl())])
>> td.Concat() >> td.FC(state_size, name='FC_' + name))
# OneOf will dispatch its input to the appropriate case, based on the value
# in the 'op'.'name' field.
cases = td.OneOf(
lambda x: x['op']['name'], {
'NUM': digit,
'PLUS': bin_op('PLUS'),
'MINUS': bin_op('MINUS'),
'TIMES': bin_op('TIMES'),
'DIV': bin_op('DIV')
})
# We do preprocessing to add 'NUM' as a distinct case.
expression = td.InputTransform(preprocess_expression) >> cases
expr_decl.resolve_to(expression)
# Get logits from the root of the expression tree
expression_logits = (
expression >> td.FC(NUM_LABELS, activation=None, name='FC_logits'))
# The result is stored in the expression itself.
# We ignore it in td.Record above, and pull it out here.
expression_label = (
td.GetItem('result') >> td.InputTransform(result_sign) >>
td.OneHot(NUM_LABELS))
# For the overall model, return a pair of (logits, labels)
# The AllOf block will run each of its children on the same input.
model = td.AllOf(expression_logits, expression_label)
self._compiler = td.Compiler.create(model)
# Get the tensorflow tensors that correspond to the outputs of model.
# `logits` and `labels` are TF tensors, and we can use them to
# compute losses in the usual way.
(logits, labels) = self._compiler.output_tensors
self._loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
self._accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(labels, 1), tf.argmax(logits, 1)),
dtype=tf.float32))
self._global_step = tf.Variable(0, name='global_step', trainable=False)
optr = tf.train.GradientDescentOptimizer(0.01)
self._train_op = optr.minimize(self._loss,
global_step=self._global_step)