def setup_plan(plan):
"""Sets up a TensorFlow Fold plan for MNIST.
The inputs are 28 x 28 images represented as 784-dimensional float32
vectors (scaled to [0, 1] and categorical digit labels in [0, 9].
The training loss is softmax cross-entropy. There is only one
metric, accuracy. In inference mode, the output is a class label.
Dropout is applied before every layer (including on the inputs).
Args:
plan: A TensorFlow Fold plan to set up.
"""
# Convert the input NumPy array into a tensor.
model_block = td.Vector(INPUT_LENGTH)
# Create a placeholder for dropout, if we are in train mode.
keep_prob = (tf.placeholder_with_default(1.0, [], name='keep_prob')
if plan.mode == plan.mode_keys.TRAIN else None)
# Add the fully connected hidden layers.
for _ in xrange(FLAGS.num_layers):
model_block >>= td.FC(FLAGS.num_units, input_keep_prob=keep_prob)
# Add the linear output layer.
model_block >>= td.FC(NUM_LABELS, activation=None, input_keep_prob=keep_prob)
if plan.mode == plan.mode_keys.INFER:
# In inference mode, we run the model directly on images.
plan.compiler = td.Compiler.create(model_block)
logits, = plan.compiler.output_tensors
else:
# In training/eval mode, we run the model on (image, label) pairs.
plan.compiler = td.Compiler.create(
td.Record((model_block, td.Scalar(tf.int64))))
logits, y_ = plan.compiler.output_tensors
y = tf.argmax(logits, 1) # create the predicted output tensor
datasets = tf.contrib.learn.datasets.mnist.load_mnist(FLAGS.logdir_base)
if plan.mode == plan.mode_keys.INFER:
plan.examples = datasets.test.images
plan.outputs = [y]
else:
# Create loss and accuracy tensors, and add them to the plan.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=y_)
plan.losses['cross_entropy'] = loss
accuracy = tf.reduce_mean(tf.cast(tf.equal(y, y_), tf.float32))
plan.metrics['accuracy'] = accuracy
if plan.mode == plan.mode_keys.TRAIN:
plan.examples = zip(datasets.train.images, datasets.train.labels)
plan.dev_examples = zip(datasets.validation.images,
datasets.validation.labels)
# Turn dropout on for training, off for validation.
plan.train_feeds[keep_prob] = FLAGS.keep_prob
else:
assert plan.mode == plan.mode_keys.EVAL
plan.examples = zip(datasets.test.images, datasets.test.labels)
def embed_tree(logits_and_state, is_root):
"""Creates a block that embeds trees; output is tree LSTM state."""
return td.InputTransform(tokenize) >> td.OneOf(
key_fn=lambda pair: pair[0] == '2', # label 2 means neutral
case_blocks=(add_metrics(
is_root, is_neutral=False), add_metrics(is_root, is_neutral=True)),
pre_block=(td.Scalar('int32'), logits_and_state))
def create_model(word_embedding, word_idx, lstm_num_units, mlp_size, keep_prob=1):
"""Creates a sentiment model. Returns (compiler, mean metrics)."""
tree_lstm = td.ScopedLayer(
tf.contrib.rnn.DropoutWrapper(
BinaryTreeLSTMCell(lstm_num_units, keep_prob=keep_prob),
input_keep_prob=keep_prob, output_keep_prob=keep_prob),
name_or_scope='tree_lstm')
embed_subtree = td.ForwardDeclaration(output_type=tree_lstm.state_size)
unknown_idx = len(word_idx)
def lookup_word(word):
return word_idx.get(word, unknown_idx)
word2vec = (td.GetItem(0) >> td.InputTransform(lookup_word) >>
td.Scalar('int32') >> word_embedding)
child2vec = td.GetItem(1) >> td.TupleType(td.Map(embed_subtree()))
tree2vec = td.AllOf(word2vec, child2vec)
tree = tree2vec >> tree_lstm
embed_subtree.resolve_to(tree)
expression_logits = (td.GetItem(0) >> td.Map(tree) >> td.Concat()
>> td.FC(mlp_size, activation='relu', name='mlp1')
>> td.FC(mlp_size, activation='relu', name='mlp2'))
expression_label = td.GetItem(1) >> td.Scalar('int32')
model = td.AllOf(expression_logits, expression_label)
compiler = td.Compiler.create(model)
logits, labels = compiler.output_tensors
_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=labels))
_accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(labels, 1),
tf.argmax(logits, 1)),
dtype=tf.float32))
return compiler, _loss, _accuracy
def setup_plan(plan):
"""Sets up a TensorFlow Fold plan for language identification."""
if plan.mode != 'train': raise ValueError('only train mode is supported')
embed_char = (td.Scalar(tf.int32) >> td.Embedding(
NUM_LETTERS, FLAGS.char_embed_vector_length))
process_word = (td.InputTransform(word_to_char_index) >> rnn_block(
embed_char, FLAGS.char_state_vector_length))
sentence = (td.InputTransform(sentence_to_words) >> rnn_block(
process_word, FLAGS.word_state_vector_length) >> td.FC(
NUM_LABELS, activation=None))
label = td.Scalar('int64')
# This is the final model. It takes a dictionary (i.e, a record) of the
# form
#
# {
# 'sentence': sentence-string,
# 'label': label-integer
# }
#
# as input and produces a tuple of (unscaled_logits, label) as output.
root_block = td.Record([('sentence', sentence), ('label', label)])
# Compile root_block to get a tensorflow model that we can run.
plan.compiler = td.Compiler.create(root_block)
# Get the tensorflow tensors that correspond to the outputs of root_block.
# These are TF tensors, and we can use them to compute loss in the usual way.
logits, labels = plan.compiler.output_tensors
plan.losses[
'cross_entropy'] = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels)
predictions = tf.argmax(logits, 1)
plan.metrics['accuracy'] = tf.reduce_mean(
tf.cast(tf.equal(predictions, labels), dtype=tf.float32))
plan.examples = sentence_rows(FLAGS.train_file)
plan.dev_examples = sentence_rows(FLAGS.dev_file)
def build_sequence_transcoder(vocab_filepath, word_embedding_size):
vocab_size = 5
# From words to list of integers
vocab = load_vocab(vocab_filepath)
words2integers = td.InputTransform(
lambda s: [word2index(vocab, w) for w in s])
# From interger to word embedding
word2embedding = td.Scalar('int32') >> td.Function(
td.Embedding(vocab_size, word_embedding_size))
# From word to array of embeddings
sequence_transcoder = words2integers >> td.Map(word2embedding)
return sequence_transcoder
def setup_plan(plan):
# Convert the input Numpy array into a tensor.
model_block = td.Vector(INPUT_LENGTH)
# Create a placeholder for dropout, if we are in train mode
keep_prob = (tf.placeholder_with_default(1.0, [], name='keep_prob') if plan.mode == plan.mode_keys.TRAIN else None)
# add fc layers(using function) with drop out
for _ in xrange(FLAGS.num_layers):
model_block >>= td.FC(FLAGS.num_units, input_keep_prob=keep_prob)
# add output layer with drop out
model_block >>= td.FC(NUM_LABELS, activation=None, input_keep_prob=keep_prob)
if plan.mode == plan.mode_keys.INFER:
print("inference:")
# compiler with model
plan.compiler = td.Compiler.create(model_block)
logits, = plan.compiler.output_tensors
else:
print("train:")
# compiler with model, ground truth
plan.compiler = td.Compiler.create(td.Record((model_block, td.Scalar(tf.int64))))
logits, y_ = plan.compiler.output_tensors
# prediction result
y = tf.argmax(logits, 1)
# load minist datasets
datasets = tf.contrib.learn.datasets.mnist.load_mnist(FLAGS.logdir_base)
if plan.mode == plan.mode_keys.INFER:
plan.examples = datasets.test.images
plan.outputs = [y]
else:
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_)
plan.losses['cross_entropy'] = loss
accuracy = tf.reduce_mean(tf.cast(tf.equal(y, y_), tf.float32))
plan.metrics['accuracy'] = accuracy
if plan.mode == plan.mode_keys.TRAIN:
plan.examples = zip(datasets.train.images, datasets.train.labels)
plan.dev_examples = zip(datasets.validation.images, datasets.validation.labels)
# Turn dropout on for training, off for validation.
plan.train_feeds[keep_prob] = FLAGS.keep_prob
else:
assert plan.mode == plan.mode_keys.EVAL
plan.examples = zip(datasets.test.images, datasets.tesst.labels)
def logits_and_state():
"""Creates a block that goes from tokens to (logits, state) tuples."""
word2vec = (td.GetItem(0) >> td.InputTransform(lookup_word) >>
td.Scalar('int32') >> word_embedding)
pair2vec = (embed_subtree(), embed_subtree())
# Trees are binary, so the tree layer takes two states as its input_state.
zero_state = td.Zeros((tree_lstm.state_size, ) * 2)
# Input is a word vector.
zero_inp = td.Zeros(word_embedding.output_type.shape[0])
word_case = td.AllOf(word2vec, zero_state)
pair_case = td.AllOf(zero_inp, pair2vec)
tree2vec = td.OneOf(len, [(1, word_case), (2, pair_case)])
return tree2vec >> tree_lstm >> (output_layer, td.Identity())
def train(fn, batch_size=100):
net_block = reduce_net_block()
compiler = td.Compiler.create((net_block, td.Scalar()))
# compiler have 2 tds. output_tensor for each tds.
y, y_ = compiler.output_tensors
loss = tf.nn.l2_loss(y - y_)
train = tf.train.AdamOptimizer().minimize(loss)
sess.run(tf.global_variables_initializer())
validation_fd = compiler.build_feed_dict(
random_example(fn) for _ in range(1000))
for i in range(2000):
sess.run(
train,
compiler.build_feed_dict(
random_example(fn) for _ in range(batch_size)))
if i % 100 == 0:
print("step %d: loss %f" % (i, sess.run(loss, validation_fd)))
return net_block
def __init__(self, state_size):
# Expressions are either constants, or calculator ops that take other
# expressions as their arguments. Since an Expression is a recursive type,
# the model must likewise be recursive. A ForwardDeclaration declares the
# type of expression, so it can be used before it before it is defined.
expr_decl = td.ForwardDeclaration(td.PyObjectType(), state_size)
# Create a block for each type of expression.
# The terminals are the digits 0-9, which we map to vectors using
# an embedding table.
digit = (
td.GetItem('number') >> td.Scalar(dtype='int32') >> td.Function(
td.Embedding(10, state_size, name='terminal_embed')))
# For non terminals, recursively apply expression to the left/right sides,
# concatenate the results, and pass them through a fully-connected layer.
# Each operation uses different weights in the FC layer.
def bin_op(name):
return (td.Record([('left', expr_decl()), ('right', expr_decl())])
>> td.Concat() >> td.FC(state_size, name='FC_' + name))
# OneOf will dispatch its input to the appropriate case, based on the value
# in the 'op'.'name' field.
cases = td.OneOf(
lambda x: x['op']['name'], {
'NUM': digit,
'PLUS': bin_op('PLUS'),
'MINUS': bin_op('MINUS'),
'TIMES': bin_op('TIMES'),
'DIV': bin_op('DIV')
})
# We do preprocessing to add 'NUM' as a distinct case.
expression = td.InputTransform(preprocess_expression) >> cases
expr_decl.resolve_to(expression)
# Get logits from the root of the expression tree
expression_logits = (
expression >> td.FC(NUM_LABELS, activation=None, name='FC_logits'))
# The result is stored in the expression itself.
# We ignore it in td.Record above, and pull it out here.
expression_label = (
td.GetItem('result') >> td.InputTransform(result_sign) >>
td.OneHot(NUM_LABELS))
# For the overall model, return a pair of (logits, labels)
# The AllOf block will run each of its children on the same input.
model = td.AllOf(expression_logits, expression_label)
self._compiler = td.Compiler.create(model)
# Get the tensorflow tensors that correspond to the outputs of model.
# `logits` and `labels` are TF tensors, and we can use them to
# compute losses in the usual way.
(logits, labels) = self._compiler.output_tensors
self._loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
self._accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(labels, 1), tf.argmax(logits, 1)),
dtype=tf.float32))
self._global_step = tf.Variable(0, name='global_step', trainable=False)
optr = tf.train.GradientDescentOptimizer(0.01)
self._train_op = optr.minimize(self._loss,
global_step=self._global_step)
# evaluating indivisual inputs
print("one hot vector ======")
onehot_block = td.OneHot(5)
# evaluating block using eval
print(onehot_block.eval(3))
# => array([0., 0., 0., 1., 0.], dtype=float32)
# others
print(onehot_block.eval(0))
print(onehot_block.eval(1))
print(onehot_block.eval(2))
print(onehot_block.eval(4))
print("composite blocks =====")
composite_blocks = td.Scalar() >> td.AllOf(td.Function(tf.negative),
td.Function(tf.square))
print(composite_blocks.eval(2)[0]) #negative
print(composite_blocks.eval(2)[1]) #square
print("batching inputs =====")
# Compiler compiles out model down to TensorFlow graph.
# Compiler will also do type inference and validation on the model.
# The outputs are ordinary TensorFlow tensors, which can be connected to Tensorflow loss function and optimizer.
print("blocks have associated input and output types =====")
print(td.Scalar().input_type)
scalar_block = td.Scalar()
print(td.Scalar().output_type)
print(scalar_block.eval(5))
def reduce_net_block():
net_block = td.Concat() >> td.FC(20) >> td.FC(
1, activation=None) >> td.Function(lambda xs: tf.squeeze(xs, axis=1))
return td.Map(td.Scalar()) >> td.Reduce(net_block)
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import random
from six.moves import xrange # pylint: disable=redefined-builtin
from six.moves import zip # pylint: disable=redefined-builtin
import tensorflow as tf
sess = tf.InteractiveSession()
import tensorflow_fold.public.blocks as td
print("[Sandbox] start. ==========")
# Map and Reduce
print("Map Reduce Processing sample. ==========")
print(td.Map(td.Scalar()))
print((td.Map(td.Scalar()) >> td.Reduce(td.Function(tf.multiply))).eval(
range(1, 10)))
# simple fc network
def reduce_net_block():
net_block = td.Concat() >> td.FC(20) >> td.FC(
1, activation=None) >> td.Function(lambda xs: tf.squeeze(xs, axis=1))
return td.Map(td.Scalar()) >> td.Reduce(net_block)
# generate ramdom example data, result
def random_example(fn):
length = random.randrange(1, 10)
data = [random.uniform(0, 1) for _ in range(length)]