def __init__(self, num_options, tag=''):
input_shape = loom.TypeShape(tf.float32, (num_options, ))
temperature_shape = loom.TypeShape(tf.float32, [])
output_shape = loom.TypeShape(tf.int64, [], tag)
super(MultinomialLoomOp,
self).__init__([input_shape, temperature_shape], [output_shape])
def __init__(self, emb_size, reverse=False):
"""Constructor.
Args:
emb_size: Size of input.
reverse: If True, compute KL(p(z) || q(z | x)) in addition.
"""
emb_shape = loom.TypeShape(tf.float32, (emb_size, ))
cost_shape = loom.TypeShape(tf.float32, [], 'kl_div')
self.w = framework.variable('q_z_x_w', [emb_size, emb_size * 2])
self.b = framework.variable('q_z_x_b', [emb_size * 2],
initializer=tf.constant_initializer(0.0))
self.emb_size = emb_size
self.reverse = reverse
if reverse:
rev_cost_shape = loom.TypeShape(tf.float32, [], 'rev_kl_div')
super(KLDivPosteriorPriorLoomOp, self).__init__(
[emb_shape],
[cost_shape, rev_cost_shape, emb_shape, emb_shape])
else:
super(KLDivPosteriorPriorLoomOp,
self).__init__([emb_shape],
[cost_shape, emb_shape, emb_shape])
def __init__(self, num_options, tag_name='xent_cost'):
logits_shape = loom.TypeShape(tf.float32, (num_options, ))
labels_shape = loom.TypeShape(tf.int64, [])
output_shape = loom.TypeShape(tf.float32, [], tag_name)
super(SparseSoftmaxCrossEntropyLossLoomOp,
self).__init__([logits_shape, labels_shape], [output_shape])
def testAsLoomOpNames(self):
# Test that as_loom_op makes scopes unique in the order the decorators are
# called, not the order TensorLoom calls instantiate batch in.
n = 29
ts = loom.TypeShape(tf.int64, ())
def make_op(i):
@model_utils.as_loom_op([ts], ts)
def name(x):
var = tf.get_variable('var%d' % i,
shape=(),
dtype=tf.int64,
initializer=tf.constant_initializer(i))
return x + var
return name
# Make a loom with a bunch of ops, with names unrelated to the order
ops = {'op%d' % (i * 11 % n): make_op(i) for i in range(n)}
loom.Loom(named_ops=ops)
# Check that the variables were created in the right order
self.assertItemsEqual([('name_%d/var%d' % (i, i)).replace('_0', '')
for i in range(n)],
[v.op.name for v in tf.global_variables()])
def __init__(self, embedding_length):
self._embedding_length = embedding_length
self._named_tensors = {}
for n in xrange(10):
# Note: the examples only have the numbers 0 through 9 as terminal nodes.
name = 'terminal_' + str(n)
self._named_tensors[name] = tf.Variable(tf.truncated_normal(
[embedding_length], dtype=tf.float32, stddev=1),
name=name)
self._combiner_weights = {}
self._loom_ops = {}
for name in calculator_pb2.CalculatorExpression.OpCode.keys():
weights_var = tf.Variable(tf.truncated_normal(
[2 * embedding_length, embedding_length],
dtype=tf.float32,
stddev=1),
name=name)
self._combiner_weights[name] = weights_var
self._loom_ops[name] = CombineLoomOp(2, embedding_length,
weights_var)
self._loom = loom.Loom(named_tensors=self._named_tensors,
named_ops=self._loom_ops)
self._output = self._loom.output_tensor(
loom.TypeShape('float32', [embedding_length]))
def __init__(self, passthrough_types, tags):
self.passthrough_types = passthrough_types
in_ts = _get_typeshapes(passthrough_types)
out_ts = [
loom.TypeShape(ts.dtype, ts.shape, tag)
for ts, tag in zip(in_ts, tags)
]
super(TaggingPassThroughOp, self).__init__(in_ts, out_ts)
def __init__(self,
name,
num_options,
emb_size,
partitions=1,
input_tag=''):
input_shape = loom.TypeShape(tf.int64, [], input_tag)
output_shape = loom.TypeShape(tf.float32, (emb_size, ))
options_per_partition = (num_options - 1) // partitions + 1
self.embeddings = [
tf.get_variable(name + '_emb', [options_per_partition, emb_size],
initializer=tf.random_uniform_initializer(
-0.05, 0.05)) for _ in xrange(partitions)
]
super(EmbeddingLookupLoomOp, self).__init__([input_shape],
[output_shape])
def __init__(self, num_args, embedding_length, weights):
"""A LoomOp for recursive neural nets.
Args:
num_args: the number of inputs being fused.
embedding_length: the length of the input and output vectors.
weights: a (num_args * embedding_length) x embedding_length matrix.
"""
self._num_args = num_args
self._embedding_length = embedding_length
self._type_shape = loom.TypeShape('float32', (embedding_length, ))
self._weights = weights
self._input_type_shapes = [self._type_shape] * self._num_args
self._output_type_shapes = [self._type_shape]
def __init__(self, shape, dtype='float32'):
"""Creates a tensor type.
Args:
shape: A tuple or list of non-negative integers.
dtype: A `tf.DType`, or stringified version thereof (e.g. `'int64'`).
Raises:
TypeError: If `shape` is not a tuple or list of non-negative integers.
TypeError: If `dtype` cannot be converted to a TF dtype.
"""
if not isinstance(shape, (tuple, list)):
raise TypeError('shape must be a tuple or list: %s' % str(shape))
self._type_shape = loom.TypeShape(dtype, shape)
def register_tensor_type(self, tensor_type):
"""Ensures that loom can support tensor_type, even if it is not in an op.
Loom internally indexes tensors by TypeShape. By default the set
of TypeShapes that get indexed over is the union of all input and
output types from loom ops. This is generally sufficient, but
fails in the edge case of a constant having a unique TypeShape
that is used as an output, because there will be no op having such
a TypeShape. We handle this by explicitly registering all tensor
types as part of the block compilation process, and passing them
to the Loom constructor.
Args:
tensor_type: An instance of `td.TensorType`.
"""
type_shape = loom.TypeShape(tensor_type.dtype, tensor_type.shape)
self._extra_type_shapes.add(type_shape)
def __init__(self, name, emb_size, activation=tf.tanh, layer_norm=False):
emb_shape = loom.TypeShape(tf.float32, (emb_size, ))
with tf.variable_scope(name):
if layer_norm:
self.w_h = framework.variable('w_h', [emb_size, emb_size * 4])
self.w_x = framework.variable('w_x', [emb_size, emb_size * 4])
self.alpha_h = framework.variable(
'alpha_h', [emb_size * 4],
initializer=tf.constant_initializer(1.0))
self.alpha_x = framework.variable(
'alpha_x', [emb_size * 4],
initializer=tf.constant_initializer(1.0))
else:
self.w = framework.variable('w', [emb_size * 2, emb_size * 4])
self.b = framework.variable(
'b', [emb_size * 4], initializer=tf.constant_initializer(0.0))
self.layer_norm = layer_norm
self.activation = activation
super(LSTMLoomOp, self).__init__([emb_shape] * 3, [emb_shape] * 2)
def __init__(self, emb_size):
emb_shape = loom.TypeShape(tf.float32, (emb_size, ))
super(NegativeLogPdfWithMultivariateNormalLoomOp,
self).__init__([emb_shape, emb_shape, emb_shape],
[loom.TypeShape(tf.float32, ())])
def index_type():
return loom.TypeShape("int32", ())
def __init__(self, emb_size):
emb_shape = loom.TypeShape(tf.float32, (emb_size, ))
super(SampleFromGaussianLoomOp, self).__init__([emb_shape, emb_shape],
[emb_shape])
def __init__(self, emb_size):
self._prior = distributions.Normal(0., 1.)
super(LogPdfWithStandardNormalLoomOp,
self).__init__([loom.TypeShape(tf.float32, (emb_size, ))],
[loom.TypeShape(tf.float32, ())])
import os
import numpy as np
import tensorflow as tf
from tensorflow_fold.public import loom
from deepmath.eprover import prover_clause_examples_pb2
from deepmath.guidance import clause_loom
from deepmath.guidance import inputs
from deepmath.util import model_utils
FLAGS = tf.flags.FLAGS
# TypeShapes
VOCAB_ID = clause_loom.VOCAB_ID
DEPTH = loom.TypeShape(tf.int32, (), 'value')
COMBINATION = loom.TypeShape(tf.string, (2,), 'combination')
def embeddings(layers):
"""One scalar string TypeShape per layer."""
assert layers in (1, 2)
return [loom.TypeShape(tf.string, (), 'embedding%d' % i)
for i in range(layers)]
def assert_same(x, y):
check = tf.Assert(tf.reduce_all(tf.equal(x, y)), [x, y])
with tf.control_dependencies([check]):
return tf.identity(x)
def _asLoomOpTest(self, max_depth):
with tf.Graph().as_default() as graph:
tf.set_random_seed(8)
np.random.seed(7)
ts = loom.TypeShape(tf.int64, ())
initializer = tf.random_uniform_initializer(dtype=tf.int64,
minval=0,
maxval=1 << 60)
@model_utils.as_loom_op([ts, ts], ts)
def f(x, y):
# Use a variable to make sure variable sharing works correctly
rand = tf.get_variable('rand',
shape=(),
dtype=tf.int64,
initializer=initializer)
return rand - x - y
@model_utils.as_loom_op([ts, ts], [ts, ts])
def g(x, y):
# Test multiple outputs
return x - y, x + y
def make_h():
# Ensure that we can reuse names for different calls to as_loom_op.
# Also test that the name argument to as_loom_op works.
@model_utils.as_loom_op([ts], ts, name='h')
def not_h(x):
v = tf.get_variable('yo',
shape=(),
dtype=tf.int64,
initializer=initializer)
return x + v
return not_h
# Make two h's to ensure they make separate variables
h1 = make_h()
h2 = make_h()
simple_loom = loom.Loom(named_ops={
'f': f,
'g': g,
'h1': h1,
'h2': h2
},
max_depth=max_depth)
self.assertEqual(['f/rand', 'h/yo', 'h_1/yo'],
[v.op.name for v in tf.global_variables()])
# Use f twice and (g,h1,h2) once each
weaver = simple_loom.make_weaver()
x, y, z = np.random.randint(1 << 60, size=3)
wx, wy, wz = weaver(x), weaver(y), weaver(z)
weaver.add_output(weaver.f(wx, weaver.f(wy, wz)))
plus, minus = weaver.g(wx, wy)
weaver.add_output(plus)
weaver.add_output(minus)
weaver.add_output(weaver.h1(wx))
weaver.add_output(weaver.h2(wx))
with self.test_session(graph=graph):
tf.global_variables_initializer().run()
out = simple_loom.output_tensor(ts).eval(
weaver.build_feed_dict())
self.assertEqual(out.shape, (5, ))
# out[0] works only if variables are shared between layers:
# rand - x - (rand - y - z) = y + z - x
self.assertEqual(out[0], y + z - x)
# out[1] and out[2] are simple
self.assertEqual(out[1], x - y)
self.assertEqual(out[2], x + y)
# out[3] and out[4] should use different random variables
self.assertNotEqual(out[3], out[4])
def embeddings(layers):
"""One scalar string TypeShape per layer."""
assert layers in (1, 2)
return [loom.TypeShape(tf.string, (), 'embedding%d' % i)
for i in range(layers)]
def weave_clauses(examples,
vocab,
embed,
conjecture_apply,
conjecture_not,
conjecture_or,
conjecture_and,
clause_apply,
clause_not,
clause_or,
combine,
shuffle=True,
seed=None):
"""Weave serialized ProverClauseExamples using TensorLoom.
Computes results for a batch of ProverClauseExamples protos. For each
ProverClauseExamples with at least one positive and negative, one positive
and one negative are selected and reduced to results according to the given
loom ops.
In the description of the LoomOps below, vocab_id must be VOCAB_ID.
Args:
examples: 1-D `string` tensor of serialized `ProverClauseExamples`.
vocab: Path to vocabulary file.
embed: LoomOp to embed vocabulary ids: vocab_id -> embedding.
conjecture_apply: LoomOp for curried function application:
embedding -> embedding -> embedding.
conjecture_not: LoomOp for negation: embedding -> embedding.
conjecture_or: LoomOp for or: embedding -> embedding -> embedding.
conjecture_and: LooOp for and: embedding -> embedding -> embedding.
clause_apply: LoomOp for curried function application:
embedding -> embedding -> embedding.
clause_not: LoomOp for negation: embedding -> embedding.
clause_or: LoomOp for or: embedding -> embedding -> embedding.
combine : LoomOp to merge results:
conjecture_embedding -> clause_embedding -> combination.
shuffle: Whether to randomly shuffle ands and ors.
seed: Optional seed for random number generation.
Returns:
combinations: The results of combine for each positive and negative.
labels: The labels of each positive and negative.
"""
def weaver_op(**kwds):
seed1, seed2 = tf.get_seed(seed)
return gen_clause_ops.random_clauses_weaver(examples=examples,
vocab=vocab,
shuffle=shuffle,
seed=seed1,
seed2=seed2,
**kwds)
ops = {
'embed': embed,
'conjecture_apply': conjecture_apply,
'conjecture_not': conjecture_not,
'conjecture_or': conjecture_or,
'conjecture_and': conjecture_and,
'clause_apply': clause_apply,
'clause_not': clause_not,
'clause_or': clause_or,
'combine': combine
}
label = loom.TypeShape(tf.bool, (), 'label')
clause_loom = loom.Loom(named_ops=ops,
extra_type_shapes=[label],
weaver_op=weaver_op)
combination, = combine.output_type_shapes
return (clause_loom.output_tensor(combination),
clause_loom.output_tensor(label))
def loom_model(weave, vocab, hparams, mode):
"""Tree RNN model to compute logits from from ProverClauseExamples.
Args:
weave: Called with the loom op keyword arguments described in
clause_loom.weave_clauses.
vocab: Path to vocabulary file.
hparams: Hyperparameters.
mode: Either 'train' or 'eval'.
Returns:
The results of the call to `weave`.
"""
hidden_size = hparams.hidden_size
embedding_size = hparams.embedding_size
vocab_size, _ = inputs.read_vocab(vocab)
per_layer = 2 if hparams.cell == 'lstm' else 1
# TypeShapes
vocab_id = clause_loom.VOCAB_ID
logit = loom.TypeShape(tf.float32, (), 'logit')
# Use separate embedding type shapes for separate layers to avoid confusion.
# TODO(geoffreyi): Allow different sizes for different layers?
embeddings = tuple(
loom.TypeShape(tf.float32, (hidden_size,), 'embedding%d' % i)
for i in range(hparams.layers * per_layer))
@model_utils.as_loom_op([vocab_id], embeddings)
def embed(ids):
"""Embed tokens and use a linear layer to get the right size."""
values = model_utils.embedding_layer(ids, dim=embedding_size,
size=vocab_size)
if embedding_size < hidden_size:
values = layers.linear(values, hidden_size)
elif embedding_size > hidden_size:
raise ValueError('embedding_size = %d > hidden_size = %d' %
(embedding_size, hidden_size))
# Use relu layers to give one value per layer
values = [values]
for _ in range(hparams.layers - 1):
# TODO(geoffreyi): Should these be relu or some other activation?
values.append(layers.relu(values[-1], hidden_size))
# If we're using LSTMs, initialize the memory cells to zero.
if hparams.cell == 'lstm':
memory = tf.zeros_like(values[0])
values = [v for hidden in values for v in (memory, hidden)]
return values
def merge(arity, name):
"""Merge arity inputs with rule according to hparams.cell."""
@model_utils.as_loom_op(embeddings * arity, embeddings, name=name)
def merge(*args):
"""Process one batch of RNN inputs."""
# shape = (arity, layers) for RNNs, (arity, layers, 2) for LSTMs,
# where the 2 dimension is (memory, hidden).
shape = (arity, hparams.layers) + (per_layer,) * (per_layer > 1)
args = np.asarray(args).reshape(shape)
below = () # Information flowing up from the previous layer
outputs = [] # Results of each layer
if hparams.cell == 'rnn-relu':
# Vanilla RNN with relu nonlinearities
if hparams.keep_prob != 1:
raise ValueError('No dropout allowed for vanilla RNNs')
for layer in range(hparams.layers):
output = layers.relu(
tf.concat(below + tuple(args[:, layer]), 1), hidden_size)
outputs.append(output)
below = output,
elif hparams.cell == 'lstm':
# Tree LSTM with separate forget gates per input and optional recurrent
# dropout. For details, see
# 1. Improved semantic representations from tree-structured LSTM
# networks, http://arxiv.org/abs/1503.00075.
# 2. Recurrent dropout without memory loss,
# http://arxiv.org/abs/1603.05118.
# 3. http://colab/v2/notebook#fileId=0B2ewRpEjJXEFYjhtaExiZVBXbUk.
memory, hidden = np.rollaxis(args, axis=-1)
for layer in range(hparams.layers):
raw = layers.linear(
tf.concat(below + tuple(hidden[:, layer]), 1),
(3 + arity) * hidden_size)
raw = tf.split(value=raw, num_or_size_splits=3 + arity, axis=1)
(i, j, o), fs = raw[:3], raw[3:]
j = tf.tanh(j)
j = layers.dropout(j, keep_prob=hparams.keep_prob,
is_training=mode == 'train')
new_c = tf.add_n([tf.sigmoid(i) * j] +
[c * tf.sigmoid(f + hparams.forget_bias)
for c, f in zip(memory[:, layer], fs)])
new_h = tf.tanh(new_c) * tf.sigmoid(o)
outputs.extend((new_c, new_h))
below = new_h,
else:
# TODO(geoffreyi): Implement tree GRU?
raise ValueError('Unknown rnn cell type %r' % hparams.cell)
return outputs
return merge
@model_utils.as_loom_op(embeddings * 2, logit)
def classify(*args):
"""Compute logits from conjecture and clause embeddings."""
# Use the top layer, and either cell state, hidden state, or both
which = {'cell': 0, 'hidden': 1, 'both': (0, 1)}[hparams.output_mode]
args = np.asarray(args).reshape(2, hparams.layers, per_layer)
args = args[:, -1, which]
value = layers.relu(tf.concat(tuple(args.flat), 1), hidden_size)
return tf.squeeze(layers.linear(value, 1), [1])
return weave(
embed=embed,
conjecture_apply=merge(
2, name='conjecture/apply'),
conjecture_not=merge(
1, name='conjecture/not'),
conjecture_or=merge(
2, name='conjecture/or'),
conjecture_and=merge(
2, name='conjecture/and'),
clause_apply=merge(
2, name='clause/apply'),
clause_not=merge(
1, name='clause/not'),
clause_or=merge(
2, name='clause/or'),
combine=classify)
def _build_loom_types(self):
"""Method to build loom types for given setting.
"""
params = self.params
encode_size = params['lstm_size']
# create and save loom types
types = {}
types['time'] = loom.TypeShape('int32', (1,), 'time')
types['round'] = loom.TypeShape('int32', (1,), 'round')
types['float'] = loom.TypeShape('float32', (1,))
types['context'] = loom.TypeShape('float32', (encode_size,), 'context')
types['align'] = loom.TypeShape('float32', (encode_size,), 'align')
size = (params['num_rounds'], params['text_embed_size'])
types['fact'] = loom.TypeShape('float32', size, 'fact')
size = (params['num_rounds'], params['max_dec_len'],
params['text_embed_size'])
types['text'] = loom.TypeShape('float32', size, 'text')
types['caption'] = loom.TypeShape('float32', size, 'caption')
size = (params['text_embed_size'],)
types['text_slice'] = loom.TypeShape('float32', size, 'text_slice')
# this depends on whether we want all features
concat_dim = params['text_embed_size']
size = (params['num_rounds'], params['max_dec_len'], concat_dim)
types['text_feat'] = loom.TypeShape('float32', size, 'text_feat')
types['cap_feat'] = loom.TypeShape('float32', size, 'cap_feat')
size = (concat_dim,)
types['text_feat_slice'] = loom.TypeShape('float32', size, 'text_feat_slice')
# include spatial dimensions (x, y), add 2
size = (params['h_feat'], params['w_feat'], params['d_feat'] + 2)
types['image'] = loom.TypeShape('float32', size, 'image')
size = (params['h_feat'], params['w_feat'], 1)
types['attention'] = loom.TypeShape('float32', size, 'att')
return types
def __init__(self, dtype, shape, input_tag='', output_tag=''):
super(TagLoomOp,
self).__init__([loom.TypeShape(dtype, shape, input_tag)],
[loom.TypeShape(dtype, shape, output_tag)])
def __init__(self,
name,
num_inputs,
input_size,
num_outputs,
output_size,
hidden_sizes=(),
activation=tf.nn.relu,
hidden_activation=None,
layer_norm=False):
"""Constructor.
Args:
name: Name of the op, used to create variables.
num_inputs: Number of input vectors accepted by the LoomOp.
input_size: The length of each input vector.
num_outputs: Number of output vectors produced by the LoomOp.
output_size: The length of each output vector.
hidden_sizes: Sizes of hidden layers.
activation: Function to apply on last layer before
splitting into `num_outputs`.
hidden_activation: Function to apply to all but the last layer. Same as
`activation` if not specified.
layer_norm: If True, apply layer normalization before activation.
"""
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.layer_norm = layer_norm
self.num_layers = len(hidden_sizes) + 1
if hidden_activation is None:
hidden_activation = activation
self.activations = ((hidden_activation, ) * len(hidden_sizes) +
(activation, ))
hidden_sizes = tuple(hidden_sizes)
input_shape = loom.TypeShape(tf.float32, (input_size, ))
output_shape = loom.TypeShape(tf.float32, (output_size, ))
with tf.variable_scope(name):
if self.layer_norm:
dims = (input_size, ) + hidden_sizes + (num_outputs *
output_size, )
# For the first layer, we apply layer norm separately to each input
# (like what the paper did for GRUs and LSTMs).
# After that, there is only one "input" at each layer.
# There are num_layers + num_inputs - 1 weights.
inputs_per_layer = [xrange(self.num_inputs)
] + [1] * len(hidden_sizes)
self.w = [[
framework.variable('w_%d_layer_%d' % (i, l),
[input_dim, output_dim])
for i in xrange(num_inputs)
] for l, (num_inputs, input_dim, output_dim
) in enumerate(zip(inputs_per_layer, dims, dims[1:]))
]
self.alpha = [[
framework.variable(
'alpha_%d_layer_%d' % (i, l), [dim],
initializer=tf.constant_initializer(1.0))
for i in xrange(num_inputs)
] for l, (num_inputs,
dim) in enumerate(zip(inputs_per_layer, dims[1:]))]
else:
dims = ((num_inputs * input_size, ) + hidden_sizes +
(num_outputs * output_size, ))
self.w = [
framework.variable('w_layer_%d' % l,
[input_dim, output_dim])
for l, (input_dim,
output_dim) in enumerate(zip(dims, dims[1:]))
]
self.b = [
framework.variable('b_layer_%d' % l, [dim],
initializer=tf.constant_initializer(0.0))
for l, dim in enumerate(dims[1:])
]
super(LinearLoomOp, self).__init__([input_shape] * self.num_inputs,
[output_shape] * self.num_outputs)
def vector_type():
return loom.TypeShape("float32", (FLAGS.vector_size, ))
def test_conversion(self):
t = tdt.TensorType((1, 2), 'int32')
self.assertEqual(repr(t), 'TensorType((1, 2), \'int32\')')
self.assertEqual(np.ones_like(t).dtype, np.int32)
np.testing.assert_equal(np.ones_like(t), np.ones((1, 2)))
self.assertEqual(t._type_shape, loom.TypeShape('int32', (1, 2)))
def __init__(self,
name,
emb_size,
num_inputs,
num_outputs,
hidden_outputs=(),
activation=tf.nn.relu,
hidden_activation=None,
layer_norm=False,
gate_type='sigmoid',
tied_gates=False,
g_c_bias=-1.0,
g_x_bias=1.0):
"""Constructor.
Args:
name: Name of the op, used to create variables.
emb_size: The length of each input and output vector.
num_inputs: Number of input vectors accepted by the LoomOp.
num_outputs: Number of output vectors produced by the LoomOp.
hidden_outputs: Number of hidden output vectors in each layer.
activation: Function to apply to `c` at the last layer.
hidden_activation: Function to apply to `c` at all but the last layer.
Same as `activation` if not specified.
layer_norm: If True, apply layer normalization before activation.
gate_type: 'sigmoid' or 'softmax'. With 'softmax', gates on `x_i` and `c`
sum to 1. With 'sigmoid', each gate is between 0 and 1 but the values
do not sum to 1.
tied_gates: If True, the unactivated gate value for each x_i is identical
across all outputs. If False, it is different for each output.
g_c_bias: Added to `g_c` as a constant. Positive values imply greater
weight on `c`.
g_x_bias: Added to `g_x` as a constant. Positive values imply greater
weight on `x` (the inputs).
"""
self.emb_size = emb_size
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.activation = activation
self.layer_norm = layer_norm
self.gate_type = gate_type
self.tied_gates = tied_gates
self.g_c_bias = g_c_bias
self.g_x_bias = g_x_bias
emb_shape = loom.TypeShape(tf.float32, (emb_size, ))
hidden_outputs = tuple(hidden_outputs)
self.layer_counts = (num_inputs, ) + hidden_outputs + (num_outputs, )
self.intermediate_count = []
self.w, self.alpha, self.b = [], [], []
if hidden_activation is None:
hidden_activation = activation
self.activations = ([hidden_activation] * len(hidden_outputs) +
[activation])
with tf.variable_scope(name):
for l, (num_inputs, num_outputs) in enumerate(
zip(self.layer_counts, self.layer_counts[1:])):
# TODO(ricshin): If gate_type = softmax, num_inputs = 1, and num_outputs
# = 1, reduce gate count by 1 and use sigm, 1 - sigm instead.
if tied_gates:
intermediate_count = num_inputs + 2 * num_outputs
else:
intermediate_count = (num_inputs + 2) * num_outputs
self.intermediate_count.append(intermediate_count)
if self.layer_norm:
self.w.append([
framework.variable(
'w_%d_layer_%d' % (i, l),
[emb_size, intermediate_count * emb_size])
for i in xrange(num_inputs)
])
self.alpha.append([
framework.variable(
'alpha_%d_layer_%d' % (i, l),
[intermediate_count * emb_size],
initializer=tf.constant_initializer(1.0))
for i in xrange(num_inputs)
])
else:
self.w.append(
framework.variable('w_layer_%d' % l, [
num_inputs * emb_size,
intermediate_count * emb_size
]))
self.alpha.append(None)
self.b.append(
framework.variable(
'b_layer_%d' % l, [intermediate_count * emb_size],
initializer=tf.constant_initializer(0.0)))
super(GatedLinearLoomOp, self).__init__([emb_shape] * self.num_inputs,
[emb_shape] * self.num_outputs)
def __init__(self, emb_size):
named_tensors = {}
for n in xrange(10):
name = 'number_%d' % n
named_tensors[name] = tf.get_variable(name, [emb_size])
if FLAGS.root_emb == 'static':
named_tensors['root_emb'] = tf.get_variable('root_emb', [emb_size])
elif FLAGS.root_emb in ('random', 'vae'):
named_tensors['root_emb'] = tf.random_normal([emb_size])
named_ops = {}
for binop in ('plus', 'minus'):
named_ops['split_' + binop] = loom_ops.split(
'split_' + binop, 2, emb_size)
named_ops['merge_' + binop] = loom_ops.merge(
'merge_' + binop, 2, emb_size)
named_ops['split_number'] = loom_ops.split('split_number', 1, emb_size)
named_ops['merge_number'] = loom_ops.merge('merge_number', 1, emb_size)
named_ops['which_term'] = loom_ops.which('which_term', 3, emb_size)
named_ops['which_number'] = loom_ops.which('which_number', 10,
emb_size)
named_ops['sample_term'] = loom_ops.MultinomialLoomOp(
3, 'sampled_term')
named_ops['sample_number'] = loom_ops.MultinomialLoomOp(
10, 'sampled_number')
named_ops['kl_cost'] = loom_ops.KLDivPosteriorPriorLoomOp(emb_size)
named_ops['sample_z'] = loom_ops.SampleFromGaussianLoomOp(emb_size)
self.loom = loom.Loom(named_tensors=named_tensors, named_ops=named_ops)
# Setup the overall TensorFlow graph
self.term_labels = tf.placeholder(tf.int32, [None])
self.number_labels = tf.placeholder(tf.int32, [None])
self.term_outputs = self.loom.output_tensor(
loom.TypeShape(tf.float32, (3, )))
self.number_outputs = self.loom.output_tensor(
loom.TypeShape(tf.float32, (10, )))
# Used for sampling
self.sampled_terms = self.loom.output_tensor(
loom.TypeShape(tf.int64, (), 'sampled_term'))
self.sampled_numbers = self.loom.output_tensor(
loom.TypeShape(tf.int64, (), 'sampled_number'))
self.embs = self.loom.output_tensor(
loom.TypeShape(tf.float32, (emb_size, )))
term_losses = (
tf.contrib.nn.
deprecated_flipped_sparse_softmax_cross_entropy_with_logits( # pylint: disable=line-too-long
self.term_outputs, self.term_labels))
number_losses = (
tf.contrib.nn.
deprecated_flipped_sparse_softmax_cross_entropy_with_logits( # pylint: disable=line-too-long
self.number_outputs, self.number_labels))
self.loss = tf.reduce_mean(tf.concat([term_losses, number_losses], 0))
if FLAGS.root_emb == 'vae':
kl_div = self.loom.output_tensor(
loom.TypeShape(tf.float32, (), 'kl_div'))
self.kl_div_mean = tf.reduce_mean(kl_div)
self.loss += self.kl_div_mean
else:
self.kl_div_mean = tf.constant(0.0)
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Loom model processing ProverClauseExamples."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow_fold.public import loom
from deepmath.guidance import gen_clause_ops
# TypeShapes
VOCAB_ID = loom.TypeShape(tf.int64, (), 'vocab_id')
def weave_clauses(examples,
vocab,
embed,
conjecture_apply,
conjecture_not,
conjecture_or,
conjecture_and,
clause_apply,
clause_not,
clause_or,
combine,
shuffle=True,
seed=None):
