def __init__(self, num_options, tag=''): input_shape = loom.TypeShape(tf.float32, (num_options, )) temperature_shape = loom.TypeShape(tf.float32, []) output_shape = loom.TypeShape(tf.int64, [], tag) super(MultinomialLoomOp, self).__init__([input_shape, temperature_shape], [output_shape])
def __init__(self, emb_size, reverse=False): """Constructor. Args: emb_size: Size of input. reverse: If True, compute KL(p(z) || q(z | x)) in addition. """ emb_shape = loom.TypeShape(tf.float32, (emb_size, )) cost_shape = loom.TypeShape(tf.float32, [], 'kl_div') self.w = framework.variable('q_z_x_w', [emb_size, emb_size * 2]) self.b = framework.variable('q_z_x_b', [emb_size * 2], initializer=tf.constant_initializer(0.0)) self.emb_size = emb_size self.reverse = reverse if reverse: rev_cost_shape = loom.TypeShape(tf.float32, [], 'rev_kl_div') super(KLDivPosteriorPriorLoomOp, self).__init__( [emb_shape], [cost_shape, rev_cost_shape, emb_shape, emb_shape]) else: super(KLDivPosteriorPriorLoomOp, self).__init__([emb_shape], [cost_shape, emb_shape, emb_shape])
def __init__(self, num_options, tag_name='xent_cost'): logits_shape = loom.TypeShape(tf.float32, (num_options, )) labels_shape = loom.TypeShape(tf.int64, []) output_shape = loom.TypeShape(tf.float32, [], tag_name) super(SparseSoftmaxCrossEntropyLossLoomOp, self).__init__([logits_shape, labels_shape], [output_shape])
def testAsLoomOpNames(self): # Test that as_loom_op makes scopes unique in the order the decorators are # called, not the order TensorLoom calls instantiate batch in. n = 29 ts = loom.TypeShape(tf.int64, ()) def make_op(i): @model_utils.as_loom_op([ts], ts) def name(x): var = tf.get_variable('var%d' % i, shape=(), dtype=tf.int64, initializer=tf.constant_initializer(i)) return x + var return name # Make a loom with a bunch of ops, with names unrelated to the order ops = {'op%d' % (i * 11 % n): make_op(i) for i in range(n)} loom.Loom(named_ops=ops) # Check that the variables were created in the right order self.assertItemsEqual([('name_%d/var%d' % (i, i)).replace('_0', '') for i in range(n)], [v.op.name for v in tf.global_variables()])
def __init__(self, embedding_length): self._embedding_length = embedding_length self._named_tensors = {} for n in xrange(10): # Note: the examples only have the numbers 0 through 9 as terminal nodes. name = 'terminal_' + str(n) self._named_tensors[name] = tf.Variable(tf.truncated_normal( [embedding_length], dtype=tf.float32, stddev=1), name=name) self._combiner_weights = {} self._loom_ops = {} for name in calculator_pb2.CalculatorExpression.OpCode.keys(): weights_var = tf.Variable(tf.truncated_normal( [2 * embedding_length, embedding_length], dtype=tf.float32, stddev=1), name=name) self._combiner_weights[name] = weights_var self._loom_ops[name] = CombineLoomOp(2, embedding_length, weights_var) self._loom = loom.Loom(named_tensors=self._named_tensors, named_ops=self._loom_ops) self._output = self._loom.output_tensor( loom.TypeShape('float32', [embedding_length]))
def __init__(self, passthrough_types, tags): self.passthrough_types = passthrough_types in_ts = _get_typeshapes(passthrough_types) out_ts = [ loom.TypeShape(ts.dtype, ts.shape, tag) for ts, tag in zip(in_ts, tags) ] super(TaggingPassThroughOp, self).__init__(in_ts, out_ts)
def __init__(self, name, num_options, emb_size, partitions=1, input_tag=''): input_shape = loom.TypeShape(tf.int64, [], input_tag) output_shape = loom.TypeShape(tf.float32, (emb_size, )) options_per_partition = (num_options - 1) // partitions + 1 self.embeddings = [ tf.get_variable(name + '_emb', [options_per_partition, emb_size], initializer=tf.random_uniform_initializer( -0.05, 0.05)) for _ in xrange(partitions) ] super(EmbeddingLookupLoomOp, self).__init__([input_shape], [output_shape])
def __init__(self, num_args, embedding_length, weights): """A LoomOp for recursive neural nets. Args: num_args: the number of inputs being fused. embedding_length: the length of the input and output vectors. weights: a (num_args * embedding_length) x embedding_length matrix. """ self._num_args = num_args self._embedding_length = embedding_length self._type_shape = loom.TypeShape('float32', (embedding_length, )) self._weights = weights self._input_type_shapes = [self._type_shape] * self._num_args self._output_type_shapes = [self._type_shape]
def __init__(self, shape, dtype='float32'): """Creates a tensor type. Args: shape: A tuple or list of non-negative integers. dtype: A `tf.DType`, or stringified version thereof (e.g. `'int64'`). Raises: TypeError: If `shape` is not a tuple or list of non-negative integers. TypeError: If `dtype` cannot be converted to a TF dtype. """ if not isinstance(shape, (tuple, list)): raise TypeError('shape must be a tuple or list: %s' % str(shape)) self._type_shape = loom.TypeShape(dtype, shape)
def register_tensor_type(self, tensor_type): """Ensures that loom can support tensor_type, even if it is not in an op. Loom internally indexes tensors by TypeShape. By default the set of TypeShapes that get indexed over is the union of all input and output types from loom ops. This is generally sufficient, but fails in the edge case of a constant having a unique TypeShape that is used as an output, because there will be no op having such a TypeShape. We handle this by explicitly registering all tensor types as part of the block compilation process, and passing them to the Loom constructor. Args: tensor_type: An instance of `td.TensorType`. """ type_shape = loom.TypeShape(tensor_type.dtype, tensor_type.shape) self._extra_type_shapes.add(type_shape)
def __init__(self, name, emb_size, activation=tf.tanh, layer_norm=False): emb_shape = loom.TypeShape(tf.float32, (emb_size, )) with tf.variable_scope(name): if layer_norm: self.w_h = framework.variable('w_h', [emb_size, emb_size * 4]) self.w_x = framework.variable('w_x', [emb_size, emb_size * 4]) self.alpha_h = framework.variable( 'alpha_h', [emb_size * 4], initializer=tf.constant_initializer(1.0)) self.alpha_x = framework.variable( 'alpha_x', [emb_size * 4], initializer=tf.constant_initializer(1.0)) else: self.w = framework.variable('w', [emb_size * 2, emb_size * 4]) self.b = framework.variable( 'b', [emb_size * 4], initializer=tf.constant_initializer(0.0)) self.layer_norm = layer_norm self.activation = activation super(LSTMLoomOp, self).__init__([emb_shape] * 3, [emb_shape] * 2)
def __init__(self, emb_size): emb_shape = loom.TypeShape(tf.float32, (emb_size, )) super(NegativeLogPdfWithMultivariateNormalLoomOp, self).__init__([emb_shape, emb_shape, emb_shape], [loom.TypeShape(tf.float32, ())])
def index_type(): return loom.TypeShape("int32", ())
def __init__(self, emb_size): emb_shape = loom.TypeShape(tf.float32, (emb_size, )) super(SampleFromGaussianLoomOp, self).__init__([emb_shape, emb_shape], [emb_shape])
def __init__(self, emb_size): self._prior = distributions.Normal(0., 1.) super(LogPdfWithStandardNormalLoomOp, self).__init__([loom.TypeShape(tf.float32, (emb_size, ))], [loom.TypeShape(tf.float32, ())])
import os import numpy as np import tensorflow as tf from tensorflow_fold.public import loom from deepmath.eprover import prover_clause_examples_pb2 from deepmath.guidance import clause_loom from deepmath.guidance import inputs from deepmath.util import model_utils FLAGS = tf.flags.FLAGS # TypeShapes VOCAB_ID = clause_loom.VOCAB_ID DEPTH = loom.TypeShape(tf.int32, (), 'value') COMBINATION = loom.TypeShape(tf.string, (2,), 'combination') def embeddings(layers): """One scalar string TypeShape per layer.""" assert layers in (1, 2) return [loom.TypeShape(tf.string, (), 'embedding%d' % i) for i in range(layers)] def assert_same(x, y): check = tf.Assert(tf.reduce_all(tf.equal(x, y)), [x, y]) with tf.control_dependencies([check]): return tf.identity(x)
def _asLoomOpTest(self, max_depth): with tf.Graph().as_default() as graph: tf.set_random_seed(8) np.random.seed(7) ts = loom.TypeShape(tf.int64, ()) initializer = tf.random_uniform_initializer(dtype=tf.int64, minval=0, maxval=1 << 60) @model_utils.as_loom_op([ts, ts], ts) def f(x, y): # Use a variable to make sure variable sharing works correctly rand = tf.get_variable('rand', shape=(), dtype=tf.int64, initializer=initializer) return rand - x - y @model_utils.as_loom_op([ts, ts], [ts, ts]) def g(x, y): # Test multiple outputs return x - y, x + y def make_h(): # Ensure that we can reuse names for different calls to as_loom_op. # Also test that the name argument to as_loom_op works. @model_utils.as_loom_op([ts], ts, name='h') def not_h(x): v = tf.get_variable('yo', shape=(), dtype=tf.int64, initializer=initializer) return x + v return not_h # Make two h's to ensure they make separate variables h1 = make_h() h2 = make_h() simple_loom = loom.Loom(named_ops={ 'f': f, 'g': g, 'h1': h1, 'h2': h2 }, max_depth=max_depth) self.assertEqual(['f/rand', 'h/yo', 'h_1/yo'], [v.op.name for v in tf.global_variables()]) # Use f twice and (g,h1,h2) once each weaver = simple_loom.make_weaver() x, y, z = np.random.randint(1 << 60, size=3) wx, wy, wz = weaver(x), weaver(y), weaver(z) weaver.add_output(weaver.f(wx, weaver.f(wy, wz))) plus, minus = weaver.g(wx, wy) weaver.add_output(plus) weaver.add_output(minus) weaver.add_output(weaver.h1(wx)) weaver.add_output(weaver.h2(wx)) with self.test_session(graph=graph): tf.global_variables_initializer().run() out = simple_loom.output_tensor(ts).eval( weaver.build_feed_dict()) self.assertEqual(out.shape, (5, )) # out[0] works only if variables are shared between layers: # rand - x - (rand - y - z) = y + z - x self.assertEqual(out[0], y + z - x) # out[1] and out[2] are simple self.assertEqual(out[1], x - y) self.assertEqual(out[2], x + y) # out[3] and out[4] should use different random variables self.assertNotEqual(out[3], out[4])
def embeddings(layers): """One scalar string TypeShape per layer.""" assert layers in (1, 2) return [loom.TypeShape(tf.string, (), 'embedding%d' % i) for i in range(layers)]
def weave_clauses(examples, vocab, embed, conjecture_apply, conjecture_not, conjecture_or, conjecture_and, clause_apply, clause_not, clause_or, combine, shuffle=True, seed=None): """Weave serialized ProverClauseExamples using TensorLoom. Computes results for a batch of ProverClauseExamples protos. For each ProverClauseExamples with at least one positive and negative, one positive and one negative are selected and reduced to results according to the given loom ops. In the description of the LoomOps below, vocab_id must be VOCAB_ID. Args: examples: 1-D `string` tensor of serialized `ProverClauseExamples`. vocab: Path to vocabulary file. embed: LoomOp to embed vocabulary ids: vocab_id -> embedding. conjecture_apply: LoomOp for curried function application: embedding -> embedding -> embedding. conjecture_not: LoomOp for negation: embedding -> embedding. conjecture_or: LoomOp for or: embedding -> embedding -> embedding. conjecture_and: LooOp for and: embedding -> embedding -> embedding. clause_apply: LoomOp for curried function application: embedding -> embedding -> embedding. clause_not: LoomOp for negation: embedding -> embedding. clause_or: LoomOp for or: embedding -> embedding -> embedding. combine : LoomOp to merge results: conjecture_embedding -> clause_embedding -> combination. shuffle: Whether to randomly shuffle ands and ors. seed: Optional seed for random number generation. Returns: combinations: The results of combine for each positive and negative. labels: The labels of each positive and negative. """ def weaver_op(**kwds): seed1, seed2 = tf.get_seed(seed) return gen_clause_ops.random_clauses_weaver(examples=examples, vocab=vocab, shuffle=shuffle, seed=seed1, seed2=seed2, **kwds) ops = { 'embed': embed, 'conjecture_apply': conjecture_apply, 'conjecture_not': conjecture_not, 'conjecture_or': conjecture_or, 'conjecture_and': conjecture_and, 'clause_apply': clause_apply, 'clause_not': clause_not, 'clause_or': clause_or, 'combine': combine } label = loom.TypeShape(tf.bool, (), 'label') clause_loom = loom.Loom(named_ops=ops, extra_type_shapes=[label], weaver_op=weaver_op) combination, = combine.output_type_shapes return (clause_loom.output_tensor(combination), clause_loom.output_tensor(label))
def loom_model(weave, vocab, hparams, mode): """Tree RNN model to compute logits from from ProverClauseExamples. Args: weave: Called with the loom op keyword arguments described in clause_loom.weave_clauses. vocab: Path to vocabulary file. hparams: Hyperparameters. mode: Either 'train' or 'eval'. Returns: The results of the call to `weave`. """ hidden_size = hparams.hidden_size embedding_size = hparams.embedding_size vocab_size, _ = inputs.read_vocab(vocab) per_layer = 2 if hparams.cell == 'lstm' else 1 # TypeShapes vocab_id = clause_loom.VOCAB_ID logit = loom.TypeShape(tf.float32, (), 'logit') # Use separate embedding type shapes for separate layers to avoid confusion. # TODO(geoffreyi): Allow different sizes for different layers? embeddings = tuple( loom.TypeShape(tf.float32, (hidden_size,), 'embedding%d' % i) for i in range(hparams.layers * per_layer)) @model_utils.as_loom_op([vocab_id], embeddings) def embed(ids): """Embed tokens and use a linear layer to get the right size.""" values = model_utils.embedding_layer(ids, dim=embedding_size, size=vocab_size) if embedding_size < hidden_size: values = layers.linear(values, hidden_size) elif embedding_size > hidden_size: raise ValueError('embedding_size = %d > hidden_size = %d' % (embedding_size, hidden_size)) # Use relu layers to give one value per layer values = [values] for _ in range(hparams.layers - 1): # TODO(geoffreyi): Should these be relu or some other activation? values.append(layers.relu(values[-1], hidden_size)) # If we're using LSTMs, initialize the memory cells to zero. if hparams.cell == 'lstm': memory = tf.zeros_like(values[0]) values = [v for hidden in values for v in (memory, hidden)] return values def merge(arity, name): """Merge arity inputs with rule according to hparams.cell.""" @model_utils.as_loom_op(embeddings * arity, embeddings, name=name) def merge(*args): """Process one batch of RNN inputs.""" # shape = (arity, layers) for RNNs, (arity, layers, 2) for LSTMs, # where the 2 dimension is (memory, hidden). shape = (arity, hparams.layers) + (per_layer,) * (per_layer > 1) args = np.asarray(args).reshape(shape) below = () # Information flowing up from the previous layer outputs = [] # Results of each layer if hparams.cell == 'rnn-relu': # Vanilla RNN with relu nonlinearities if hparams.keep_prob != 1: raise ValueError('No dropout allowed for vanilla RNNs') for layer in range(hparams.layers): output = layers.relu( tf.concat(below + tuple(args[:, layer]), 1), hidden_size) outputs.append(output) below = output, elif hparams.cell == 'lstm': # Tree LSTM with separate forget gates per input and optional recurrent # dropout. For details, see # 1. Improved semantic representations from tree-structured LSTM # networks, http://arxiv.org/abs/1503.00075. # 2. Recurrent dropout without memory loss, # http://arxiv.org/abs/1603.05118. # 3. http://colab/v2/notebook#fileId=0B2ewRpEjJXEFYjhtaExiZVBXbUk. memory, hidden = np.rollaxis(args, axis=-1) for layer in range(hparams.layers): raw = layers.linear( tf.concat(below + tuple(hidden[:, layer]), 1), (3 + arity) * hidden_size) raw = tf.split(value=raw, num_or_size_splits=3 + arity, axis=1) (i, j, o), fs = raw[:3], raw[3:] j = tf.tanh(j) j = layers.dropout(j, keep_prob=hparams.keep_prob, is_training=mode == 'train') new_c = tf.add_n([tf.sigmoid(i) * j] + [c * tf.sigmoid(f + hparams.forget_bias) for c, f in zip(memory[:, layer], fs)]) new_h = tf.tanh(new_c) * tf.sigmoid(o) outputs.extend((new_c, new_h)) below = new_h, else: # TODO(geoffreyi): Implement tree GRU? raise ValueError('Unknown rnn cell type %r' % hparams.cell) return outputs return merge @model_utils.as_loom_op(embeddings * 2, logit) def classify(*args): """Compute logits from conjecture and clause embeddings.""" # Use the top layer, and either cell state, hidden state, or both which = {'cell': 0, 'hidden': 1, 'both': (0, 1)}[hparams.output_mode] args = np.asarray(args).reshape(2, hparams.layers, per_layer) args = args[:, -1, which] value = layers.relu(tf.concat(tuple(args.flat), 1), hidden_size) return tf.squeeze(layers.linear(value, 1), [1]) return weave( embed=embed, conjecture_apply=merge( 2, name='conjecture/apply'), conjecture_not=merge( 1, name='conjecture/not'), conjecture_or=merge( 2, name='conjecture/or'), conjecture_and=merge( 2, name='conjecture/and'), clause_apply=merge( 2, name='clause/apply'), clause_not=merge( 1, name='clause/not'), clause_or=merge( 2, name='clause/or'), combine=classify)
def _build_loom_types(self): """Method to build loom types for given setting. """ params = self.params encode_size = params['lstm_size'] # create and save loom types types = {} types['time'] = loom.TypeShape('int32', (1,), 'time') types['round'] = loom.TypeShape('int32', (1,), 'round') types['float'] = loom.TypeShape('float32', (1,)) types['context'] = loom.TypeShape('float32', (encode_size,), 'context') types['align'] = loom.TypeShape('float32', (encode_size,), 'align') size = (params['num_rounds'], params['text_embed_size']) types['fact'] = loom.TypeShape('float32', size, 'fact') size = (params['num_rounds'], params['max_dec_len'], params['text_embed_size']) types['text'] = loom.TypeShape('float32', size, 'text') types['caption'] = loom.TypeShape('float32', size, 'caption') size = (params['text_embed_size'],) types['text_slice'] = loom.TypeShape('float32', size, 'text_slice') # this depends on whether we want all features concat_dim = params['text_embed_size'] size = (params['num_rounds'], params['max_dec_len'], concat_dim) types['text_feat'] = loom.TypeShape('float32', size, 'text_feat') types['cap_feat'] = loom.TypeShape('float32', size, 'cap_feat') size = (concat_dim,) types['text_feat_slice'] = loom.TypeShape('float32', size, 'text_feat_slice') # include spatial dimensions (x, y), add 2 size = (params['h_feat'], params['w_feat'], params['d_feat'] + 2) types['image'] = loom.TypeShape('float32', size, 'image') size = (params['h_feat'], params['w_feat'], 1) types['attention'] = loom.TypeShape('float32', size, 'att') return types
def __init__(self, dtype, shape, input_tag='', output_tag=''): super(TagLoomOp, self).__init__([loom.TypeShape(dtype, shape, input_tag)], [loom.TypeShape(dtype, shape, output_tag)])
def __init__(self, name, num_inputs, input_size, num_outputs, output_size, hidden_sizes=(), activation=tf.nn.relu, hidden_activation=None, layer_norm=False): """Constructor. Args: name: Name of the op, used to create variables. num_inputs: Number of input vectors accepted by the LoomOp. input_size: The length of each input vector. num_outputs: Number of output vectors produced by the LoomOp. output_size: The length of each output vector. hidden_sizes: Sizes of hidden layers. activation: Function to apply on last layer before splitting into `num_outputs`. hidden_activation: Function to apply to all but the last layer. Same as `activation` if not specified. layer_norm: If True, apply layer normalization before activation. """ self.num_inputs = num_inputs self.num_outputs = num_outputs self.layer_norm = layer_norm self.num_layers = len(hidden_sizes) + 1 if hidden_activation is None: hidden_activation = activation self.activations = ((hidden_activation, ) * len(hidden_sizes) + (activation, )) hidden_sizes = tuple(hidden_sizes) input_shape = loom.TypeShape(tf.float32, (input_size, )) output_shape = loom.TypeShape(tf.float32, (output_size, )) with tf.variable_scope(name): if self.layer_norm: dims = (input_size, ) + hidden_sizes + (num_outputs * output_size, ) # For the first layer, we apply layer norm separately to each input # (like what the paper did for GRUs and LSTMs). # After that, there is only one "input" at each layer. # There are num_layers + num_inputs - 1 weights. inputs_per_layer = [xrange(self.num_inputs) ] + [1] * len(hidden_sizes) self.w = [[ framework.variable('w_%d_layer_%d' % (i, l), [input_dim, output_dim]) for i in xrange(num_inputs) ] for l, (num_inputs, input_dim, output_dim ) in enumerate(zip(inputs_per_layer, dims, dims[1:])) ] self.alpha = [[ framework.variable( 'alpha_%d_layer_%d' % (i, l), [dim], initializer=tf.constant_initializer(1.0)) for i in xrange(num_inputs) ] for l, (num_inputs, dim) in enumerate(zip(inputs_per_layer, dims[1:]))] else: dims = ((num_inputs * input_size, ) + hidden_sizes + (num_outputs * output_size, )) self.w = [ framework.variable('w_layer_%d' % l, [input_dim, output_dim]) for l, (input_dim, output_dim) in enumerate(zip(dims, dims[1:])) ] self.b = [ framework.variable('b_layer_%d' % l, [dim], initializer=tf.constant_initializer(0.0)) for l, dim in enumerate(dims[1:]) ] super(LinearLoomOp, self).__init__([input_shape] * self.num_inputs, [output_shape] * self.num_outputs)
def vector_type(): return loom.TypeShape("float32", (FLAGS.vector_size, ))
def test_conversion(self): t = tdt.TensorType((1, 2), 'int32') self.assertEqual(repr(t), 'TensorType((1, 2), \'int32\')') self.assertEqual(np.ones_like(t).dtype, np.int32) np.testing.assert_equal(np.ones_like(t), np.ones((1, 2))) self.assertEqual(t._type_shape, loom.TypeShape('int32', (1, 2)))
def __init__(self, name, emb_size, num_inputs, num_outputs, hidden_outputs=(), activation=tf.nn.relu, hidden_activation=None, layer_norm=False, gate_type='sigmoid', tied_gates=False, g_c_bias=-1.0, g_x_bias=1.0): """Constructor. Args: name: Name of the op, used to create variables. emb_size: The length of each input and output vector. num_inputs: Number of input vectors accepted by the LoomOp. num_outputs: Number of output vectors produced by the LoomOp. hidden_outputs: Number of hidden output vectors in each layer. activation: Function to apply to `c` at the last layer. hidden_activation: Function to apply to `c` at all but the last layer. Same as `activation` if not specified. layer_norm: If True, apply layer normalization before activation. gate_type: 'sigmoid' or 'softmax'. With 'softmax', gates on `x_i` and `c` sum to 1. With 'sigmoid', each gate is between 0 and 1 but the values do not sum to 1. tied_gates: If True, the unactivated gate value for each x_i is identical across all outputs. If False, it is different for each output. g_c_bias: Added to `g_c` as a constant. Positive values imply greater weight on `c`. g_x_bias: Added to `g_x` as a constant. Positive values imply greater weight on `x` (the inputs). """ self.emb_size = emb_size self.num_inputs = num_inputs self.num_outputs = num_outputs self.activation = activation self.layer_norm = layer_norm self.gate_type = gate_type self.tied_gates = tied_gates self.g_c_bias = g_c_bias self.g_x_bias = g_x_bias emb_shape = loom.TypeShape(tf.float32, (emb_size, )) hidden_outputs = tuple(hidden_outputs) self.layer_counts = (num_inputs, ) + hidden_outputs + (num_outputs, ) self.intermediate_count = [] self.w, self.alpha, self.b = [], [], [] if hidden_activation is None: hidden_activation = activation self.activations = ([hidden_activation] * len(hidden_outputs) + [activation]) with tf.variable_scope(name): for l, (num_inputs, num_outputs) in enumerate( zip(self.layer_counts, self.layer_counts[1:])): # TODO(ricshin): If gate_type = softmax, num_inputs = 1, and num_outputs # = 1, reduce gate count by 1 and use sigm, 1 - sigm instead. if tied_gates: intermediate_count = num_inputs + 2 * num_outputs else: intermediate_count = (num_inputs + 2) * num_outputs self.intermediate_count.append(intermediate_count) if self.layer_norm: self.w.append([ framework.variable( 'w_%d_layer_%d' % (i, l), [emb_size, intermediate_count * emb_size]) for i in xrange(num_inputs) ]) self.alpha.append([ framework.variable( 'alpha_%d_layer_%d' % (i, l), [intermediate_count * emb_size], initializer=tf.constant_initializer(1.0)) for i in xrange(num_inputs) ]) else: self.w.append( framework.variable('w_layer_%d' % l, [ num_inputs * emb_size, intermediate_count * emb_size ])) self.alpha.append(None) self.b.append( framework.variable( 'b_layer_%d' % l, [intermediate_count * emb_size], initializer=tf.constant_initializer(0.0))) super(GatedLinearLoomOp, self).__init__([emb_shape] * self.num_inputs, [emb_shape] * self.num_outputs)
def __init__(self, emb_size): named_tensors = {} for n in xrange(10): name = 'number_%d' % n named_tensors[name] = tf.get_variable(name, [emb_size]) if FLAGS.root_emb == 'static': named_tensors['root_emb'] = tf.get_variable('root_emb', [emb_size]) elif FLAGS.root_emb in ('random', 'vae'): named_tensors['root_emb'] = tf.random_normal([emb_size]) named_ops = {} for binop in ('plus', 'minus'): named_ops['split_' + binop] = loom_ops.split( 'split_' + binop, 2, emb_size) named_ops['merge_' + binop] = loom_ops.merge( 'merge_' + binop, 2, emb_size) named_ops['split_number'] = loom_ops.split('split_number', 1, emb_size) named_ops['merge_number'] = loom_ops.merge('merge_number', 1, emb_size) named_ops['which_term'] = loom_ops.which('which_term', 3, emb_size) named_ops['which_number'] = loom_ops.which('which_number', 10, emb_size) named_ops['sample_term'] = loom_ops.MultinomialLoomOp( 3, 'sampled_term') named_ops['sample_number'] = loom_ops.MultinomialLoomOp( 10, 'sampled_number') named_ops['kl_cost'] = loom_ops.KLDivPosteriorPriorLoomOp(emb_size) named_ops['sample_z'] = loom_ops.SampleFromGaussianLoomOp(emb_size) self.loom = loom.Loom(named_tensors=named_tensors, named_ops=named_ops) # Setup the overall TensorFlow graph self.term_labels = tf.placeholder(tf.int32, [None]) self.number_labels = tf.placeholder(tf.int32, [None]) self.term_outputs = self.loom.output_tensor( loom.TypeShape(tf.float32, (3, ))) self.number_outputs = self.loom.output_tensor( loom.TypeShape(tf.float32, (10, ))) # Used for sampling self.sampled_terms = self.loom.output_tensor( loom.TypeShape(tf.int64, (), 'sampled_term')) self.sampled_numbers = self.loom.output_tensor( loom.TypeShape(tf.int64, (), 'sampled_number')) self.embs = self.loom.output_tensor( loom.TypeShape(tf.float32, (emb_size, ))) term_losses = ( tf.contrib.nn. deprecated_flipped_sparse_softmax_cross_entropy_with_logits( # pylint: disable=line-too-long self.term_outputs, self.term_labels)) number_losses = ( tf.contrib.nn. deprecated_flipped_sparse_softmax_cross_entropy_with_logits( # pylint: disable=line-too-long self.number_outputs, self.number_labels)) self.loss = tf.reduce_mean(tf.concat([term_losses, number_losses], 0)) if FLAGS.root_emb == 'vae': kl_div = self.loom.output_tensor( loom.TypeShape(tf.float32, (), 'kl_div')) self.kl_div_mean = tf.reduce_mean(kl_div) self.loss += self.kl_div_mean else: self.kl_div_mean = tf.constant(0.0)
# See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Loom model processing ProverClauseExamples.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import tensorflow as tf from tensorflow_fold.public import loom from deepmath.guidance import gen_clause_ops # TypeShapes VOCAB_ID = loom.TypeShape(tf.int64, (), 'vocab_id') def weave_clauses(examples, vocab, embed, conjecture_apply, conjecture_not, conjecture_or, conjecture_and, clause_apply, clause_not, clause_or, combine, shuffle=True, seed=None):