def test_explicit_dtype(self):
self.assertConverts('int64', tdt.TensorType((), 'int64'))
self.assertConverts(['int64'], tdt.TensorType((), 'int64'))
self.assertConverts([1, 'int64'], tdt.TensorType((1, ), 'int64'))
self.assertConverts([1, 2, 'int64'], tdt.TensorType((1, 2), 'int64'))
self.assertConverts([1, 2, 3, 'int64'],
tdt.TensorType((1, 2, 3), 'int64'))
def test_forward_declaration_orphaned_nested(self):
fwd1 = tdb.ForwardDeclaration(tdt.VoidType(), tdt.TensorType([]))
fwd2 = tdb.ForwardDeclaration(tdt.SequenceType(tdt.TensorType([])),
tdt.TensorType([]))
b = tdb.Map(tdb.Scalar()) >> fwd2() >> tdb.Function(tf.negative)
fwd2.resolve_to(tdb.Fold(tdb.Function(tf.add), fwd1()))
fwd1.resolve_to(tdb.FromTensor(tf.ones([])))
self.assertBuilds(-8., b, [3, 4], max_depth=3)
def test_conversion(self):
t = tdt.TupleType(tdt.TensorType(()), tdt.TensorType((1, ), 'int32'))
self.assertEqual(
repr(t),
'TupleType(TensorType((), \'float32\'), TensorType((1,), \'int32\'))'
)
self.assertEqual(repr(tdt.TupleType(tdt.TensorType(()))),
'TupleType(TensorType((), \'float32\'))')
def test_size(self):
scalar = tdt.TensorType(())
vector3 = tdt.TensorType((3, ))
seq = tdt.SequenceType(scalar)
self.assertEqual(tdt.TupleType().size, 0)
self.assertEqual(tdt.TupleType(scalar).size, 1)
self.assertEqual(tdt.TupleType(scalar, seq, scalar).size, None)
self.assertEqual(tdt.TupleType(scalar, vector3, scalar).size, 5)
def test_flatten_unflatten(self):
instance = [([(1, 2), (3, 4)], 5)]
t = tdt.SequenceType(
tdt.TupleType(
tdt.SequenceType(
tdt.TupleType(tdt.TensorType([6]), tdt.TensorType([7]))),
tdt.TensorType([8])))
self.assertEqual(list(t.terminal_types()),
list(tdt.convert_to_type(([6], [7], [8]))))
flat = t.flatten(instance)
self.assertEqual(flat, [1, 2, 3, 4, 5])
self.assertEqual(t.unflatten(iter(flat), [([(0, 0), (0, 0)], 0)]),
instance)
def test_flatten_unflatten(self):
t = tdt.BroadcastSequenceType(tdt.SequenceType(tdt.TensorType([])))
instance = itertools.repeat([1, 2])
self.assertEqual(t.flatten(instance), [1, 2])
unflat = t.unflatten(iter([1, 2]), itertools.repeat([0, 0]))
self.assertTrue(isinstance(unflat, itertools.repeat))
self.assertEqual(next(unflat), [1, 2])
def test_function_otype_inference_tuple_to_tuple(self):
infer = tdb._infer_tf_output_type_from_input_type
def f(x, y):
return [tf.matmul(x, y), tf.placeholder('int32', [None, 42])]
self.assertEqual(
tdt.TupleType(tdt.TensorType([1, 1]), tdt.TensorType([42], 'int32')),
infer(f, tdt.TupleType(tdt.TensorType([1, 1]), tdt.TensorType([1, 1]))))
self.assertEqual(
tdt.TupleType(tdt.TensorType([3, 5]), tdt.TensorType([42], 'int32')),
infer(f, tdt.TupleType(tdt.TensorType([3, 2]), tdt.TensorType([2, 5]))))
def test_function_otype_inference_tuple_to_tensor(self):
infer = tdb._infer_tf_output_type_from_input_type
f = tf.matmul
self.assertEqual(tdt.TensorType([1, 1]), infer(
f, tdt.TupleType(tdt.TensorType([1, 1]), tdt.TensorType([1, 1]))))
self.assertEqual(tdt.TensorType([3, 5]), infer(
f, tdt.TupleType(tdt.TensorType([3, 2]), tdt.TensorType([2, 5]))))
def test_pair(self):
self.assertConvertsPairs(
[[], [1], [1, 2], [1, 2, 3], 'int64', ['int64'], [1, 'int64'],
[1, 2, 'int64'], [1, 2, 3, 'int64']], [
tdt.TensorType(()),
tdt.TensorType((1, )),
tdt.TensorType((1, 2)),
tdt.TensorType((
1,
2,
3,
)),
tdt.TensorType((), 'int64'),
tdt.TensorType((), 'int64'),
tdt.TensorType((1, ), 'int64'),
tdt.TensorType((1, 2), 'int64'),
tdt.TensorType((1, 2, 3), 'int64')
])
def test_tuple(self):
t = tdt.TupleType(tdt.TensorType(()), tdt.TensorType((1, ), 'int32'))
self.assertEqual(tuple(t), (tdt.TensorType(
()), tdt.TensorType((1, ), 'int32')))
self.assertNotEqual(tuple(t), t)
self.assertEqual(len(t), 2)
self.assertEqual(t[0], tdt.TensorType(()))
self.assertEqual(t[:], t)
self.assertEqual(t[0:1], tdt.TupleType(tdt.TensorType(())))
def test_forward_declarations(self):
# Define a simple expression data structure
nlit = lambda x: {'op': 'lit', 'val': x}
nadd = lambda x, y: {'op': 'add', 'left': x, 'right': y}
nexpr = nadd(nadd(nlit(3.0), nlit(5.0)), nlit(2.0))
# Define a recursive block using forward declarations
expr_fwd = tdb.ForwardDeclaration(tdt.PyObjectType(),
tdt.TensorType((), 'float32'))
lit_case = tdb.GetItem('val') >> tdb.Scalar()
add_case = (tdb.Record({'left': expr_fwd(), 'right': expr_fwd()})
>> tdb.Function(tf.add))
expr = tdb.OneOf(lambda x: x['op'], {'lit': lit_case, 'add': add_case})
expr_fwd.resolve_to(expr)
self.assertBuilds(10.0, expr, nexpr, max_depth=2)
def __init__(self, num_units_out, activation=tf.nn.relu, initializer=None,
input_keep_prob=None, output_keep_prob=None,
normalization_fn=None, weight_norm=False, name=None):
"""Initializes the layer.
Args:
num_units_out: The number of output units in the layer.
activation: The activation function. Default is ReLU. Use `None` to get a
linear layer.
initializer: The initializer for the weights. Defaults to uniform unit
scaling with factor derived in <http://arxiv.org/pdf/1412.6558v3.pdf>
if activation is ReLU, ReLU6, tanh, or linear. Otherwise defaults to
truncated normal initialization with a standard deviation of 0.01.
input_keep_prob: Optional scalar float32 tensor for dropout on input.
Feed 1.0 at serving to disable dropout.
output_keep_prob: Optional scalar float32 tensor for dropout on output.
Feed 1.0 at serving to disable dropout.
normalization_fn: Optional normalization function that will be inserted
before nonlinearity.
weight_norm: A bool to control whether weight normalization is used. See
https://arxiv.org/abs/1602.07868 for how it works.
name: An optional string name. Defaults to `FC_%d % num_units_out`. Used
to name the variable scope where the variables for the layer live.
"""
self.set_constructor_args('td.FC', *get_local_arguments(FC.__init__, True))
if not initializer:
# TODO(SamEisenstat): This constant is calibrated for ReLU, something else
# might be better for ReLU6.
if activation in [tf.nn.relu, tf.nn.relu6]:
initializer = tf.uniform_unit_scaling_initializer(1.43)
elif activation == tf.tanh:
initializer = tf.uniform_unit_scaling_initializer(1.15)
elif not activation:
initializer = tf.uniform_unit_scaling_initializer(1.0)
else:
initializer = tf.truncated_normal_initializer(stddev=0.01)
self._activation = activation
self._initializer = initializer
self._input_keep_prob = input_keep_prob
self._output_keep_prob = output_keep_prob
self._normalization_fn = normalization_fn
self._weight_norm = weight_norm
if name is None: name = 'FC_%d' % num_units_out
super(FC, self).__init__(
output_type=tdt.TensorType([num_units_out]), name_or_scope=name)
def __init__(self, num_buckets, num_units_out, initializer=None, name=None,
trainable=True, mod_inputs=True, use_cpu=True):
"""Initializes the layer.
Args:
num_buckets: How many buckets the embedding has.
num_units_out: The number of output units in the layer.
initializer: the initializer for the weights. Defaults to uniform unit
scaling. The initializer can also be a Tensor or numpy array, in which
case the weights are initialized to this value and shape. Note that in
this case the weights will still be trainable unless you also pass
`trainable=False`.
name: An optional string name. Defaults to
`Embedding_%d_%d % (num_buckets, num_units_out)`. Used to name the
variable scope where the variables for the layer live.
trainable: Whether or not to make the weights trainable.
mod_inputs: Whether or not to mod the input by the number of buckets.
use_cpu: Whether to use cpu (Adagrad seems to work only with cpu for
sentiment)
Raises:
ValueError: If the shape of `weights` is not
`(num_buckets, num_units_out)`.
"""
self.set_constructor_args('td.Embedding',
*get_local_arguments(Embedding.__init__, True))
self._weights_shape = (num_buckets, num_units_out)
if name is None: name = 'Embedding_%d_%d' % self._weights_shape
if initializer is None:
initializer = tf.uniform_unit_scaling_initializer(1.0)
elif isinstance(initializer, np.ndarray):
initializer = tf.convert_to_tensor(initializer)
if isinstance(initializer, tf.Tensor):
initializer.set_shape(self._weights_shape)
self._weights_shape = None # otherwise get_variable barfs
self._initializer = initializer
self._num_buckets = num_buckets
self._num_units_out = num_units_out
self._trainable = trainable
self._mod_inputs = bool(mod_inputs)
self._use_cpu = use_cpu
super(Embedding, self).__init__(
output_type=tdt.TensorType([num_units_out]), name_or_scope=name)
def __init__(self,
num_units_out,
activation=tf.nn.relu,
initializer=None,
input_keep_prob=None,
output_keep_prob=None,
name=None):
"""Initializes the layer.
Args:
num_units_out: The number of output units in the layer.
activation: The activation function. Default is ReLU. Use `None` to get a
linear layer.
initializer: The initializer for the weights. Defaults to uniform unit
scaling with factor derived in <http://arxiv.org/pdf/1412.6558v3.pdf>
if activation is ReLU, ReLU6, tanh, or linear. Otherwise defaults to
truncated normal initialization with a standard deviation of 0.01.
input_keep_prob: Optional scalar float32 tensor for dropout on input.
Feed 1.0 at serving to disable dropout.
output_keep_prob: Optional scalar float32 tensor for dropout on output.
Feed 1.0 at serving to disable dropout.
name: An optional string name. Defaults to `FC_%d % num_units_out`. Used
to name the variable scope where the variables for the layer live.
"""
if not initializer:
# TODO(SamEisenstat): This constant is calibrated for ReLU, something else
# might be better for ReLU6.
if activation in [tf.nn.relu, tf.nn.relu6]:
initializer = tf.uniform_unit_scaling_initializer(1.43)
elif activation == tf.tanh:
initializer = tf.uniform_unit_scaling_initializer(1.15)
elif not activation:
initializer = tf.uniform_unit_scaling_initializer(1.0)
else:
initializer = tf.truncated_normal_initializer(stddev=0.01)
self._activation = activation
self._initializer = initializer
self._input_keep_prob = input_keep_prob
self._output_keep_prob = output_keep_prob
if name is None: name = 'FC_%d' % num_units_out
super(FC, self).__init__(output_type=tdt.TensorType([num_units_out]),
name_or_scope=name)
def __call__(self, batch):
"""Calls the function associated with this layer on a batch of inputs.
Creates the variables for this layer if they don't already exist.
Args:
batch: A batch tensor.
Returns:
A tensor of this layer's output type.
Raises:
ValueError: If the layer was previously called with a batch of a different
dtype or shape (not considering the leading dimension).
"""
self.set_input_type(
tdt.TensorType(batch.get_shape().as_list()[1:], batch.dtype))
self.create_variables()
with tf.variable_scope(self._vscope):
return self._process_batch(batch)
def test_hash(self):
self.assertHashes(tdt.SequenceType, tdt.TensorType(()))
def test_function_otype_inference_tensor_to_tensor(self):
infer = tdb._infer_tf_output_type_from_input_type
self.assertEqual(tdt.TensorType([]),
infer(tf.negative, tdt.TensorType([])))
self.assertEqual(tdt.TensorType([2, 3]),
infer(tf.negative, tdt.TensorType([2, 3])))
self.assertEqual(tdt.TensorType([], 'int32'),
infer(tf.negative, tdt.TensorType([], 'int32')))
self.assertEqual(tdt.TensorType([2, 3], 'int32'),
infer(tf.negative, tdt.TensorType([2, 3], 'int32')))
f = lambda x: tf.cast(x, 'int32')
self.assertEqual(tdt.TensorType([], 'int32'),
infer(f, tdt.TensorType([], 'float32')))
self.assertEqual(tdt.TensorType([2, 3], 'int32'),
infer(f, tdt.TensorType([2, 3], 'float64')))
def test_hash(self):
self.assertHashes(tdt.TupleType, tdt.TensorType(()))
def test_flatten_unflatten(self):
t = tdt.TensorType(())
self.assertEqual(list(t.terminal_types()), [t])
self.assertEqual(t.flatten(42), [42])
self.assertEqual(t.unflatten(iter([42]), None), 42)
def test_size(self):
self.assertEqual(tdt.TensorType(()).size, 1)
self.assertEqual(tdt.TensorType((1, 2, 3)).size, 6)
self.assertEqual(tdt.TensorType((1, 0, 3)).size, 0)
def test_conversion(self):
t = tdt.TensorType((1, 2), 'int32')
self.assertEqual(repr(t), 'TensorType((1, 2), \'int32\')')
self.assertEqual(np.ones_like(t).dtype, np.int32)
np.testing.assert_equal(np.ones_like(t), np.ones((1, 2)))
self.assertEqual(t._type_shape, loom.TypeShape('int32', (1, 2)))
def infer(result): itype = tdt.TensorType([]) f = lambda _: result return tdb._infer_tf_output_type_from_input_type(f, itype)
def test_forward_declaration_orphaned(self): fwd = tdb.ForwardDeclaration(tdt.VoidType(), tdt.TensorType([])) b = tdb.AllOf(fwd(), fwd()) >> tdb.Sum() fwd.resolve_to(tdb.FromTensor(tf.ones([]))) self.assertBuilds(2., b, None)
def test_default_dtype(self):
self.assertConverts([], tdt.TensorType(()))
self.assertConverts([1], tdt.TensorType((1, )))
self.assertConverts([1, 2], tdt.TensorType((1, 2)))
self.assertConverts([1, 2, 3], tdt.TensorType((1, 2, 3)))
def test_tensor_shape(self):
self.assertConverts(tf.TensorShape([]), tdt.TensorType(()))
self.assertConverts(tf.TensorShape([1]), tdt.TensorType((1, )))
self.assertConverts(tf.TensorShape([1, 2]), tdt.TensorType((1, 2)))
self.assertConverts(tf.TensorShape([1, 2, 3]), tdt.TensorType(
(1, 2, 3)))
def test_optional_default_none_type_inference(self): child = tdb.Scalar() >> tdb.Function(tf.negative) block = tdb.Optional(child) self.assertEqual(child.output_type, None) child.set_output_type([]) self.assertEqual(block.output_type, tdt.TensorType([]))
def test_zeros_void(self): block = tdb.Zeros(tdt.TupleType(tdt.VoidType(), tdt.TensorType(()))) self.assertBuildsConst((None, 0.0), block, None)
def test_terminal_types(self):
t = tdt.SequenceType(
tdt.TupleType(tdt.TensorType([]), tdt.VoidType(),
tdt.TupleType(tdt.PyObjectType())))
t_elem = t.element_type
self.assertEqual(list(t.terminal_types()), [t_elem[0], t_elem[2][0]])
def test_conversion(self):
t = tdt.SequenceType(tdt.TensorType((1, 2)))
self.assertEqual(repr(t),
'SequenceType(TensorType((1, 2), \'float32\'))')
self.assertEqual(t.element_type, tdt.TensorType((1, 2)))
def test_terminal_types(self):
t0 = tdt.TensorType([])
t1 = tdt.TensorType([1, 2])
t = tdt.TupleType(tdt.TupleType(t0),
tdt.TupleType(tdt.TupleType(t1, t1), t0))
self.assertEqual(list(t.terminal_types()), [t0, t1, t1, t0])
