def test_lstm_cell(self):
# Test to make sure examples from the documentation compile.
td = tdb
num_hidden = 32
# Create an LSTM cell, the hard way.
lstm_cell = td.Composition()
with lstm_cell.scope():
in_state = td.Identity().reads(lstm_cell.input[1])
bx = td.Concat().reads(lstm_cell.input[0], in_state[1])
bi = td.Function(tdl.FC(num_hidden, tf.nn.sigmoid)).reads(bx)
bf = td.Function(tdl.FC(num_hidden, tf.nn.sigmoid)).reads(bx)
bo = td.Function(tdl.FC(num_hidden, tf.nn.sigmoid)).reads(bx)
bg = td.Function(tdl.FC(num_hidden, tf.nn.tanh)).reads(bx)
bc = td.Function(lambda c, i, f, g: c*f + i*g).reads(
in_state[0], bi, bf, bg)
by = td.Function(lambda c, o: tf.tanh(c) * o).reads(bc, bo)
out_state = td.Identity().reads(bc, by)
lstm_cell.output.reads(by, out_state)
str_lstm = (td.InputTransform(lambda s: [ord(c) for c in s]) >>
td.Map(td.Scalar('int32') >>
td.Function(tdl.Embedding(128, 16))) >>
td.RNN(lstm_cell,
initial_state=td.AllOf(tf.zeros(32), tf.zeros(32))))
with self.test_session():
str_lstm.eval('The quick brown fox.')
def test_fc_raises(self):
six.assertRaisesRegex(self, TypeError,
'FC input dtype must be float32', tdl.FC(1),
tf.constant([0], dtype='int64'))
six.assertRaisesRegex(self, TypeError, 'FC input shape must be 1D',
tdl.FC(1), tf.constant(0, dtype='float32'))
fc = tdl.FC(1)
fc(tf.constant([[0]], 'float32'))
six.assertRaisesRegex(self, TypeError,
'Type mismatch between input type', fc,
tf.constant([[0, 0]], 'float32'))
def test_tf_dropout(self):
tf.set_random_seed(123)
with self.test_session() as sess:
in_kp = tf.placeholder(tf.float32)
out_kp = tf.placeholder(tf.float32)
fc = tdl.FC(1,
None,
tf.constant_initializer(2.),
input_keep_prob=in_kp,
output_keep_prob=out_kp)
out = fc(tf.ones([1, 1]))
sess.run(tf.global_variables_initializer())
self.assertAllEqual([[2]], sess.run(out, {
in_kp: 1.0,
out_kp: 1.0
}))
self.assertAllEqual([[0]],
sess.run(out, {
in_kp: 1e-10,
out_kp: 1.0
})) # kp=0 -> NaN
self.assertAllEqual([[0]],
sess.run(out, {
in_kp: 1.0,
out_kp: 1e-10
})) # kp=0 -> NaN
def test_fractalnet_smoketest_random_drop_path(self):
input_placeholder = tf.placeholder(tf.float32, [None, 3])
output_placeholder = tf.placeholder(tf.float32, [None, 3])
fractal_net = tdl.FractalNet(3,
2,
lambda name: tdl.FC(3, name=name),
drop_path=True)
result = fractal_net(input_placeholder)
loss = tf.nn.l2_loss(result - output_placeholder)
optr = tf.train.GradientDescentOptimizer(0.001)
trainer = optr.minimize(loss)
dataset = np.random.standard_normal([10, 3])
answers = np.random.standard_normal([10, 3])
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
feed_dict = {
input_placeholder: dataset,
output_placeholder: answers
}
np.random.seed(42)
fractal_net.drop_path = False
old_loss = loss.eval(feed_dict)
for unused_iteration in range(100):
fractal_net.drop_path = True
sess.run([trainer], feed_dict)
fractal_net.drop_path = False
new_loss = loss.eval(feed_dict)
self.assertLess(new_loss, old_loss)
def test_fc_linear(self):
fc = tdl.FC(1, None, tf.constant_initializer(2.0))
with self.test_session() as sess:
out = [
fc(tf.constant([x], 'float32'))
for x in [[0, 0], [1, 0], [1, 1]]
]
sess.run(tf.global_variables_initializer())
self.assertAllEqual([[[0]], [[2]], [[4]]], sess.run(out))
def test_fc_normalization(self):
def fn(inp):
return tf.div(inp, tf.constant([2.0], 'float32'))
fc = tdl.FC(2,
normalization_fn=fn,
initializer=tf.constant_initializer(4.0))
with self.test_session() as sess:
out = [fc(tf.constant([x], 'float32')) for x in [[-1], [1]]]
sess.run(tf.global_variables_initializer())
self.assertAllEqual([[[0, 0]], [[2, 2]]], sess.run(out))
def test_record_doc_example(self):
# Test to make sure examples from the documentation compile.
example_datum = {'id': 8,
'name': 'Joe Smith',
'location': (2.5, 7.0)}
num_ids = 16
embed_len = 16
td = tdb
char_rnn = (td.InputTransform(lambda s: [ord(c) for c in s]) >>
td.Map(td.Scalar('int32') >>
td.Function(tdl.Embedding(128, 16))) >>
td.Fold(td.Concat() >> td.Function(tdl.FC(32)),
td.FromTensor(tf.zeros(32))))
r = (td.Record([('id', (td.Scalar('int32') >>
td.Function(tdl.Embedding(num_ids, embed_len)))),
('name', char_rnn),
('location', td.Vector(2))])
>> td.Concat() >> td.Function(tdl.FC(256)))
with self.test_session():
r.eval(example_datum)
def test_fc_linear_weight_norm(self):
fc = tdl.FC(1, None, tf.constant_initializer(2.0), weight_norm=True)
normalized_weight = 2.0 / math.sqrt(8.0)
with self.test_session() as sess:
out = [
fc(tf.constant([x], 'float32'))
for x in [[0, 0], [1, 0], [1, 1]]
]
sess.run(tf.global_variables_initializer())
np.testing.assert_almost_equal(
[[[0]], [[normalized_weight]], [[2 * normalized_weight]]],
sess.run(out))
sess.run(fc.scales.assign([2.0]))
np.testing.assert_almost_equal(
[[[0]], [[2 * normalized_weight]], [[4 * normalized_weight]]],
sess.run(out))
def test_fc_relu(self):
fc = tdl.FC(2, initializer=tf.constant_initializer(3.0))
with self.test_session() as sess:
out = [fc(tf.constant([x], 'float32')) for x in [[-1], [1]]]
sess.run(tf.global_variables_initializer())
self.assertAllEqual([[[0, 0]], [[3, 3]]], sess.run(out))
def test_rrshift(self):
block = tf.constant([3.0]) >> tdl.FC(1, None,
tf.constant_initializer(2.0))
with self.test_session():
self.assertEqual([6.0], block.eval(None, tolist=True))
def test_rshift(self):
block = (tdl.FC(1, None, tf.constant_initializer(2.0)) >> tdl.FC(
1, None, tf.constant_initializer(3.0)))
with self.test_session():
self.assertEqual([6.0], (tdb.Vector(1) >> block).eval([1.0],
tolist=True))
