def test_lstm_cell(self): # Test to make sure examples from the documentation compile. td = tdb num_hidden = 32 # Create an LSTM cell, the hard way. lstm_cell = td.Composition() with lstm_cell.scope(): in_state = td.Identity().reads(lstm_cell.input[1]) bx = td.Concat().reads(lstm_cell.input[0], in_state[1]) bi = td.Function(tdl.FC(num_hidden, tf.nn.sigmoid)).reads(bx) bf = td.Function(tdl.FC(num_hidden, tf.nn.sigmoid)).reads(bx) bo = td.Function(tdl.FC(num_hidden, tf.nn.sigmoid)).reads(bx) bg = td.Function(tdl.FC(num_hidden, tf.nn.tanh)).reads(bx) bc = td.Function(lambda c, i, f, g: c*f + i*g).reads( in_state[0], bi, bf, bg) by = td.Function(lambda c, o: tf.tanh(c) * o).reads(bc, bo) out_state = td.Identity().reads(bc, by) lstm_cell.output.reads(by, out_state) str_lstm = (td.InputTransform(lambda s: [ord(c) for c in s]) >> td.Map(td.Scalar('int32') >> td.Function(tdl.Embedding(128, 16))) >> td.RNN(lstm_cell, initial_state=td.AllOf(tf.zeros(32), tf.zeros(32)))) with self.test_session(): str_lstm.eval('The quick brown fox.')
def test_fc_raises(self): six.assertRaisesRegex(self, TypeError, 'FC input dtype must be float32', tdl.FC(1), tf.constant([0], dtype='int64')) six.assertRaisesRegex(self, TypeError, 'FC input shape must be 1D', tdl.FC(1), tf.constant(0, dtype='float32')) fc = tdl.FC(1) fc(tf.constant([[0]], 'float32')) six.assertRaisesRegex(self, TypeError, 'Type mismatch between input type', fc, tf.constant([[0, 0]], 'float32'))
def test_tf_dropout(self): tf.set_random_seed(123) with self.test_session() as sess: in_kp = tf.placeholder(tf.float32) out_kp = tf.placeholder(tf.float32) fc = tdl.FC(1, None, tf.constant_initializer(2.), input_keep_prob=in_kp, output_keep_prob=out_kp) out = fc(tf.ones([1, 1])) sess.run(tf.global_variables_initializer()) self.assertAllEqual([[2]], sess.run(out, { in_kp: 1.0, out_kp: 1.0 })) self.assertAllEqual([[0]], sess.run(out, { in_kp: 1e-10, out_kp: 1.0 })) # kp=0 -> NaN self.assertAllEqual([[0]], sess.run(out, { in_kp: 1.0, out_kp: 1e-10 })) # kp=0 -> NaN
def test_fractalnet_smoketest_random_drop_path(self): input_placeholder = tf.placeholder(tf.float32, [None, 3]) output_placeholder = tf.placeholder(tf.float32, [None, 3]) fractal_net = tdl.FractalNet(3, 2, lambda name: tdl.FC(3, name=name), drop_path=True) result = fractal_net(input_placeholder) loss = tf.nn.l2_loss(result - output_placeholder) optr = tf.train.GradientDescentOptimizer(0.001) trainer = optr.minimize(loss) dataset = np.random.standard_normal([10, 3]) answers = np.random.standard_normal([10, 3]) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) feed_dict = { input_placeholder: dataset, output_placeholder: answers } np.random.seed(42) fractal_net.drop_path = False old_loss = loss.eval(feed_dict) for unused_iteration in range(100): fractal_net.drop_path = True sess.run([trainer], feed_dict) fractal_net.drop_path = False new_loss = loss.eval(feed_dict) self.assertLess(new_loss, old_loss)
def test_fc_linear(self): fc = tdl.FC(1, None, tf.constant_initializer(2.0)) with self.test_session() as sess: out = [ fc(tf.constant([x], 'float32')) for x in [[0, 0], [1, 0], [1, 1]] ] sess.run(tf.global_variables_initializer()) self.assertAllEqual([[[0]], [[2]], [[4]]], sess.run(out))
def test_fc_normalization(self): def fn(inp): return tf.div(inp, tf.constant([2.0], 'float32')) fc = tdl.FC(2, normalization_fn=fn, initializer=tf.constant_initializer(4.0)) with self.test_session() as sess: out = [fc(tf.constant([x], 'float32')) for x in [[-1], [1]]] sess.run(tf.global_variables_initializer()) self.assertAllEqual([[[0, 0]], [[2, 2]]], sess.run(out))
def test_record_doc_example(self): # Test to make sure examples from the documentation compile. example_datum = {'id': 8, 'name': 'Joe Smith', 'location': (2.5, 7.0)} num_ids = 16 embed_len = 16 td = tdb char_rnn = (td.InputTransform(lambda s: [ord(c) for c in s]) >> td.Map(td.Scalar('int32') >> td.Function(tdl.Embedding(128, 16))) >> td.Fold(td.Concat() >> td.Function(tdl.FC(32)), td.FromTensor(tf.zeros(32)))) r = (td.Record([('id', (td.Scalar('int32') >> td.Function(tdl.Embedding(num_ids, embed_len)))), ('name', char_rnn), ('location', td.Vector(2))]) >> td.Concat() >> td.Function(tdl.FC(256))) with self.test_session(): r.eval(example_datum)
def test_fc_linear_weight_norm(self): fc = tdl.FC(1, None, tf.constant_initializer(2.0), weight_norm=True) normalized_weight = 2.0 / math.sqrt(8.0) with self.test_session() as sess: out = [ fc(tf.constant([x], 'float32')) for x in [[0, 0], [1, 0], [1, 1]] ] sess.run(tf.global_variables_initializer()) np.testing.assert_almost_equal( [[[0]], [[normalized_weight]], [[2 * normalized_weight]]], sess.run(out)) sess.run(fc.scales.assign([2.0])) np.testing.assert_almost_equal( [[[0]], [[2 * normalized_weight]], [[4 * normalized_weight]]], sess.run(out))
def test_fc_relu(self): fc = tdl.FC(2, initializer=tf.constant_initializer(3.0)) with self.test_session() as sess: out = [fc(tf.constant([x], 'float32')) for x in [[-1], [1]]] sess.run(tf.global_variables_initializer()) self.assertAllEqual([[[0, 0]], [[3, 3]]], sess.run(out))
def test_rrshift(self): block = tf.constant([3.0]) >> tdl.FC(1, None, tf.constant_initializer(2.0)) with self.test_session(): self.assertEqual([6.0], block.eval(None, tolist=True))
def test_rshift(self): block = (tdl.FC(1, None, tf.constant_initializer(2.0)) >> tdl.FC( 1, None, tf.constant_initializer(3.0))) with self.test_session(): self.assertEqual([6.0], (tdb.Vector(1) >> block).eval([1.0], tolist=True))