def test_device_instance(self): dev = Device.get_default() self.assertIs(dev, self.device) tensor = tF.input([0], Shape([])) dev = tensor.device() self.assertIs(dev, self.device) node = F.input([0], Shape([])) dev = node.device() self.assertIs(dev, self.device) my_device = Naive() self.assertIsNot(my_device, self.device) node = F.input([0], Shape([]), device=my_device) dev = node.device() self.assertIs(dev, my_device) dev = self.graph.get_device(node) self.assertIs(dev, my_device) param = Parameter(Shape([])) dev = param.device() self.assertIs(dev, self.device)
def test_TensorTest_CheckInplaceSubtractNN(self): for dev in TensorTest.devices: a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] b_data = [0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9] y_data = [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] a = tF.raw_input(Shape([2, 2], 3), a_data, dev) b = tF.raw_input(Shape([2, 2], 3), b_data, dev) a -= b self.assertEqual(y_data, a.to_list())
def test_Parameter_argument(self): # no argument p = Parameter() self.assertFalse(p.valid()) # shape w/ Initializer p = Parameter(Shape([4, 3]), I.Constant(1)) self.assertEqual(p.shape(), Shape([4, 3])) self.assertEqual(p.value.to_list(), [1] * 12)
def test_TensorTest_CheckInplaceAddNN(self): for dev in TensorTest.devices: a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] b_data = [0, -1, -2, -3, -3, -4, -5, -6, -6, -7, -8, -9] y_data = [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] a = tF.raw_input(Shape([2, 2], 3), a_data, dev) b = tF.raw_input(Shape([2, 2], 3), b_data, dev) a += b self.assertEqual(y_data, a.to_list())
def test_Parameter_argument(self): # shape w/o data p = Parameter(Shape([2, 3])) self.assertEqual(p.shape(), Shape([2, 3])) # shape w/ Initializer p = Parameter(Shape([4, 3]), I.Constant(1)) self.assertEqual(p.shape(), Shape([4, 3])) self.assertEqual(p.value.to_list(), [1] * 12) # shape w/ list[float] p = Parameter(Shape([4, 3]), self.list_data[:12]) self.assertEqual(p.shape(), Shape([4, 3])) self.assertEqual(p.value.to_list(), self.list_data[:12]) # ndarray w/o shape p = Parameter(init=self.ndarray_data[0]) self.assertEqual(p.shape(), Shape([4, 3])) self.assertEqual(p.value.to_list(), self.list_data[:12]) # ndarray w/ shape p = Parameter(Shape([2, 6]), init=self.ndarray_data[0]) self.assertEqual(p.shape(), Shape([2, 6])) self.assertEqual(p.value.to_list(), self.list_data[:12]) # list[float] w/o shape self.assertRaises(TypeError, lambda: Parameter(init=self.list_data[:12]))
def test_TensorTest_CheckCopyValidToNew(self): for dev in TensorTest.devices: print(dev) tmp = tF.raw_input(Shape([2], 3), [1, 2, 3, 4, 5, 6], dev) x = Tensor(tmp) self.assertTrue(x.valid()) self.assertTrue(tmp.valid()) self.assertEqual(Shape([2], 3), x.shape()) self.assertEqual(Shape([2], 3), tmp.shape()) self.assertEqual([1, 2, 3, 4, 5, 6], x.to_list()) self.assertEqual([1, 2, 3, 4, 5, 6], tmp.to_list())
def test_TensorTest_InplaceMultiplyConst(self): for dev in TensorTest.devices: x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] y_data = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24] x = tF.raw_input(Shape([2, 2], 3), x_data, dev) x *= 2 self.assertEqual(y_data, x.to_list()) for dev in TensorTest.devices: x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] y_data = [.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5, 6] x = tF.raw_input(Shape([2, 2], 3), x_data, dev) x *= .5 self.assertEqual(y_data, x.to_list())
def test_operators_input_argument(self): # list[ndarray] w/o shape x = F.input(self.ndarray_data) self.assertEqual(x.to_list(), self.list_data) self.assertEqual(x.shape(), Shape([4, 3], 2)) # list[ndarray] w/ shape x = F.input(self.ndarray_data, Shape([2, 3], 4)) self.assertEqual(x.to_list(), self.list_data) self.assertEqual(x.shape(), Shape([2, 3], 4)) # ndarray w/o shape x = F.input(self.ndarray_data[0]) self.assertEqual(x.to_list(), self.list_data[:12]) self.assertEqual(x.shape(), Shape([4, 3], 1)) # ndarray w/ shape x = F.input(self.ndarray_data[0], Shape([2, 3], 2)) self.assertEqual(x.to_list(), self.list_data[:12]) self.assertEqual(x.shape(), Shape([2, 3], 2)) # list[float] w/o shape self.assertRaises(TypeError, lambda: F.input(self.list_data)) # list[float] w/ shape x = F.input(self.list_data, shape=Shape([4, 3], 2)) self.assertEqual(x.to_list(), self.list_data) self.assertEqual(x.shape(), Shape([4, 3], 2))
def test_TesnorTest_CheckResetValuesByConstant(self): for dev in TensorTest.devices: x = tF.raw_input(Shape([2, 2], 2), [42] * 8, dev) self.assertEqual([42] * 8, x.to_list()) x = tF.raw_input(Shape([2, 2], 2), [0] * 8, dev) x.reset(42) self.assertEqual([42] * 8, x.to_list()) x = tF.raw_input(Shape([2, 2], 2), [123] * 8, dev) copied = Tensor(x) x.reset(42) self.assertEqual([42] * 8, x.to_list()) self.assertEqual([123] * 8, copied.to_list())
def test_parameter_stats(self): self.p.add_stats("stat1", Shape([2, 3])) self.p.add_stats("stat2", Shape([2, 4])) st1 = self.p.stats["stat1"] st1.reset(0) self.assertTrue((st1.to_ndarrays()[0] == np.zeros([2, 3])).all()) self.p.stats["stat1"] = tF.input(np.ones([2, 3])) self.assertTrue((st1.to_ndarrays()[0] == np.ones([2, 3])).all()) self.assertIn("stat1", self.p.stats) self.assertIn("stat2", self.p.stats) self.assertNotIn("stat3", self.p.stats) with self.assertRaises(NotImplementedError): del self.p.stats["stat1"] with self.assertRaises(AttributeError): self.p.stats = ParameterStatistics(self.p)
def test_TensorTest_CheckInvalidInplaceOps(self): for dev in TensorTest.devices: shapes = [ Shape(), Shape([], 3), Shape([2, 2], 2), ] a = tF.raw_input(Shape([2, 2], 3), [0] * 12, dev) for shape in shapes: b = tF.raw_input(shape, [0] * shape.size(), dev) with self.assertRaises(RuntimeError): a += b with self.assertRaises(RuntimeError): a -= b
def test_ModelTest_CheckSaveLoad_Same(self): shape = Shape([2, 2]) values1 = [1, 2, 3, 4] values2 = [5, 6, 7, 8] tmp = tempfile.NamedTemporaryFile() m1 = Model() m2 = Model() p1 = Parameter(shape, I.Constant(0)) p1.value += tF.raw_input(shape, values1) p2 = Parameter(shape, I.Constant(0)) p2.value += tF.raw_input(shape, values2) m1.add("p", p1) m2.add("p", p2) m1.add("sm", m2) m1.save(tmp.name) m1 = Model() m2 = Model() p1 = Parameter() p2 = Parameter() m1.add("p", p1) m2.add("p", p2) m1.add("sm", m2) m1.load(tmp.name) self.assertTrue(p1.valid()) self.assertTrue(p2.valid()) self.assertEqual(shape, p1.shape()) self.assertEqual(shape, p2.shape()) self.assertEqual(values1, p1.value.to_list()) self.assertEqual(values2, p2.value.to_list())
def test_TensorTest_CheckArgMinDims(self): data = [ 3, 4, 5, 0, 1, 2, 6, 7, 8, 0, -1, -2, -6, -7, -8, -3, -4, -5, ] expected = [ [0, 0, 0, 2, 2, 2], [1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ] for dev in TensorTest.devices: a = tF.raw_input(Shape([3, 3], 2), data, dev) for i, exp in enumerate(expected): self.assertEqual(exp, a.argmin(i))
def test_graph_instance(self): g = Graph.get_default() self.assertIs(g, self.graph) node = F.input([0], Shape([])) g = node.graph() self.assertIs(g, self.graph)
def test_ModelTest_CheckSaveLoad_Insufficient(self): shape = Shape([2, 2]) values1 = [1, 2, 3, 4] values2 = [5, 6, 7, 8] tmp = tempfile.NamedTemporaryFile() m1 = Model() m2 = Model() p1 = Parameter(shape, I.Constant(0)) p1.value += tF.raw_input(shape, values1) p2 = Parameter(shape, I.Constant(0)) p2.value += tF.raw_input(shape, values2) m1.add("p", p1) m2.add("p", p2) m1.add("sm", m2) m1.save(tmp.name) m1 = Model() m2 = Model() p1 = Parameter() m1.add("p", p1) m1.add("sm", m2) with self.assertRaises(RuntimeError): m1.load(tmp.name)
def test_functions_input_argument(self): # list[ndarray] w/o shape x = F.input(self.ndarray_data) self.assertEqual(x.to_list(), self.list_data) self.assertEqual(x.shape(), Shape([4, 3], 2)) # ndarray w/o shape x = F.input(self.ndarray_data[0]) self.assertEqual(x.to_list(), self.list_data[:12]) self.assertEqual(x.shape(), Shape([4, 3], 1)) # list[float] w/o shape self.assertRaises(TypeError, lambda: F.input(self.list_data)) # list[float] w/ shape x = F.raw_input(Shape([4, 3], 2), self.list_data) self.assertEqual(x.to_list(), self.list_data) self.assertEqual(x.shape(), Shape([4, 3], 2))
def test_TensorTest_CheckResetValuesByVector(self): for dev in TensorTest.devices: data = [1, 2, 3, 4, 5, 6, 7, 8] x = tF.raw_input(Shape([2, 2], 2), data, dev) self.assertEqual(data, x.to_list()) data = [1, 2, 3, 4, 5, 6, 7, 8] x = tF.raw_input(Shape([2, 2], 2), [0] * 8, dev) x.reset_by_vector(data) self.assertEqual(data, x.to_list()) data = [1, 2, 3, 4, 5, 6, 7, 8] x = tF.raw_input(Shape([2, 2], 2), [123] * 8, dev) copied = Tensor(x) x.reset_by_vector(data) self.assertEqual(data, x.to_list()) self.assertEqual([123] * 8, copied.to_list())
def train_func(trainer): dev = D.Naive(12345) Device.set_default(dev) g = Graph() Graph.set_default(g) pw1 = Parameter([8, 2], I.XavierUniform()) pb1 = Parameter([8], I.Constant(0)) pw2 = Parameter([1, 8], I.XavierUniform()) pb2 = Parameter([1], I.Constant(0)) trainer.add_parameter(pw1) trainer.add_parameter(pb1) trainer.add_parameter(pw2) trainer.add_parameter(pb2) input_data = [1, 1, 1, -1, -1, 1, -1, -1] output_data = [1, -1, -1, 1] for i in range(10): g.clear() x = F.input(input_data, Shape([2], 4)) w1 = F.parameter(pw1) b1 = F.parameter(pb1) w2 = F.parameter(pw2) b2 = F.parameter(pb2) h = F.tanh(w1 @ x + b1) y = w2 @ h + b2 t = F.input(output_data, Shape([], 4)) diff = t - y loss = F.batch.mean(diff * diff) trainer.reset_gradients() loss.backward() trainer.update() return [ pw1.value.to_list(), pb1.value.to_list(), pw2.value.to_list(), pb2.value.to_list() ]
def test_TensorTest_CheckNewMatrixMinibatchWithData(self): for dev in TensorTest.devices: data = [ 3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3, 2, 3, 8, 4, 6, 2, 6, 4, ] data_ndarray = [ np.array([[3, 4, 5], [1, 1, 9]]), np.array([[2, 5, 5], [6, 3, 8]]), np.array([[9, 9, 2], [7, 3, 3]]), np.array([[8, 6, 6], [4, 2, 4]]), ] x = tF.raw_input(Shape([2, 3], 4), data, dev) self.assertTrue(x.valid()) self.assertIs(dev, x.device()) self.assertEqual(Shape([2, 3], 4), x.shape()) self.assertEqual(data, x.to_list()) with self.assertRaises(RuntimeError): x.to_float() self.assertTrue(np.array_equal(data_ndarray, x.to_ndarrays()))
def test_TensorTest_CheckNewScalarWithData(self): for dev in TensorTest.devices: x = tF.raw_input([], [1], dev) x_ndarray = [ np.array([1]), ] self.assertTrue(x.valid()) self.assertIs(dev, x.device()) self.assertEqual(Shape(), x.shape()) self.assertEqual([1], x.to_list()) self.assertEqual(1.0, x.to_float()) self.assertEqual(x_ndarray, x.to_ndarrays())
def forward(self, inputs): batch_size = len(inputs[0]) wlookup = F.parameter(self.pwlookup) wxs = F.parameter(self.pwxs) wsy = F.parameter(self.pwsy) s = F.zeros(Shape([NUM_HIDDEN_UNITS], batch_size)) outputs = [] for i in range(len(inputs) - 1): w = F.pick(wlookup, inputs[i], 1) x = w + s s = F.sigmoid(wxs @ x) outputs.append(wsy @ s) return outputs
def test_TensorTest_CheckNewMatrixWithData(self): for dev in TensorTest.devices: data = [1, 2, 3, 4, 5, 6] data_ndarray = [ np.array([[1, 3, 5], [2, 4, 6]]), ] x = tF.raw_input([2, 3], data, dev) self.assertTrue(x.valid()) self.assertIs(dev, x.device()) self.assertEqual(Shape([2, 3]), x.shape()) self.assertEqual(data, x.to_list()) with self.assertRaises(RuntimeError): x.to_float() self.assertTrue(np.array_equal(data_ndarray, x.to_ndarrays()))
def test_tensor_instance(self): param = Parameter(Shape([])) t_origin = param.gradient t = param.gradient self.assertIs(t, t_origin) t = Tensor(t_origin) self.assertEqual(t.to_list(), t.to_list()) self.assertIsNot(t, t_origin) t = t_origin t *= 2 self.assertIs(t, t_origin) t = t * 2 self.assertIsNot(t, t_origin)
def decode_step(self, trg_words, train): sentence_len = self.concat_fb.shape()[1] b = self.whw_ @ self.trg_lstm_.get_h() b = F.reshape(b, Shape([1, b.shape()[0]])) b = F.broadcast(b, 0, sentence_len) x = F.tanh(self.t_concat_fb @ self.wfbw_ + b) atten_prob = F.softmax(x @ self.wwe_, 0) c = self.concat_fb @ atten_prob e = F.pick(self.trg_lookup_, trg_words, 1) e = F.dropout(e, self.dropout_rate_, train) h = self.trg_lstm_.forward(F.concat([e, c], 0)) h = F.dropout(h, self.dropout_rate_, train) j = F.tanh(self.whj_ @ h + self.bj_) return self.wjy_ @ j + self.by_
def test_pytrainer_not_implemented(self): dev = D.Naive() Device.set_default(dev) trainer = IncompleteTrainer() p = Parameter(Shape([])) with self.assertRaises(NotImplementedError): trainer.add_parameter(p) with self.assertRaises(NotImplementedError): trainer.update() with self.assertRaises(NotImplementedError): Trainer.get_configs(trainer) with self.assertRaises(NotImplementedError): Trainer.set_configs(trainer, {'Trainer.epoch': 1}, { 'Trainer.clip_threshold': 0.0, 'Trainer.lr_scale': 1.0, 'Trainer.l2_strength': 0.0 })
def test_ModelTest_CheckSaveLoadWithStats(self): shape = Shape([2, 2]) values1 = [1, 2, 3, 4] values2 = [5, 6, 7, 8] stats1 = [10, 20, 30, 40] stats2 = [50, 60, 70, 80] tmp = tempfile.NamedTemporaryFile() m1 = Model() m2 = Model() p1 = Parameter(shape, I.Constant(0)) p1.value += tF.raw_input(shape, values1) p2 = Parameter(shape, I.Constant(0)) p2.value += tF.raw_input(shape, values2) p1.add_stats("a", shape) p2.add_stats("b", shape) p1.stats["a"].reset_by_vector(stats1); p2.stats["b"].reset_by_vector(stats2); m1.add("p", p1) m2.add("p", p2) m1.add("sm", m2) m1.save(tmp.name) m1 = Model() m2 = Model() p1 = Parameter() p2 = Parameter() m1.add("p", p1) m2.add("p", p2) m1.add("sm", m2) m1.load(tmp.name) self.assertTrue(p1.valid()) self.assertTrue(p2.valid()) self.assertEqual(shape, p1.shape()) self.assertEqual(shape, p2.shape()) self.assertEqual(values1, p1.value.to_list()) self.assertEqual(values2, p2.value.to_list()) self.assertTrue("a" in p1.stats) self.assertTrue("b" in p2.stats) self.assertEqual(stats1, p1.stats["a"].to_list()) self.assertEqual(stats2, p2.stats["b"].to_list())
def test_pytrainer_propagate_exception(self): dev = D.Naive() Device.set_default(dev) trainer = ExceptionTrainer() p = Parameter(Shape([])) with self.assertRaises(TestException) as ctx: trainer.add_parameter(p) self.assertEqual(str(ctx.exception), "configure_parameter") with self.assertRaises(TestException) as ctx: trainer.update() self.assertEqual(str(ctx.exception), "update_parameter") with self.assertRaises(TestException) as ctx: Trainer.get_configs(trainer) self.assertEqual(str(ctx.exception), "get_configs") with self.assertRaises(TestException) as ctx: Trainer.set_configs(trainer, {'Trainer.epoch': 1}, { 'Trainer.clip_threshold': 0.0, 'Trainer.lr_scale': 1.0, 'Trainer.l2_strength': 0.0 }) self.assertEqual(str(ctx.exception), "set_configs")
def main(): # Loads data train_inputs = load_images("data/train-images-idx3-ubyte", NUM_TRAIN_SAMPLES) train_labels = load_labels("data/train-labels-idx1-ubyte", NUM_TRAIN_SAMPLES) test_inputs = load_images("data/t10k-images-idx3-ubyte", NUM_TEST_SAMPLES) test_labels = load_labels("data/t10k-labels-idx1-ubyte", NUM_TEST_SAMPLES) dev = D.CUDA(0) Device.set_default(dev) g = Graph() Graph.set_default(g) # Parameters of CNNs # Shape: {kernel_height, kernel_width, in_channels, out_channels} pw_cnn1 = Parameter(Shape([KERNEL_SIZE1, KERNEL_SIZE1, 1, NUM_CHANNELS1]), I.XavierUniformConv2D()) pw_cnn2 = Parameter( Shape([KERNEL_SIZE2, KERNEL_SIZE2, NUM_CHANNELS1, NUM_CHANNELS2]), I.XavierUniformConv2D()) # Parameters of FC layers pw_fc1 = Parameter(Shape([NUM_HIDDEN_UNITS, NUM_INPUT_UNITS]), I.XavierUniform()) pw_fc2 = Parameter(Shape([NUM_OUTPUT_UNITS, NUM_HIDDEN_UNITS]), I.XavierUniform()) pb_fc1 = Parameter(Shape([NUM_HIDDEN_UNITS]), I.Constant(0)) pb_fc2 = Parameter(Shape([NUM_OUTPUT_UNITS]), I.Constant(0)) # Optimizer optimizer = O.SGD(.1) optimizer.add(pw_cnn1, pw_cnn2, pw_fc1, pw_fc2, pb_fc1, pb_fc2) # Helper lambda to construct the predictor network. def make_graph(inputs, train): # Input and parameters. #x = F.input(Shape([IMAGE_HEIGHT, IMAGE_WIDTH], BATCH_SIZE), inputs) x = F.input(inputs) w_cnn1 = F.parameter(pw_cnn1) w_cnn2 = F.parameter(pw_cnn2) w_fc1 = F.parameter(pw_fc1) w_fc2 = F.parameter(pw_fc2) b_fc1 = F.parameter(pb_fc1) b_fc2 = F.parameter(pb_fc2) # CNNs h_cnn1 = F.relu(F.conv2d(x, w_cnn1, PADDING1, PADDING1, 1, 1, 1, 1)) h_pool1 = F.max_pool2d(h_cnn1, 2, 2, 0, 0, 2, 2) h_cnn2 = F.relu( F.conv2d(h_pool1, w_cnn2, PADDING2, PADDING2, 1, 1, 1, 1)) h_pool2 = F.max_pool2d(h_cnn2, 2, 2, 0, 0, 2, 2) # FC layers x_fc = F.dropout(F.flatten(h_pool2), .5, train) h_fc = F.dropout(F.relu(F.matmul(w_fc1, x_fc) + b_fc1), .5, train) return F.matmul(w_fc2, h_fc) + b_fc2 # Batch randomizer ids = list(range(NUM_TRAIN_SAMPLES)) for epoch in range(MAX_EPOCH): # Shuffles sample IDs. random.shuffle(ids) # Training loop for batch in range(NUM_TRAIN_BATCHES): print("\rTraining... %d / %d" % (batch + 1, NUM_TRAIN_BATCHES), end="") # Makes a minibatch for training. inputs = [ train_inputs[ids[batch * BATCH_SIZE + i]] for i in range(BATCH_SIZE) ] labels = [ train_labels[ids[batch * BATCH_SIZE + i]] for i in range(BATCH_SIZE) ] # Constructs the graph. g.clear() y = make_graph(inputs, True) loss = F.softmax_cross_entropy(y, labels, 0) avg_loss = F.batch.mean(loss) # Dump computation graph at the first time. # if epoch == 0 and batch == 0: # print(g.dump("dot")) # Implicit forward, backward, and updates parameters. optimizer.reset_gradients() avg_loss.backward() optimizer.update() print() match = 0 # Test loop for batch in range(NUM_TEST_BATCHES): print("\rTesting... %d / %d" % (batch + 1, NUM_TEST_BATCHES), end="") # Makes a test minibatch. inputs = [ test_inputs[batch * BATCH_SIZE + i] for i in range(BATCH_SIZE) ] # Constructs the graph. g.clear() y = make_graph(inputs, False) # Gets outputs, argmax, and compares them with the label. y_val = y.to_list() for i in range(BATCH_SIZE): maxval = -1e10 argmax = -1 for j in range(NUM_OUTPUT_UNITS): v = y_val[j + i * NUM_OUTPUT_UNITS] if v > maxval: maxval = v argmax = j if argmax == test_labels[i + batch * BATCH_SIZE]: match += 1 accuracy = 100.0 * match / NUM_TEST_SAMPLES print("epoch %d: accuracy: %.2f%%" % (epoch, accuracy)) return 0