def test_ModelTest_CheckGetParameterRecursiveByTuple(self):
m = Model()
sm = Model()
p1 = Parameter()
p2 = Parameter()
p3 = Parameter()
m.add("p1", p1)
sm.add("p2", p2)
sm.add("p3", p3)
m.add("sm", sm)
self.assertIs(p1, m["p1"])
self.assertIs(p2, m["sm", "p2"])
self.assertIs(p3, m["sm", "p3"])
self.assertIs(p2, sm["p2"])
self.assertIs(p3, sm["p3"])
with self.assertRaises(TypeError):
m["p2"]
with self.assertRaises(TypeError):
m["p3"]
m["sm"]
with self.assertRaises(TypeError):
m["sm", "p1"]
with self.assertRaises(TypeError):
sm["p1"]
with self.assertRaises(TypeError):
m["x"]
def test_ModelTest_CheckAddSubmodel(self):
m = Model()
sm1 = Model()
sm2 = Model()
p1 = Parameter()
p2 = Parameter()
m.add("p1", p1)
m.add("sm1", sm1)
m.add("sm1", sm1)
with self.assertRaises(RuntimeError):
m.add("x", sm1)
with self.assertRaises(RuntimeError):
m.add("p1", p2)
with self.assertRaises(RuntimeError):
m.add("sm1", p2)
with self.assertRaises(RuntimeError):
m.add("p1", sm2)
with self.assertRaises(RuntimeError):
m.add("sm1", sm2)
m.add("p2", p2)
m.add("sm2", sm2)
m.add("sm2", sm2)
with self.assertRaises(RuntimeError):
m.add("x", sm2)
def test_ModelTest_CheckSaveLoad_Same(self):
shape = Shape([2, 2])
values1 = [1, 2, 3, 4]
values2 = [5, 6, 7, 8]
tmp = tempfile.NamedTemporaryFile()
m1 = Model()
m2 = Model()
p1 = Parameter(shape, I.Constant(0))
p1.value += tF.raw_input(shape, values1)
p2 = Parameter(shape, I.Constant(0))
p2.value += tF.raw_input(shape, values2)
m1.add("p", p1)
m2.add("p", p2)
m1.add("sm", m2)
m1.save(tmp.name)
m1 = Model()
m2 = Model()
p1 = Parameter()
p2 = Parameter()
m1.add("p", p1)
m2.add("p", p2)
m1.add("sm", m2)
m1.load(tmp.name)
self.assertTrue(p1.valid())
self.assertTrue(p2.valid())
self.assertEqual(shape, p1.shape())
self.assertEqual(shape, p2.shape())
self.assertEqual(values1, p1.value.to_list())
self.assertEqual(values2, p2.value.to_list())
def test_ModelTest_CheckSaveLoad_Insufficient(self):
shape = Shape([2, 2])
values1 = [1, 2, 3, 4]
values2 = [5, 6, 7, 8]
tmp = tempfile.NamedTemporaryFile()
m1 = Model()
m2 = Model()
p1 = Parameter(shape, I.Constant(0))
p1.value += tF.raw_input(shape, values1)
p2 = Parameter(shape, I.Constant(0))
p2.value += tF.raw_input(shape, values2)
m1.add("p", p1)
m2.add("p", p2)
m1.add("sm", m2)
m1.save(tmp.name)
m1 = Model()
m2 = Model()
p1 = Parameter()
m1.add("p", p1)
m1.add("sm", m2)
with self.assertRaises(RuntimeError):
m1.load(tmp.name)
def test_ModelTest_CheckAddParameter(self):
m = Model()
p1 = Parameter()
p2 = Parameter()
p3 = Parameter()
m.add("p1", p1)
m.add("p1", p1)
with self.assertRaises(RuntimeError):
m.add("x", p1)
with self.assertRaises(RuntimeError):
m.add("p1", p2)
m.add("p2", p2)
m.add("p2", p2)
with self.assertRaises(RuntimeError):
m.add("x", p2)
with self.assertRaises(RuntimeError):
m.add("p1", p3)
with self.assertRaises(RuntimeError):
m.add("p2", p3)
m.add("p3", p3)
m.add("p3", p3)
with self.assertRaises(RuntimeError):
m.add("x", p3)
def test_ModelTest_CheckGetTrainableParametersWithSubmodels(self):
m1 = Model()
m2 = Model()
m3 = Model()
p1 = Parameter()
p2 = Parameter()
p3 = Parameter()
m1.add("p", p1)
m2.add("p", p2)
m3.add("p", p3)
m1.add("sm", m2)
m2.add("sm", m3)
params1 = m1.get_trainable_parameters()
self.assertEqual(3, len(params1))
self.assertIsInstance(params1, dict)
self.assertIs(p1, params1[("p",)])
self.assertIs(p2, params1[("sm", "p",)])
self.assertIs(p3, params1[("sm", "sm", "p",)])
params2 = m2.get_trainable_parameters()
self.assertEqual(2, len(params2))
self.assertIsInstance(params2, dict)
self.assertIs(p2, params2[("p",)])
self.assertIs(p3, params2[("sm", "p",)])
params3 = m3.get_trainable_parameters()
self.assertEqual(1, len(params3))
self.assertIsInstance(params3, dict)
self.assertIs(p3, params3[("p",)])
def main():
with DefaultScopeDevice(CPUDevice()):
pw1 = Parameter("w1", [8, 2], I.XavierUniform())
pb1 = Parameter("b1", [8], I.Constant(0))
pw2 = Parameter("w2", [1, 8], I.XavierUniform())
pb2 = Parameter("b2", [], I.Constant(0))
trainer = T.SGD(0.1)
trainer.add_parameter(pw1)
trainer.add_parameter(pb1)
trainer.add_parameter(pw2)
trainer.add_parameter(pb2)
input_data = np.array(
[
[1, 1], # Sample 1
[1, -1], # Sample 2
[-1, 1], # Sample 3
[-1, -1], # Sample 4
],
dtype=np.float32)
output_data = np.array(
[
1, # Label 1
-1, # Label 2
-1, # Label 3
1, # Label 4
],
dtype=np.float32)
for i in range(100):
g = Graph()
with DefaultScopeGraph(g):
# Builds a computation graph.
#x = F.input(shape=Shape([2], 4), data=input_data)
x = F.input(data=input_data)
w1 = F.input(param=pw1)
b1 = F.input(param=pb1)
w2 = F.input(param=pw2)
b2 = F.input(param=pb2)
h = F.tanh(F.matmul(w1, x) + b1)
y = F.matmul(w2, h) + b2
# Calculates values.
y_val = g.forward(y).to_list()
print("epoch ", i, ":")
for j in range(4):
print(" [", j, "]: ", y_val[j])
#t = F.input(shape=Shape([], 4), data=output_data)
t = F.input(data=output_data)
diff = t - y
loss = F.batch.mean(diff * diff)
loss_val = g.forward(loss).to_list()[0]
print(" loss: ", loss_val)
trainer.reset_gradients()
g.backward(loss)
trainer.update()
def test_model_invalid_operation(self):
model1 = Model()
model2 = Model()
model1.add("m", model2)
param = Parameter()
model1.add("p", param)
with self.assertRaises(TypeError) as e:
model1["notfound"]
self.assertEqual(
str(e.exception),
"'name' is not a name of neither parameter nor submodel")
with self.assertRaises(TypeError):
del model1["p"]
with self.assertRaises(TypeError):
del model1["m"]
with self.assertRaises(TypeError):
del model1[0]
with self.assertRaises(TypeError):
model1[(0, 1)]
with self.assertRaises(TypeError):
model1[[0, 1]]
model3 = TestModel()
model3.p = Parameter()
model3.m = TestModel()
model3.a = "test"
del model3.a
self.assertNotIn("a", model3.__dict__)
with self.assertRaises(TypeError):
del model3.p
self.assertIn("p", model3.__dict__)
with self.assertRaises(TypeError):
del model3.m
self.assertIn("m", model3.__dict__)
def __init__(self, dropout):
self.dropout = dropout
self.pw1 = Parameter()
self.pb1 = Parameter()
self.pw2 = Parameter()
self.pb2 = Parameter()
self.scan_attributes()
def __init__(self, max_len, dropout):
self.max_len = max_len
self.dropout = dropout
self.pe = None
self.plookup = Parameter()
self.pby = Parameter()
self.scan_attributes()
def __init__(self, in_size, out_size, trainer):
self.out_size_ = out_size
self.pwxh_ = Parameter([4 * out_size, in_size], I.Uniform(-0.1, 0.1))
self.pwhh_ = Parameter([4 * out_size, out_size], I.Uniform(-0.1, 0.1))
self.pbh_ = Parameter([4 * out_size], I.Constant(0))
trainer.add_parameter(self.pwxh_)
trainer.add_parameter(self.pwhh_)
trainer.add_parameter(self.pbh_)
def __init__(self, in_size, out_size, trainer):
self.out_size_ = out_size
self.pw_ = Parameter([3 * out_size, in_size], I.Uniform(-0.1, 0.1))
self.pbf_ = Parameter([out_size], I.Constant(0))
self.pbr_ = Parameter([out_size], I.Constant(0))
trainer.add_parameter(self.pw_)
trainer.add_parameter(self.pbf_)
trainer.add_parameter(self.pbr_)
def __init__(self):
self.dropout_rate = DROPOUT_RATE
self.psrc_lookup = Parameter()
self.ptrg_lookup = Parameter()
self.pwhy = Parameter()
self.pby = Parameter()
self.src_lstm = LSTM()
self.trg_lstm = LSTM()
self.scan_attributes()
def test_Parameter_argument(self):
# no argument
p = Parameter()
self.assertFalse(p.valid())
# shape w/ Initializer
p = Parameter(Shape([4, 3]), I.Constant(1))
self.assertEqual(p.shape(), Shape([4, 3]))
self.assertEqual(p.value.to_list(), [1] * 12)
def __init__(self, vocab_size, eos_id):
self.eos_id = eos_id
self.pwlookup = Parameter([NUM_HIDDEN_UNITS, vocab_size],
I.XavierUniform())
self.pwxs = Parameter([NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS],
I.XavierUniform())
self.pwsy = Parameter([vocab_size, NUM_HIDDEN_UNITS],
I.XavierUniform())
self.add_all_parameters()
def __init__(self, n_heads, dropout):
self.dropout = dropout
self.n_heads = n_heads
self.pwq = Parameter()
self.pwk = Parameter()
self.pwv = Parameter()
self.pwo = Parameter()
self.attention = ScaledDotProductAttention(dropout)
self.scan_attributes()
def __init__(self, vocab_size, eos_id, trainer):
self.eos_id_ = eos_id
self.pwlookup_ = Parameter([NUM_HIDDEN_UNITS, vocab_size],
I.XavierUniform())
self.pwxs_ = Parameter([NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS],
I.XavierUniform())
self.pwsy_ = Parameter([vocab_size, NUM_HIDDEN_UNITS],
I.XavierUniform())
trainer.add_parameter(self.pwlookup_)
trainer.add_parameter(self.pwxs_)
trainer.add_parameter(self.pwsy_)
def __init__(self, name, src_vocab_size, trg_vocab_size, embed_size,
hidden_size, dropout_rate):
self.name_ = name
self.dropout_rate_ = dropout_rate
self.psrc_lookup_ = Parameter([embed_size, src_vocab_size],
I.XavierUniform())
self.ptrg_lookup_ = Parameter([embed_size, trg_vocab_size],
I.XavierUniform())
self.pwhy_ = Parameter([trg_vocab_size, hidden_size],
I.XavierUniform())
self.pby_ = Parameter([trg_vocab_size], I.Constant(0))
self.src_lstm_ = LSTM(name + "_src_lstm", embed_size, hidden_size)
self.trg_lstm_ = LSTM(name + "_trg_lstm", embed_size, hidden_size)
def test_ModelTest_CheckGetTrainableParameters(self):
m = Model()
p1 = Parameter()
p2 = Parameter()
p3 = Parameter()
m.add("p1", p1)
m.add("p2", p2)
m.add("p3", p3)
params = m.get_trainable_parameters()
self.assertEqual(3, len(params))
self.assertIsInstance(params, dict)
self.assertIs(p1, params[("p1",)]);
self.assertIs(p2, params[("p2",)]);
self.assertIs(p3, params[("p3",)]);
def test_device_instance(self):
dev = Device.get_default()
self.assertIs(dev, self.device)
tensor = tF.raw_input([], [0])
dev = tensor.device()
self.assertIs(dev, self.device)
node = F.raw_input([], [0])
dev = node.device()
self.assertIs(dev, self.device)
my_device = Naive()
self.assertIsNot(my_device, self.device)
node = F.raw_input([], [0], device=my_device)
dev = node.device()
self.assertIs(dev, my_device)
dev = self.graph.get_device(node)
self.assertIs(dev, my_device)
param = Parameter([], I.Constant(1))
dev = param.device()
self.assertIs(dev, self.device)
def test_pyoptimizer_parameter(self):
dev = D.Naive()
Device.set_default(dev)
pw1 = Parameter([8, 2], I.XavierUniform())
self.t.add(pw1)
self.assertIn("testadam-m1", pw1.stats)
self.assertIn("testadam-m2", pw1.stats)
def __init__(self, vocab_size, eos_id):
self.eos_id = eos_id
self.plookup = Parameter([NUM_HIDDEN_UNITS, vocab_size], I.Uniform(-0.1, 0.1))
self.rnn1 = LSTM(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS)
self.rnn2 = LSTM(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS)
self.hy = Affine(NUM_HIDDEN_UNITS, vocab_size)
self.scan_attributes()
def __init__(self, vocab_size, eos_id, trainer):
self.eos_id_ = eos_id
self.plookup_ = Parameter([NUM_HIDDEN_UNITS, vocab_size],
I.Uniform(-0.1, 0.1))
self.rnn1_ = SRU(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS, trainer)
self.rnn2_ = SRU(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS, trainer)
self.hy_ = Affine(NUM_HIDDEN_UNITS, vocab_size, trainer)
trainer.add_parameter(self.plookup_)
def __init__(self, vocab_size, eos_id):
self.eos_id = eos_id
self.plookup = Parameter([NUM_HIDDEN_UNITS, vocab_size], I.Uniform(-0.1, 0.1))
self.rnn1 = LSTM(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS)
self.rnn2 = LSTM(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS)
self.hy = Affine(NUM_HIDDEN_UNITS, vocab_size)
self.add_all_parameters()
self.add_all_submodels()
def test_model_setattr(self):
model = TestModel()
model.p1 = Parameter()
model.p2 = Parameter()
model.p3 = Parameter()
model.m1 = TestModel()
model.m2 = TestModel()
model.m3 = TestModel()
self.assertIs(model["p1"], model.p1)
self.assertIs(model["p2"], model.p2)
self.assertIs(model["p3"], model.p3)
model.p4 = Parameter()
self.assertIs(model["m1"], model.m1)
self.assertIs(model["m2"], model.m2)
self.assertIs(model["m3"], model.m3)
model.m4 = TestModel()
self.assertIs(model["p4"], model.p4)
self.assertIs(model["m4"], model.m4)
def test_ModelTest_CheckGetParameteer(self):
m = Model()
sm = Model()
p1 = Parameter()
p2 = Parameter()
p3 = Parameter()
m.add("p1", p1)
m.add("p2", p2)
sm.add("p3", p3)
m.add("sm", sm)
self.assertIs(p1, m["p1"])
self.assertIs(p2, m["p2"])
with self.assertRaises(TypeError):
m["p3"]
self.assertIs(sm, m["sm"])
with self.assertRaises(TypeError):
m["x"]
def test_model_get_all_parameters(self):
submodel = TestModel()
submodel.sp1 = Parameter()
submodel.sp2 = Parameter()
submodel.add("sp1", submodel.sp1)
submodel.add("sp2", submodel.sp2)
parentmodel = TestModel()
parentmodel.p1 = Parameter()
parentmodel.p2 = Parameter()
parentmodel.sub = submodel
parentmodel.add("p1", parentmodel.p1)
parentmodel.add("p2", parentmodel.p2)
parentmodel.add("sub", parentmodel.sub)
params = parentmodel.get_all_parameters()
self.assertIs(params[("p1", )], parentmodel.p1)
self.assertIs(params[("p2", )], parentmodel.p2)
self.assertIs(params[("sub", "sp1")], parentmodel.sub.sp1)
self.assertIs(params[("sub", "sp2")], parentmodel.sub.sp2)
def test_ModelTest_CheckSaveLoadWithStats(self):
shape = Shape([2, 2])
values1 = [1, 2, 3, 4]
values2 = [5, 6, 7, 8]
stats1 = [10, 20, 30, 40]
stats2 = [50, 60, 70, 80]
tmp = tempfile.NamedTemporaryFile()
m1 = Model()
m2 = Model()
p1 = Parameter(shape, I.Constant(0))
p1.value += tF.raw_input(shape, values1)
p2 = Parameter(shape, I.Constant(0))
p2.value += tF.raw_input(shape, values2)
p1.add_stats("a", shape)
p2.add_stats("b", shape)
p1.stats["a"].reset_by_vector(stats1);
p2.stats["b"].reset_by_vector(stats2);
m1.add("p", p1)
m2.add("p", p2)
m1.add("sm", m2)
m1.save(tmp.name)
m1 = Model()
m2 = Model()
p1 = Parameter()
p2 = Parameter()
m1.add("p", p1)
m2.add("p", p2)
m1.add("sm", m2)
m1.load(tmp.name)
self.assertTrue(p1.valid())
self.assertTrue(p2.valid())
self.assertEqual(shape, p1.shape())
self.assertEqual(shape, p2.shape())
self.assertEqual(values1, p1.value.to_list())
self.assertEqual(values2, p2.value.to_list())
self.assertTrue("a" in p1.stats)
self.assertTrue("b" in p2.stats)
self.assertEqual(stats1, p1.stats["a"].to_list())
self.assertEqual(stats2, p2.stats["b"].to_list())
def test_model_parameter_deep(self):
model1 = Model()
model2 = Model()
model1.add("m2", model2)
model3 = Model()
model2.add("m3", model3)
param = Parameter()
model3.add("p", param)
self.assertIs(model1["m2", "m3", "p"], param)
self.assertIs(model1["m2"]["m3"]["p"], param)
def train_func(trainer):
dev = D.Naive(12345)
Device.set_default(dev)
g = Graph()
Graph.set_default(g)
pw1 = Parameter([8, 2], I.XavierUniform())
pb1 = Parameter([8], I.Constant(0))
pw2 = Parameter([1, 8], I.XavierUniform())
pb2 = Parameter([1], I.Constant(0))
trainer.add_parameter(pw1)
trainer.add_parameter(pb1)
trainer.add_parameter(pw2)
trainer.add_parameter(pb2)
input_data = [1, 1, 1, -1, -1, 1, -1, -1]
output_data = [1, -1, -1, 1]
for i in range(10):
g.clear()
x = F.input(input_data, Shape([2], 4))
w1 = F.parameter(pw1)
b1 = F.parameter(pb1)
w2 = F.parameter(pw2)
b2 = F.parameter(pb2)
h = F.tanh(w1 @ x + b1)
y = w2 @ h + b2
t = F.input(output_data, Shape([], 4))
diff = t - y
loss = F.batch.mean(diff * diff)
trainer.reset_gradients()
loss.backward()
trainer.update()
return [
pw1.value.to_list(),
pb1.value.to_list(),
pw2.value.to_list(),
pb2.value.to_list()
]