def test_auxiliary_cost_node():
network = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("x", shape=(3, 4, 5)),
tn.AuxiliaryCostNode(
"cost1", {"target": tn.InputNode("y1", shape=(3, 4, 5))}),
tn.AddConstantNode("a1", value=2),
tn.AuxiliaryCostNode(
"cost2", {"target": tn.InputNode("y2", shape=(3, 4, 5))}),
tn.MultiplyConstantNode("m1", value=2),
tn.AuxiliaryCostNode(
"cost3", {"target": tn.InputNode("y3", shape=(3, 4, 5))}),
tn.ConstantNode("const", value=0),
tn.InputElementwiseSumNode("cost")
]),
cost_reference="cost",
cost_function=treeano.utils.squared_error,
).network()
fn = network.function(["x", "y1", "y2", "y3"], ["cost"])
x = np.random.rand(3, 4, 5).astype(fX)
ys = [np.random.rand(3, 4, 5).astype(fX) for _ in range(3)]
def mse(x, y):
return ((x - y)**2).mean()
expected_output = (mse(x, ys[0]) + mse(x + 2, ys[1]) +
mse(2 * (x + 2), ys[2]))
np.testing.assert_allclose(fn(x, *ys)[0], expected_output, rtol=1e-5)
def test_input_elementwise_sum_node():
for s in [(),
(3, 4, 5)]:
network = tn.ContainerNode(
"all",
[tn.InputElementwiseSumNode("ies"),
tn.SequentialNode(
"seq1",
[tn.InputNode("i1", shape=s),
tn.SendToNode("st1", reference="ies", to_key="in1")]),
tn.SequentialNode(
"seq2",
[tn.InputNode("i2", shape=s),
tn.SendToNode("st2", reference="ies", to_key="in2")]),
tn.SequentialNode(
"seq3",
[tn.InputNode("i3", shape=s),
tn.SendToNode("st3", reference="ies", to_key="in3")])]
).network()
fn = network.function(["i1", "i2", "i3"], ["ies"])
i1 = np.array(np.random.rand(*s), dtype=fX)
i2 = np.array(np.random.rand(*s), dtype=fX)
i3 = np.array(np.random.rand(*s), dtype=fX)
np.testing.assert_allclose(i1 + i2 + i3,
fn(i1, i2, i3)[0],
rtol=1e-5)
def test_batch_normalization_node():
network = tn.AdamNode(
"adam", {
"subtree":
tn.SequentialNode("seq", [
tn.InputNode("x", shape=(None, 10)),
batch_normalization.BatchNormalizationNode("bn"),
tn.DenseNode("d", num_units=1),
]),
"cost":
tn.TotalCostNode(
"cost", {
"target": tn.InputNode("y", shape=(None, 1)),
"pred": tn.ReferenceNode("pred_ref", reference="d"),
},
cost_function=treeano.utils.squared_error)
}).network()
fn = network.function(["x", "y"], ["cost"], include_updates=True)
x = 100 + 100 * np.random.randn(100, 10).astype(fX)
y = np.random.randn(100, 1).astype(fX)
prev_cost = fn(x, y)[0]
for _ in range(3):
cost = fn(x, y)[0]
assert cost < prev_cost
prev_cost = cost
def test_zero_grad_node():
n1 = tn.SequentialNode(
"s", [tn.InputNode("i", shape=()),
tn.toy.ScalarSumNode("ss")]).network()
n2 = tn.SequentialNode("s", [
tn.InputNode("i", shape=()),
tn.ZeroGradNode("z"),
tn.toy.ScalarSumNode("ss")
]).network()
fn1 = n1.function(["i"], [
"i",
T.grad(n1["s"].get_vw("default").variable,
n1["i"].get_vw("default").variable)
])
fn2 = n2.function(["i"], [
"i",
T.grad(n2["s"].get_vw("default").variable,
n2["i"].get_vw("default").variable)
])
# gradient should be 1 w/o zero grad node
np.testing.assert_equal(1, fn1(3)[1])
# gradient should be 0 w/ zero grad node
np.testing.assert_equal(0, fn2(3)[1])
def test_load_value_dict_not_strict_keys():
n1 = tn.SequentialNode(
"seq",
[tn.InputNode("i", shape=(10, 100)),
tn.LinearMappingNode(
"lm",
output_dim=15,
inits=[treeano.inits.NormalWeightInit()])]
).network()
n2 = tn.InputNode("i", shape=()).network()
def test1(strict_keys):
canopy.network_utils.load_value_dict(
n1,
canopy.network_utils.to_value_dict(n2),
strict_keys=strict_keys)
def test2(strict_keys):
canopy.network_utils.load_value_dict(
n2,
canopy.network_utils.to_value_dict(n1),
strict_keys=strict_keys)
nt.raises(AssertionError)(test1)(strict_keys=True)
nt.raises(AssertionError)(test2)(strict_keys=True)
test1(strict_keys=False)
test2(strict_keys=False)
def test_dense_node_and_dense_combine_node1():
# testing that dense node and dense combine node with identity child
# return the same thing
network1 = tn.HyperparameterNode("hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseNode("fc1", num_units=6),
tn.DenseNode("fc2", num_units=7),
tn.DenseNode("fc3", num_units=8)
]),
inits=[treeano.inits.ConstantInit(1)
]).network()
network2 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn1 = network1.function(["in"], ["fc3"])
fn2 = network2.function(["in"], ["fc3"])
np.testing.assert_allclose(fn1(x), fn2(x))
def test_dense_combine_node_uses_children():
network1 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.MultiplyConstantNode("mul", value=2),
tn.DenseCombineNode("fc",
[tn.IdentityNode("i1"),
tn.IdentityNode("i2")],
num_units=6)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
network2 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc", [
tn.MultiplyConstantNode("mul1", value=2),
tn.MultiplyConstantNode("mul2", value=2)
],
num_units=6)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn1 = network1.function(["in"], ["hp"])
fn2 = network2.function(["in"], ["hp"])
np.testing.assert_allclose(fn1(x), fn2(x))
def test_to_preallocated_init1():
network1 = tn.SequentialNode(
"seq",
[tn.InputNode("i", shape=(3, 4, 5)),
tn.LinearMappingNode(
"lm",
output_dim=15,
inits=[treeano.inits.NormalWeightInit(15.0)])]
).network()
inits = [canopy.network_utils.to_preallocated_init(network1)]
network2 = tn.SequentialNode(
"seq",
[tn.InputNode("i", shape=(3, 4, 5)),
tn.LinearMappingNode(
"lm",
output_dim=15,
inits=inits)]
).network()
w1 = list(canopy.network_utils.to_shared_dict(network1).values())[0]
w2 = list(canopy.network_utils.to_shared_dict(network2).values())[0]
# both networks should be using the exact same shared variables
assert w1 is w2
fn1 = network1.function(["i"], ["lm"])
fn2 = network2.function(["i"], ["lm"])
x = np.random.randn(3, 4, 5).astype(fX)
np.testing.assert_equal(fn1(x),
fn2(x))
def build_concatenate(axis, s1, s2, s3):
tn.ConcatenateNode(
"concat",
[tn.InputNode("i1", shape=s1),
tn.InputNode("i2", shape=s2),
tn.InputNode("i3", shape=s3)],
axis=axis,
).network().build()
def test_reference_node():
network = tn.SequentialNode("s", [
tn.InputNode("input1", shape=(3, 4, 5)),
tn.InputNode("input2", shape=(5, 4, 3)),
tn.ReferenceNode("ref", reference="input1"),
]).network()
fn = network.function(["input1"], ["ref"])
x = np.random.randn(3, 4, 5).astype(fX)
np.testing.assert_allclose(fn(x)[0], x)
def fcn_network(combine_fn):
network = tn.ContainerNode("c", [
tn.SequentialNode(
"s1",
[tn.InputNode("in1", shape=(3, 4, 5)),
tn.SendToNode("stn1", reference="fcn", to_key="b")]),
tn.SequentialNode(
"s2",
[tn.InputNode("in2", shape=(3, 4, 5)),
tn.SendToNode("stn2", reference="fcn", to_key="a")]),
tn.InputFunctionCombineNode("fcn", combine_fn=combine_fn)
]).network()
return network.function(["in1", "in2"], ["fcn"])
def test_total_cost_node():
network = tn.TotalCostNode(
"cost", {
"pred": tn.InputNode("x", shape=(3, 4, 5)),
"target": tn.InputNode("y", shape=(3, 4, 5))
},
cost_function=treeano.utils.squared_error).network()
fn = network.function(["x", "y"], ["cost"])
x = np.random.rand(3, 4, 5).astype(fX)
y = np.random.rand(3, 4, 5).astype(fX)
np.testing.assert_allclose(fn(x, y)[0], ((x - y)**2).mean(), rtol=1e-5)
np.testing.assert_allclose(fn(x, x)[0], 0)
np.testing.assert_allclose(fn(y, y)[0], 0)
def test_affine_spatial_transformer_node_build():
localization_network = tn.HyperparameterNode(
"loc",
tn.SequentialNode(
"loc_seq",
[tn.DenseNode("loc_fc1", num_units=50),
tn.ReLUNode("loc_relu3"),
tn.DenseNode("loc_fc2",
num_units=6,
inits=[treeano.inits.ZeroInit()])]),
num_filters=32,
filter_size=(5, 5),
pool_size=(2, 2),
)
model = tn.HyperparameterNode(
"model",
tn.SequentialNode(
"seq",
[tn.InputNode("x", shape=(None, 1, 60, 60)),
spatial_transformer.AffineSpatialTransformerNode(
"st",
localization_network,
output_shape=(20, 20)),
tn.DenseNode("fc1"),
tn.ReLUNode("relu1"),
tn.DropoutNode("do1"),
tn.DenseNode("fc2", num_units=10),
tn.SoftmaxNode("pred"),
]),
num_filters=32,
filter_size=(3, 3),
pool_size=(2, 2),
num_units=256,
dropout_probability=0.5,
inits=[treeano.inits.HeNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam",
{"subtree": model,
"cost": tn.TotalCostNode("cost", {
"pred": tn.ReferenceNode("pred_ref", reference="model"),
"target": tn.InputNode("y", shape=(None,), dtype="int32")},
)}),
cost_function=treeano.utils.categorical_crossentropy_i32,
)
network = with_updates.network()
network.build() # build eagerly to share weights
def test_override_hyperparameters2():
network = tn.toy.ConstantUpdaterNode(
"cun",
tn.SequentialNode("seq", [
tn.InputNode("i", shape=(3, 4, 5)),
tn.LinearMappingNode("lm",
output_dim=15,
inits=[treeano.inits.NormalWeightInit(15.0)])
]),
value=-0.1,
).network()
fn1 = network.function(["i"], ["lm"])
fn1u = network.function(["i"], ["lm"], include_updates=True)
fn2_args = (network, [canopy.handlers.override_hyperparameters(value=2)], {
"x": "i"
}, {
"out": "lm"
})
fn2 = canopy.handlers.handled_fn(*fn2_args)
fn2u = canopy.handlers.handled_fn(*fn2_args, include_updates=True)
x = np.random.randn(3, 4, 5).astype(fX)
np.testing.assert_equal(fn1(x)[0], fn2({"x": x})["out"])
fn1u(x)
np.testing.assert_equal(fn1(x)[0], fn2({"x": x})["out"])
fn2u({"x": x})
np.testing.assert_equal(fn1(x)[0], fn2({"x": x})["out"])
def test_find_hyperparameters():
class FooNode(core.WrapperNodeImpl):
hyperparameter_names = ("a", "b", "c")
last_foo = FooNode("last", [tn.InputNode("i", shape=(1, ))], b=1)
mid_foo = FooNode("mid", [last_foo], a=2, c=3)
top_foo = FooNode("top", [mid_foo], a=4, b=5, c=6)
network = top_foo.network(default_hyperparameters={
"a": 7,
"b": 8,
"c": 9
},
override_hyperparameters={
"a": 10,
"b": 11,
"c": 12
})
nt.assert_equal([10, 11, 12, 1, 2, 3, 4, 5, 6, 13, 7, 8, 9],
list(network["last"].find_hyperparameters(["a", "b", "c"],
13)))
nt.assert_equal([10, 11, 12, 2, 3, 4, 5, 6, 13, 7, 8, 9],
list(network["mid"].find_hyperparameters(["a", "b", "c"],
13)))
nt.assert_equal([10, 11, 12, 4, 5, 6, 13, 7, 8, 9],
list(network["top"].find_hyperparameters(["a", "b", "c"],
13)))
def test_elementwise_product_node():
for s in [(),
(3, 4, 5)]:
network = tn.ElementwiseProductNode(
"es",
[tn.InputNode("i1", shape=s),
tn.InputNode("i2", shape=s),
tn.InputNode("i3", shape=s)],
).network()
fn = network.function(["i1", "i2", "i3"], ["es"])
i1 = np.array(np.random.rand(*s), dtype=fX)
i2 = np.array(np.random.rand(*s), dtype=fX)
i3 = np.array(np.random.rand(*s), dtype=fX)
np.testing.assert_allclose(i1 * i2 * i3,
fn(i1, i2, i3)[0],
rtol=1e-5)
def test_move_node():
network1 = tn.SequentialNode("seq", [
tn.InputNode("i", shape=()),
tn.HyperparameterNode("hp1",
tn.HyperparameterNode(
"hp2", tn.AddConstantNode("ac"), value=1),
value=2)
]).network()
network2 = canopy.transforms.move_node(network1, "ac", "hp1")
ans = tn.SequentialNode(
"seq", [tn.InputNode("i", shape=()),
tn.AddConstantNode("ac")])
nt.assert_equal(ans, network2.root_node)
def test_time_per_row2():
network = tn.InputNode("i", shape=()).network()
fn = canopy.handlers.handled_fn(
network,
[canopy.handlers.time_per_row(input_key="x", key="ms_per_row")],
{"x": "i"}, {"out": "i"})
fn({"x": 0})
def test_fold_unfold_axis_into_batch_node():
network = tn.SequentialNode(
"s",
[tn.InputNode("i", shape=(2, 3, 4, 5)),
bf.FoldUnfoldAxisIntoBatchNode(
"fu1",
tn.SequentialNode(
"s2",
[tn.IdentityNode("i1"),
bf.FoldUnfoldAxisIntoBatchNode(
"fu2",
tn.SequentialNode(
"s3",
[tn.IdentityNode("i2"),
tn.DenseNode("d", num_units=11)]),
axis=1)]),
axis=3)]
).network()
fn = network.function(["i"], ["i1", "i2", "fu2", "fu1"])
x = np.zeros((2, 3, 4, 5), dtype=fX)
i1, i2, fu2, fu1 = fn(x)
nt.assert_equal((10, 3, 4), i1.shape)
nt.assert_equal((30, 4), i2.shape)
nt.assert_equal((10, 3, 11), fu2.shape)
nt.assert_equal((2, 3, 11, 5), fu1.shape)
def test_transform_root_node():
network1 = tn.toy.ConstantUpdaterNode(
"cun",
tn.SequentialNode("seq", [
tn.InputNode("i", shape=(3, 4, 5)),
tn.LinearMappingNode("lm",
output_dim=15,
inits=[treeano.inits.NormalWeightInit(15.0)])
]),
value=-0.1,
).network()
# perform build eagerly to initialize weights
network1.build()
network2 = canopy.transforms.transform_root_node(network1, fn=lambda x: x)
fn1 = network1.function(["i"], ["lm"])
fn2 = network2.function(["i"], ["lm"])
fn1u = network1.function(["i"], ["lm"], include_updates=True)
fn2u = network2.function(["i"], ["lm"], include_updates=True)
x = np.random.randn(3, 4, 5).astype(fX)
np.testing.assert_equal(fn1(x), fn2(x))
fn1u(x)
np.testing.assert_equal(fn1(x), fn2(x))
fn2u(x)
np.testing.assert_equal(fn1(x), fn2(x))
def test_apply_node():
network = tn.SequentialNode("s", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.ApplyNode("a", fn=T.sum, shape_fn=lambda x: ()),
]).network()
fn = network.function(["in"], ["s"])
x = np.random.randn(3, 4, 5).astype(fX)
np.testing.assert_allclose(fn(x)[0], x.sum(), rtol=1e-5)
def test_time_call():
network = tn.InputNode("i", shape=()).network()
fn = canopy.handlers.handled_fn(network,
[canopy.handlers.time_call(key="time")],
{"x": "i"}, {"out": "i"})
res = fn({"x": 0})
assert "time" in res
assert "out" in res
def test_repeat_node():
network = tn.SequentialNode("s", [
tn.InputNode("in", shape=(3, )),
tn.RepeatNode("r", repeats=2, axis=0)
]).network()
fn = network.function(["in"], ["s"])
x = np.random.randn(3).astype(fX)
np.testing.assert_allclose(np.repeat(x, 2, 0), fn(x)[0])
def test_kumaraswamy_unit_node():
# just testing that it runs
network = tn.SequentialNode(
"s", [tn.InputNode("i", shape=(100, )),
ku.KumaraswamyUnitNode("k")]).network()
fn = network.function(["i"], ["s"])
x = np.random.randn(100).astype(fX)
fn(x)
def get_shapes(output_dim):
network = tn.SequentialNode("s", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.LinearMappingNode("linear", output_dim=output_dim),
]).network()
weight_shape = network["linear"].get_vw("weight").shape
output_shape = network["s"].get_vw("default").shape
return weight_shape, output_shape
def axes(ndim, pos, neg):
network = tn.HyperparameterNode(
"a",
tn.InputNode("b", shape=()),
pos=pos,
neg=neg,
).network()["a"]
return treeano.utils.find_axes(network, ndim, ["pos"], ["neg"])
def test_default_recurrent_conv_2d_node():
network = tn.SequentialNode("s", [
tn.InputNode("i", shape=(3, 4, 5, 6)),
rcl.DefaultRecurrentConv2DNode(
"a", num_filters=7, filter_size=(3, 3), pad="same")
]).network()
fn = network.function(["i"], ["s"])
res = fn(np.random.randn(3, 4, 5, 6).astype(fX))[0]
np.testing.assert_equal((3, 7, 5, 6), res.shape)
def test_dropout_max_pool_2d_node2():
# testing that stochastic version works
network = tn.SequentialNode("s", [
tn.InputNode("i", shape=(1, 1, 4, 4)),
dmp.DropoutMaxPool2DNode("a", pool_size=(2, 2))
]).network()
fn = network.function(["i"], ["s"])
x = np.arange(16).astype(fX).reshape(1, 1, 4, 4)
fn(x)
def test_prelu_node():
network = tn.SequentialNode(
"s", [tn.InputNode("i", shape=(1, 4)),
prelu.PReLUNode("p")]).network()
fn = network.function(["i"], ["p"])
x = np.array([[-1.0, -0.2, 0.2, 1.0]], dtype=fX)
ans = np.array([[-0.25, -0.05, 0.2, 1.0]], dtype=fX)
np.testing.assert_allclose(fn(x)[0], ans)
def new_network():
return tn.SequentialNode(
"seq",
[tn.InputNode("i", shape=(10, 100)),
tn.LinearMappingNode(
"lm",
output_dim=15,
inits=[treeano.inits.NormalWeightInit()])]
).network()