def test_dense_combine_node_uses_children():
network1 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.MultiplyConstantNode("mul", value=2),
tn.DenseCombineNode("fc",
[tn.IdentityNode("i1"),
tn.IdentityNode("i2")],
num_units=6)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
network2 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc", [
tn.MultiplyConstantNode("mul1", value=2),
tn.MultiplyConstantNode("mul2", value=2)
],
num_units=6)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn1 = network1.function(["in"], ["hp"])
fn2 = network2.function(["in"], ["hp"])
np.testing.assert_allclose(fn1(x), fn2(x))
def architecture_children(self):
# TODO set LRN n = num_filters / 8 + 1
nodes = [
# NOTE: not explicitly giving the first conv a pad of "same",
# since the first conv can have any output shape
tn.DnnConv2DWithBiasNode(self.name + "_conv0"),
tn.IdentityNode(self.name + "_z0"),
tn.ReLUNode(self.name + "_z0_relu"),
lrn.LocalResponseNormalizationNode(self.name + "_z0_lrn"),
tn.IdentityNode(self.name + "_x0"),
]
for t in range(1, self.steps + 1):
nodes += [
tn.DnnConv2DWithBiasNode(self.name + "_conv%d" % t,
stride=(1, 1),
pad="same"),
tn.ElementwiseSumNode(self.name + "_sum%d" % t, [
tn.ReferenceNode(self.name + "_sum%d_curr" % t,
reference=self.name + "_conv%d" % t),
tn.ReferenceNode(self.name + "_sum%d_prev" % t,
reference=self.name + "_z0")
]),
tn.IdentityNode(self.name + "_z%d" % t),
tn.ReLUNode(self.name + "_z%d_relu" % t),
lrn.LocalResponseNormalizationNode(self.name + "_z%d_lrn" % t),
tn.IdentityNode(self.name + "_x%d" % t),
]
return [tn.SequentialNode(self.name + "_sequential", nodes)]
def test_dense_node_and_dense_combine_node1():
# testing that dense node and dense combine node with identity child
# return the same thing
network1 = tn.HyperparameterNode("hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseNode("fc1", num_units=6),
tn.DenseNode("fc2", num_units=7),
tn.DenseNode("fc3", num_units=8)
]),
inits=[treeano.inits.ConstantInit(1)
]).network()
network2 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn1 = network1.function(["in"], ["fc3"])
fn2 = network2.function(["in"], ["fc3"])
np.testing.assert_allclose(fn1(x), fn2(x))
def test_fold_unfold_axis_into_batch_node():
network = tn.SequentialNode(
"s",
[tn.InputNode("i", shape=(2, 3, 4, 5)),
bf.FoldUnfoldAxisIntoBatchNode(
"fu1",
tn.SequentialNode(
"s2",
[tn.IdentityNode("i1"),
bf.FoldUnfoldAxisIntoBatchNode(
"fu2",
tn.SequentialNode(
"s3",
[tn.IdentityNode("i2"),
tn.DenseNode("d", num_units=11)]),
axis=1)]),
axis=3)]
).network()
fn = network.function(["i"], ["i1", "i2", "fu2", "fu1"])
x = np.zeros((2, 3, 4, 5), dtype=fX)
i1, i2, fu2, fu1 = fn(x)
nt.assert_equal((10, 3, 4), i1.shape)
nt.assert_equal((30, 4), i2.shape)
nt.assert_equal((10, 3, 11), fu2.shape)
nt.assert_equal((2, 3, 11, 5), fu1.shape)
def test_simple_recurrent_node_serialization():
nodes.check_serialization(
nodes.recurrent.SimpleRecurrentNode("a", nodes.IdentityNode("b")))
nodes.check_serialization(
nodes.recurrent.SimpleRecurrentNode("a",
nodes.IdentityNode("b"),
num_units=32,
batch_size=2**7))
def test_dense_combine_node():
network = tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)
]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn = network.function(["in"], ["fc3"])
res = fn(x)[0]
nt.assert_equal(res.shape, (3, 8))
def GradualSimpleBatchNormalizationNode(name):
from treeano.sandbox.nodes import batch_normalization as bn
return GradNetInterpolationNode(
name, {
"early": bn.SimpleBatchNormalizationNode(name + "_bn"),
"late": tn.IdentityNode(name + "_identity")
})
def architecture_children(self):
children = self.raw_children()
gate = children["gate"]
transform = children["transform"]
# prepare gates
transform_gate = tn.SequentialNode(
self.name + "_transformgate",
[
gate,
# add initial value as bias instead
# TODO parameterize
tn.AddConstantNode(self.name + "_biastranslation", value=-4),
tn.SigmoidNode(self.name + "_transformgatesigmoid")
])
# carry gate = 1 - transform gate
carry_gate = tn.SequentialNode(self.name + "_carrygate", [
tn.ReferenceNode(self.name + "_transformgateref",
reference=transform_gate.name),
tn.MultiplyConstantNode(self.name + "_invert", value=-1),
tn.AddConstantNode(self.name + "_add", value=1)
])
# combine with gates
gated_transform = tn.ElementwiseProductNode(
self.name + "_gatedtransform", [transform_gate, transform])
gated_carry = tn.ElementwiseProductNode(
self.name + "_gatedcarry",
[carry_gate, tn.IdentityNode(self.name + "_carry")])
res = tn.ElementwiseSumNode(self.name + "_res",
[gated_carry, gated_transform])
return [res]
def test_container_node_raises():
network = tn.SequentialNode(
"s",
[tn.ContainerNode("c", []),
tn.IdentityNode("i")
]).network()
fn = network.function([], ["i"])
fn()
def GradualBatchNormalization(name, **kwargs):
from treeano.sandbox.nodes import batch_normalization as bn
return tn.HyperparameterNode(
name,
LinearInterpolationNode(
name + "_interpolate", {
"early": bn.BatchNormalizationNode(name + "_bn"),
"late": tn.IdentityNode(name + "_identity")
}), **kwargs)
def create_big_node_graph(levels):
assert levels >= 0
if levels == 0:
return tn.IdentityNode("i")
else:
prev = create_big_node_graph(levels - 1)
return tn.SequentialNode("s", [
canopy.node_utils.suffix_node(prev, "0"),
canopy.node_utils.suffix_node(prev, "1")
])
def test_postwalk_node():
names = []
def f(node):
names.append(node.name)
return node
node = tn.HyperparameterNode(
"1", tn.HyperparameterNode("2", tn.IdentityNode("3")))
canopy.node_utils.postwalk_node(node, f)
nt.assert_equal(names, ["3", "2", "1"])
def test_node_with_generated_children_can_serialize():
root_node = tn.ContainerNode("c", [
tn.SequentialNode("s1", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.SendToNode("stn1", reference="s2")
]),
tn.SequentialNode("s2", [tn.SendToNode("stn2", reference="stn3")]),
tn.SequentialNode("s3", [tn.SendToNode("stn3", reference="i")]),
tn.IdentityNode("i"),
])
root_node.network().build()
root2 = treeano.core.node_from_data(treeano.core.node_to_data(root_node))
nt.assert_equal(root_node, root2)
def test_network_doesnt_mutate():
root_node = tn.ContainerNode("c", [
tn.SequentialNode("s1", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.SendToNode("stn1", reference="s2")
]),
tn.SequentialNode("s2", [tn.SendToNode("stn2", reference="stn3")]),
tn.SequentialNode("s3", [tn.SendToNode("stn3", reference="i")]),
tn.IdentityNode("i"),
])
original_dict = copy.deepcopy(root_node.__dict__)
root_node.network().build()
nt.assert_equal(original_dict, root_node.__dict__)
def test_dense_node_and_dense_combine_node2():
# testing that summing the output of 2 dense nodes is the same as
# applying a dense combine node with 2 identities (+ bias)
# and the same as multiplying the output of 1 dense node by 2
network0 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseNode("dense1", num_units=6),
tn.MultiplyConstantNode("mul", value=2)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
network1 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.ElementwiseSumNode("sum", [
tn.DenseNode("dense1", num_units=6),
tn.DenseNode("dense2", num_units=6)
])
]),
inits=[treeano.inits.ConstantInit(1)]).network()
network2 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc",
[tn.IdentityNode("i1"),
tn.IdentityNode("i2")],
num_units=6),
tn.AddBiasNode("bias")
]),
inits=[treeano.inits.ConstantInit(1)]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn0 = network0.function(["in"], ["hp"])
fn1 = network1.function(["in"], ["hp"])
fn2 = network2.function(["in"], ["hp"])
np.testing.assert_allclose(fn0(x), fn1(x))
np.testing.assert_allclose(fn0(x), fn2(x))
def test_send_to_node():
network = tn.ContainerNode("c", [
tn.SequentialNode("s1", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.SendToNode("stn1", reference="s2")
]),
tn.SequentialNode("s2", [tn.SendToNode("stn2", reference="stn3")]),
tn.SequentialNode("s3", [tn.SendToNode("stn3", reference="i")]),
tn.IdentityNode("i"),
]).network()
fn = network.function(["in"], ["i"])
x = np.random.randn(3, 4, 5).astype(fX)
np.testing.assert_allclose(fn(x)[0], x)
def test_paired_conv_2d_with_bias_node():
network = tn.SequentialNode("s", [
tn.InputNode("i", shape=(3, 4, 5, 6)),
paired_conv.PairedConvNode("c", {
"conv": tn.Conv2DWithBiasNode("c_conv"),
"separator": tn.IdentityNode("sep")
},
filter_size=(2, 2),
num_filters=7,
pad="same")
]).network()
fn = network.function(["i"], ["s"])
res = fn(np.random.randn(3, 4, 5, 6).astype(fX))[0]
np.testing.assert_equal((3, 7, 5, 6), res.shape)
def test_fold_unfold_axis_into_batch_node2():
network = tn.SequentialNode(
"s",
[tn.InputNode("i", shape=(2, 3, 4, 5)),
bf.FoldUnfoldAxisIntoBatchNode(
"fu",
tn.IdentityNode("id"),
axis=2)]
).network()
fn = network.function(["i"], ["s"])
x = np.zeros((2, 3, 4, 5), dtype=fX)
nt.assert_equal(x.shape, fn(x)[0].shape)
nt.assert_equal(x.shape, network["s"].get_vw("default").shape)
def forget_gate_conv_2d_node(name,
num_filters,
filter_size=(3, 3),
initial_bias=0):
return tn.ElementwiseProductNode(name, [
tn.IdentityNode(name + "_identity"),
tn.SequentialNode(name + "_forget", [
tn.Conv2DWithBiasNode(name + "_conv",
num_filters=num_filters,
filter_size=filter_size,
stride=(1, 1),
pad="same"),
tn.AddConstantNode(name + "_initial_bias", value=initial_bias),
tn.SigmoidNode(name + "_sigmoid")
])
])
def architecture_children(self):
inner = self.raw_children()
input_node = tn.IdentityNode(self.name + "_input")
return [
tn.SequentialNode(self.name + "_sequential", [
input_node, inner,
tn.AuxiliaryNode(
self.name + "_auxiliary",
tn.SequentialNode(self.name + "_innerseq", [
ElementwiseContractionPenaltyNode(
self.name + "_contractionpenalty",
input_reference=input_node.name),
tn.AggregatorNode(self.name + "_aggregator"),
tn.MultiplyConstantNode(self.name + "_multiplyweight"),
tn.SendToNode(self.name + "_sendto", to_key=self.name)
]))
])
]
def test_auxiliary_dense_softmax_cce_node():
network = tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 5)),
auxiliary_costs.AuxiliaryDenseSoftmaxCCENode(
"aux",
{"target": tn.ConstantNode("target", value=np.eye(3).astype(fX))},
num_units=3,
cost_reference="foo"),
tn.IdentityNode("i"),
tn.InputElementwiseSumNode("foo", ignore_default_input=True)
]).network()
x = np.random.randn(3, 5).astype(fX)
fn = network.function(["in"], ["i", "foo", "aux_dense"])
res = fn(x)
np.testing.assert_equal(res[0], x)
loss = T.nnet.categorical_crossentropy(
np.ones((3, 3), dtype=fX) / 3.0,
np.eye(3).astype(fX),
).mean().eval()
np.testing.assert_allclose(res[1], loss)
def test_replace_node():
network1 = tn.SequentialNode("seq", [
tn.InputNode("i", shape=(3, 4, 5)),
tn.DropoutNode("do", dropout_probability=0.5)
]).network()
network2 = canopy.transforms.replace_node(network1,
{"do": tn.IdentityNode("do")})
assert "DropoutNode" in str(network1.root_node)
assert "DropoutNode" not in str(network2.root_node)
fn1 = network1.function(["i"], ["do"])
fn2 = network2.function(["i"], ["do"])
x = np.random.randn(3, 4, 5).astype(fX)
@nt.raises(AssertionError)
def fails():
np.testing.assert_equal(x, fn1(x)[0])
fails()
np.testing.assert_equal(x, fn2(x)[0])
def standard_tanh_spatial_attention_2d_node(name,
num_filters,
output_channels=None):
"""
NOTE: if output_channels is not None, this should be the number
of input channels
"""
conv2_filters = 1 if output_channels is None else output_channels
attention_nodes = [
tn.Conv2DWithBiasNode(name + "_conv1",
filter_size=(1, 1),
num_filters=num_filters),
tn.ScaledTanhNode(name + "_tanh"),
tn.Conv2DWithBiasNode(name + "_conv2",
filter_size=(1, 1),
num_filters=conv2_filters),
tn.SpatialSoftmaxNode(name + "_softmax"),
]
if output_channels is None:
attention_nodes += [
tn.AddBroadcastNode(name + "_bcast", axes=(1,)),
]
# multiply input by attention weights and sum across spatial dimensions
nodes = [
tn.ElementwiseProductNode(
name + "_combine",
[tn.IdentityNode(name + "_input"),
tn.SequentialNode(
name + "_attention",
attention_nodes
)]),
tn.FlattenNode(name + "_flatten", outdim=3),
tn.SumNode(name + "_sum", axis=2),
]
return tn.SequentialNode(name, nodes)
def test_auxilliary_cost_node_serialization():
tn.check_serialization(
tn.AuxiliaryCostNode("foo", {"target": tn.IdentityNode("bar")}))
# tn.DropoutNode("do1"),
cp.AuxiliaryContractionPenaltyNode(
"cp2",
tn.SequentialNode(
"cp_seq2",
[tn.DenseNode("fc2"),
# the cost has nan's when using ReLU's
# TODO look into why
tn.AbsNode("abs2")]),
cost_weight=1e1),
tn.DropoutNode("do2"),
tn.DenseNode("fc3", num_units=10),
tn.SoftmaxNode("pred"),
tn.TotalCostNode(
"cost",
{"pred": tn.IdentityNode("pred_id"),
"target": tn.InputNode("y", shape=(None,), dtype="int32")},
cost_function=treeano.utils.categorical_crossentropy_i32),
tn.InputElementwiseSumNode("total_cost")]),
num_units=32,
cost_reference="total_cost",
dropout_probability=0.5,
inits=[treeano.inits.XavierNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam",
{"subtree": model,
"cost": tn.ReferenceNode("cost_ref", reference="total_cost")}),
def test_total_cost_node_serialization():
tn.check_serialization(
tn.TotalCostNode("foo", {
"pred": tn.IdentityNode("foo"),
"target": tn.IdentityNode("bar")
}))
def test_elementwise_cost_node_serialization():
tn.check_serialization(
tn.ElementwiseCostNode("foo", {
"pred": tn.IdentityNode("foo"),
"target": tn.IdentityNode("bar")
}))
def preactivation_residual_block_conv_2d(name,
num_filters,
num_layers,
increase_dim=None,
initial_block=False,
conv_node=tn.Conv2DNode,
bn_node=bn.BatchNormalizationNode,
activation_node=tn.ReLUNode,
input_num_filters=None,
projection_filter_size=(1, 1),
increase_dim_stride=(2, 2),
no_identity=False):
"""
from http://arxiv.org/abs/1603.05027
"""
if increase_dim is not None:
assert increase_dim in {"projection", "pad"}
first_stride = increase_dim_stride
if increase_dim == "projection":
# TODO remove pre-activation when initial block
assert not initial_block
identity_node = tn.SequentialNode(name + "_projection", [
bn_node(name + "_projectionbn"),
activation_node(name + "_projectionactivation"),
tn.Conv2DNode(name + "_projectionconv",
num_filters=num_filters,
filter_size=projection_filter_size,
stride=first_stride,
pad="same"),
])
elif increase_dim == "pad":
assert input_num_filters is not None
identity_node = tn.SequentialNode(name + "_pad", [
StridedDownsampleNode(name + "_stride",
strides=(1, 1) + first_stride),
tn.PadNode(
name + "_addpad",
padding=(0, (num_filters - input_num_filters) // 2, 0, 0))
])
else:
first_stride = (1, 1)
identity_node = tn.IdentityNode(name + "_identity")
nodes = []
# first node
for i in range(num_layers):
if i == 0:
# first conv
# ---
# maybe remove initial activation
if not initial_block:
nodes += [
bn_node(name + "_bn%d" % i),
activation_node(name + "_activation%d" % i),
]
# same as middle convs, but with stride
nodes += [
conv_node(name + "_conv%d" % i,
num_filters=num_filters,
stride=first_stride,
pad="same"),
]
else:
nodes += [
bn_node(name + "_bn%d" % i),
activation_node(name + "_activation%d" % i),
conv_node(name + "_conv%d" % i,
num_filters=num_filters,
stride=(1, 1),
pad="same"),
]
residual_node = tn.SequentialNode(name + "_seq", nodes)
if no_identity:
# ability to disable resnet connections
return residual_node
else:
return tn.ElementwiseSumNode(name, [identity_node, residual_node])
def inner(node):
if isinstance(node, cls):
return tn.IdentityNode(node.name)
else:
return node
def test_suffix_node():
node1 = tn.HyperparameterNode(
"1", tn.HyperparameterNode("2", tn.IdentityNode("3")))
node2 = tn.HyperparameterNode(
"1_foo", tn.HyperparameterNode("2_foo", tn.IdentityNode("3_foo")))
nt.assert_equal(canopy.node_utils.suffix_node(node1, "_foo"), node2)
