def test_dense_combine_node_uses_children(): network1 = tn.HyperparameterNode( "hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.MultiplyConstantNode("mul", value=2), tn.DenseCombineNode("fc", [tn.IdentityNode("i1"), tn.IdentityNode("i2")], num_units=6) ]), inits=[treeano.inits.ConstantInit(1)]).network() network2 = tn.HyperparameterNode( "hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.DenseCombineNode("fc", [ tn.MultiplyConstantNode("mul1", value=2), tn.MultiplyConstantNode("mul2", value=2) ], num_units=6) ]), inits=[treeano.inits.ConstantInit(1)]).network() x = np.random.randn(3, 4, 5).astype(fX) fn1 = network1.function(["in"], ["hp"]) fn2 = network2.function(["in"], ["hp"]) np.testing.assert_allclose(fn1(x), fn2(x))
def architecture_children(self): # TODO set LRN n = num_filters / 8 + 1 nodes = [ # NOTE: not explicitly giving the first conv a pad of "same", # since the first conv can have any output shape tn.DnnConv2DWithBiasNode(self.name + "_conv0"), tn.IdentityNode(self.name + "_z0"), tn.ReLUNode(self.name + "_z0_relu"), lrn.LocalResponseNormalizationNode(self.name + "_z0_lrn"), tn.IdentityNode(self.name + "_x0"), ] for t in range(1, self.steps + 1): nodes += [ tn.DnnConv2DWithBiasNode(self.name + "_conv%d" % t, stride=(1, 1), pad="same"), tn.ElementwiseSumNode(self.name + "_sum%d" % t, [ tn.ReferenceNode(self.name + "_sum%d_curr" % t, reference=self.name + "_conv%d" % t), tn.ReferenceNode(self.name + "_sum%d_prev" % t, reference=self.name + "_z0") ]), tn.IdentityNode(self.name + "_z%d" % t), tn.ReLUNode(self.name + "_z%d_relu" % t), lrn.LocalResponseNormalizationNode(self.name + "_z%d_lrn" % t), tn.IdentityNode(self.name + "_x%d" % t), ] return [tn.SequentialNode(self.name + "_sequential", nodes)]
def test_dense_node_and_dense_combine_node1(): # testing that dense node and dense combine node with identity child # return the same thing network1 = tn.HyperparameterNode("hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.DenseNode("fc1", num_units=6), tn.DenseNode("fc2", num_units=7), tn.DenseNode("fc3", num_units=8) ]), inits=[treeano.inits.ConstantInit(1) ]).network() network2 = tn.HyperparameterNode( "hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6), tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7), tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8) ]), inits=[treeano.inits.ConstantInit(1)]).network() x = np.random.randn(3, 4, 5).astype(fX) fn1 = network1.function(["in"], ["fc3"]) fn2 = network2.function(["in"], ["fc3"]) np.testing.assert_allclose(fn1(x), fn2(x))
def test_fold_unfold_axis_into_batch_node(): network = tn.SequentialNode( "s", [tn.InputNode("i", shape=(2, 3, 4, 5)), bf.FoldUnfoldAxisIntoBatchNode( "fu1", tn.SequentialNode( "s2", [tn.IdentityNode("i1"), bf.FoldUnfoldAxisIntoBatchNode( "fu2", tn.SequentialNode( "s3", [tn.IdentityNode("i2"), tn.DenseNode("d", num_units=11)]), axis=1)]), axis=3)] ).network() fn = network.function(["i"], ["i1", "i2", "fu2", "fu1"]) x = np.zeros((2, 3, 4, 5), dtype=fX) i1, i2, fu2, fu1 = fn(x) nt.assert_equal((10, 3, 4), i1.shape) nt.assert_equal((30, 4), i2.shape) nt.assert_equal((10, 3, 11), fu2.shape) nt.assert_equal((2, 3, 11, 5), fu1.shape)
def test_simple_recurrent_node_serialization(): nodes.check_serialization( nodes.recurrent.SimpleRecurrentNode("a", nodes.IdentityNode("b"))) nodes.check_serialization( nodes.recurrent.SimpleRecurrentNode("a", nodes.IdentityNode("b"), num_units=32, batch_size=2**7))
def test_dense_combine_node(): network = tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6), tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7), tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8) ]).network() x = np.random.randn(3, 4, 5).astype(fX) fn = network.function(["in"], ["fc3"]) res = fn(x)[0] nt.assert_equal(res.shape, (3, 8))
def GradualSimpleBatchNormalizationNode(name): from treeano.sandbox.nodes import batch_normalization as bn return GradNetInterpolationNode( name, { "early": bn.SimpleBatchNormalizationNode(name + "_bn"), "late": tn.IdentityNode(name + "_identity") })
def architecture_children(self): children = self.raw_children() gate = children["gate"] transform = children["transform"] # prepare gates transform_gate = tn.SequentialNode( self.name + "_transformgate", [ gate, # add initial value as bias instead # TODO parameterize tn.AddConstantNode(self.name + "_biastranslation", value=-4), tn.SigmoidNode(self.name + "_transformgatesigmoid") ]) # carry gate = 1 - transform gate carry_gate = tn.SequentialNode(self.name + "_carrygate", [ tn.ReferenceNode(self.name + "_transformgateref", reference=transform_gate.name), tn.MultiplyConstantNode(self.name + "_invert", value=-1), tn.AddConstantNode(self.name + "_add", value=1) ]) # combine with gates gated_transform = tn.ElementwiseProductNode( self.name + "_gatedtransform", [transform_gate, transform]) gated_carry = tn.ElementwiseProductNode( self.name + "_gatedcarry", [carry_gate, tn.IdentityNode(self.name + "_carry")]) res = tn.ElementwiseSumNode(self.name + "_res", [gated_carry, gated_transform]) return [res]
def test_container_node_raises(): network = tn.SequentialNode( "s", [tn.ContainerNode("c", []), tn.IdentityNode("i") ]).network() fn = network.function([], ["i"]) fn()
def GradualBatchNormalization(name, **kwargs): from treeano.sandbox.nodes import batch_normalization as bn return tn.HyperparameterNode( name, LinearInterpolationNode( name + "_interpolate", { "early": bn.BatchNormalizationNode(name + "_bn"), "late": tn.IdentityNode(name + "_identity") }), **kwargs)
def create_big_node_graph(levels): assert levels >= 0 if levels == 0: return tn.IdentityNode("i") else: prev = create_big_node_graph(levels - 1) return tn.SequentialNode("s", [ canopy.node_utils.suffix_node(prev, "0"), canopy.node_utils.suffix_node(prev, "1") ])
def test_postwalk_node(): names = [] def f(node): names.append(node.name) return node node = tn.HyperparameterNode( "1", tn.HyperparameterNode("2", tn.IdentityNode("3"))) canopy.node_utils.postwalk_node(node, f) nt.assert_equal(names, ["3", "2", "1"])
def test_node_with_generated_children_can_serialize(): root_node = tn.ContainerNode("c", [ tn.SequentialNode("s1", [ tn.InputNode("in", shape=(3, 4, 5)), tn.SendToNode("stn1", reference="s2") ]), tn.SequentialNode("s2", [tn.SendToNode("stn2", reference="stn3")]), tn.SequentialNode("s3", [tn.SendToNode("stn3", reference="i")]), tn.IdentityNode("i"), ]) root_node.network().build() root2 = treeano.core.node_from_data(treeano.core.node_to_data(root_node)) nt.assert_equal(root_node, root2)
def test_network_doesnt_mutate(): root_node = tn.ContainerNode("c", [ tn.SequentialNode("s1", [ tn.InputNode("in", shape=(3, 4, 5)), tn.SendToNode("stn1", reference="s2") ]), tn.SequentialNode("s2", [tn.SendToNode("stn2", reference="stn3")]), tn.SequentialNode("s3", [tn.SendToNode("stn3", reference="i")]), tn.IdentityNode("i"), ]) original_dict = copy.deepcopy(root_node.__dict__) root_node.network().build() nt.assert_equal(original_dict, root_node.__dict__)
def test_dense_node_and_dense_combine_node2(): # testing that summing the output of 2 dense nodes is the same as # applying a dense combine node with 2 identities (+ bias) # and the same as multiplying the output of 1 dense node by 2 network0 = tn.HyperparameterNode( "hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.DenseNode("dense1", num_units=6), tn.MultiplyConstantNode("mul", value=2) ]), inits=[treeano.inits.ConstantInit(1)]).network() network1 = tn.HyperparameterNode( "hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.ElementwiseSumNode("sum", [ tn.DenseNode("dense1", num_units=6), tn.DenseNode("dense2", num_units=6) ]) ]), inits=[treeano.inits.ConstantInit(1)]).network() network2 = tn.HyperparameterNode( "hp", tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 4, 5)), tn.DenseCombineNode("fc", [tn.IdentityNode("i1"), tn.IdentityNode("i2")], num_units=6), tn.AddBiasNode("bias") ]), inits=[treeano.inits.ConstantInit(1)]).network() x = np.random.randn(3, 4, 5).astype(fX) fn0 = network0.function(["in"], ["hp"]) fn1 = network1.function(["in"], ["hp"]) fn2 = network2.function(["in"], ["hp"]) np.testing.assert_allclose(fn0(x), fn1(x)) np.testing.assert_allclose(fn0(x), fn2(x))
def test_send_to_node(): network = tn.ContainerNode("c", [ tn.SequentialNode("s1", [ tn.InputNode("in", shape=(3, 4, 5)), tn.SendToNode("stn1", reference="s2") ]), tn.SequentialNode("s2", [tn.SendToNode("stn2", reference="stn3")]), tn.SequentialNode("s3", [tn.SendToNode("stn3", reference="i")]), tn.IdentityNode("i"), ]).network() fn = network.function(["in"], ["i"]) x = np.random.randn(3, 4, 5).astype(fX) np.testing.assert_allclose(fn(x)[0], x)
def test_paired_conv_2d_with_bias_node(): network = tn.SequentialNode("s", [ tn.InputNode("i", shape=(3, 4, 5, 6)), paired_conv.PairedConvNode("c", { "conv": tn.Conv2DWithBiasNode("c_conv"), "separator": tn.IdentityNode("sep") }, filter_size=(2, 2), num_filters=7, pad="same") ]).network() fn = network.function(["i"], ["s"]) res = fn(np.random.randn(3, 4, 5, 6).astype(fX))[0] np.testing.assert_equal((3, 7, 5, 6), res.shape)
def test_fold_unfold_axis_into_batch_node2(): network = tn.SequentialNode( "s", [tn.InputNode("i", shape=(2, 3, 4, 5)), bf.FoldUnfoldAxisIntoBatchNode( "fu", tn.IdentityNode("id"), axis=2)] ).network() fn = network.function(["i"], ["s"]) x = np.zeros((2, 3, 4, 5), dtype=fX) nt.assert_equal(x.shape, fn(x)[0].shape) nt.assert_equal(x.shape, network["s"].get_vw("default").shape)
def forget_gate_conv_2d_node(name, num_filters, filter_size=(3, 3), initial_bias=0): return tn.ElementwiseProductNode(name, [ tn.IdentityNode(name + "_identity"), tn.SequentialNode(name + "_forget", [ tn.Conv2DWithBiasNode(name + "_conv", num_filters=num_filters, filter_size=filter_size, stride=(1, 1), pad="same"), tn.AddConstantNode(name + "_initial_bias", value=initial_bias), tn.SigmoidNode(name + "_sigmoid") ]) ])
def architecture_children(self): inner = self.raw_children() input_node = tn.IdentityNode(self.name + "_input") return [ tn.SequentialNode(self.name + "_sequential", [ input_node, inner, tn.AuxiliaryNode( self.name + "_auxiliary", tn.SequentialNode(self.name + "_innerseq", [ ElementwiseContractionPenaltyNode( self.name + "_contractionpenalty", input_reference=input_node.name), tn.AggregatorNode(self.name + "_aggregator"), tn.MultiplyConstantNode(self.name + "_multiplyweight"), tn.SendToNode(self.name + "_sendto", to_key=self.name) ])) ]) ]
def test_auxiliary_dense_softmax_cce_node(): network = tn.SequentialNode("seq", [ tn.InputNode("in", shape=(3, 5)), auxiliary_costs.AuxiliaryDenseSoftmaxCCENode( "aux", {"target": tn.ConstantNode("target", value=np.eye(3).astype(fX))}, num_units=3, cost_reference="foo"), tn.IdentityNode("i"), tn.InputElementwiseSumNode("foo", ignore_default_input=True) ]).network() x = np.random.randn(3, 5).astype(fX) fn = network.function(["in"], ["i", "foo", "aux_dense"]) res = fn(x) np.testing.assert_equal(res[0], x) loss = T.nnet.categorical_crossentropy( np.ones((3, 3), dtype=fX) / 3.0, np.eye(3).astype(fX), ).mean().eval() np.testing.assert_allclose(res[1], loss)
def test_replace_node(): network1 = tn.SequentialNode("seq", [ tn.InputNode("i", shape=(3, 4, 5)), tn.DropoutNode("do", dropout_probability=0.5) ]).network() network2 = canopy.transforms.replace_node(network1, {"do": tn.IdentityNode("do")}) assert "DropoutNode" in str(network1.root_node) assert "DropoutNode" not in str(network2.root_node) fn1 = network1.function(["i"], ["do"]) fn2 = network2.function(["i"], ["do"]) x = np.random.randn(3, 4, 5).astype(fX) @nt.raises(AssertionError) def fails(): np.testing.assert_equal(x, fn1(x)[0]) fails() np.testing.assert_equal(x, fn2(x)[0])
def standard_tanh_spatial_attention_2d_node(name, num_filters, output_channels=None): """ NOTE: if output_channels is not None, this should be the number of input channels """ conv2_filters = 1 if output_channels is None else output_channels attention_nodes = [ tn.Conv2DWithBiasNode(name + "_conv1", filter_size=(1, 1), num_filters=num_filters), tn.ScaledTanhNode(name + "_tanh"), tn.Conv2DWithBiasNode(name + "_conv2", filter_size=(1, 1), num_filters=conv2_filters), tn.SpatialSoftmaxNode(name + "_softmax"), ] if output_channels is None: attention_nodes += [ tn.AddBroadcastNode(name + "_bcast", axes=(1,)), ] # multiply input by attention weights and sum across spatial dimensions nodes = [ tn.ElementwiseProductNode( name + "_combine", [tn.IdentityNode(name + "_input"), tn.SequentialNode( name + "_attention", attention_nodes )]), tn.FlattenNode(name + "_flatten", outdim=3), tn.SumNode(name + "_sum", axis=2), ] return tn.SequentialNode(name, nodes)
def test_auxilliary_cost_node_serialization(): tn.check_serialization( tn.AuxiliaryCostNode("foo", {"target": tn.IdentityNode("bar")}))
# tn.DropoutNode("do1"), cp.AuxiliaryContractionPenaltyNode( "cp2", tn.SequentialNode( "cp_seq2", [tn.DenseNode("fc2"), # the cost has nan's when using ReLU's # TODO look into why tn.AbsNode("abs2")]), cost_weight=1e1), tn.DropoutNode("do2"), tn.DenseNode("fc3", num_units=10), tn.SoftmaxNode("pred"), tn.TotalCostNode( "cost", {"pred": tn.IdentityNode("pred_id"), "target": tn.InputNode("y", shape=(None,), dtype="int32")}, cost_function=treeano.utils.categorical_crossentropy_i32), tn.InputElementwiseSumNode("total_cost")]), num_units=32, cost_reference="total_cost", dropout_probability=0.5, inits=[treeano.inits.XavierNormalInit()], ) with_updates = tn.HyperparameterNode( "with_updates", tn.AdamNode( "adam", {"subtree": model, "cost": tn.ReferenceNode("cost_ref", reference="total_cost")}),
def test_total_cost_node_serialization(): tn.check_serialization( tn.TotalCostNode("foo", { "pred": tn.IdentityNode("foo"), "target": tn.IdentityNode("bar") }))
def test_elementwise_cost_node_serialization(): tn.check_serialization( tn.ElementwiseCostNode("foo", { "pred": tn.IdentityNode("foo"), "target": tn.IdentityNode("bar") }))
def preactivation_residual_block_conv_2d(name, num_filters, num_layers, increase_dim=None, initial_block=False, conv_node=tn.Conv2DNode, bn_node=bn.BatchNormalizationNode, activation_node=tn.ReLUNode, input_num_filters=None, projection_filter_size=(1, 1), increase_dim_stride=(2, 2), no_identity=False): """ from http://arxiv.org/abs/1603.05027 """ if increase_dim is not None: assert increase_dim in {"projection", "pad"} first_stride = increase_dim_stride if increase_dim == "projection": # TODO remove pre-activation when initial block assert not initial_block identity_node = tn.SequentialNode(name + "_projection", [ bn_node(name + "_projectionbn"), activation_node(name + "_projectionactivation"), tn.Conv2DNode(name + "_projectionconv", num_filters=num_filters, filter_size=projection_filter_size, stride=first_stride, pad="same"), ]) elif increase_dim == "pad": assert input_num_filters is not None identity_node = tn.SequentialNode(name + "_pad", [ StridedDownsampleNode(name + "_stride", strides=(1, 1) + first_stride), tn.PadNode( name + "_addpad", padding=(0, (num_filters - input_num_filters) // 2, 0, 0)) ]) else: first_stride = (1, 1) identity_node = tn.IdentityNode(name + "_identity") nodes = [] # first node for i in range(num_layers): if i == 0: # first conv # --- # maybe remove initial activation if not initial_block: nodes += [ bn_node(name + "_bn%d" % i), activation_node(name + "_activation%d" % i), ] # same as middle convs, but with stride nodes += [ conv_node(name + "_conv%d" % i, num_filters=num_filters, stride=first_stride, pad="same"), ] else: nodes += [ bn_node(name + "_bn%d" % i), activation_node(name + "_activation%d" % i), conv_node(name + "_conv%d" % i, num_filters=num_filters, stride=(1, 1), pad="same"), ] residual_node = tn.SequentialNode(name + "_seq", nodes) if no_identity: # ability to disable resnet connections return residual_node else: return tn.ElementwiseSumNode(name, [identity_node, residual_node])
def inner(node): if isinstance(node, cls): return tn.IdentityNode(node.name) else: return node
def test_suffix_node(): node1 = tn.HyperparameterNode( "1", tn.HyperparameterNode("2", tn.IdentityNode("3"))) node2 = tn.HyperparameterNode( "1_foo", tn.HyperparameterNode("2_foo", tn.IdentityNode("3_foo"))) nt.assert_equal(canopy.node_utils.suffix_node(node1, "_foo"), node2)