def test_add_bias_node_serialization():
tn.check_serialization(tn.AddBiasNode("a"))
tn.check_serialization(
tn.AddBiasNode(
"a",
inits=[],
# need to make broadcastable a list because json (de)serialization
# converts tuples to lists
broadcastable=[True, False, True]))
def test_tied_init():
network = tn.SequentialNode("s", [
tn.InputNode("i", shape=()),
tn.AddBiasNode("b1", inits=[treeano.inits.ConstantInit(42)]),
tn.AddBiasNode("b2", inits=[treeano.inits.TiedInit("b2", "b1")])
]).network()
fn = network.function(["i"], ["s"])
np.testing.assert_equal(84, fn(0)[0])
network["b1"].get_vw("bias").variable.set_value(43)
np.testing.assert_equal(86, fn(0)[0])
def architecture_children(self):
bias_node = tn.AddBiasNode(self.name + "_bias")
dropout_node = tn.DropoutNode(self.name + "_dropout")
return [tn.SequentialNode(
self.name + "_sequential",
[bias_node,
dropout_node])]
def architecture_children(self):
return [
tn.SequentialNode(self._name + "_sequential", [
ResnetInitConv2DNode(self._name + "_conv"),
tn.AddBiasNode(self._name + "_bias",
broadcastable_axes=(0, 2, 3))
])
]
def architecture_children(self):
return [
tn.SequentialNode(
self._name + "_sequential",
[
WeightNormalizedDnnConv2DNode(self._name + "_conv"),
# TODO add hyperparameter untie_biases
tn.AddBiasNode(self._name + "_bias",
broadcastable_axes=(0, 2, 3))
])
]
def test_add_bias_node():
network = tn.SequentialNode("s", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.AddBiasNode("b", broadcastable_axes=())
]).network()
bias_var = network["b"].get_vw("bias")
fn = network.function(["in"], ["s"])
x = np.random.randn(3, 4, 5).astype(fX)
y = np.random.randn(3, 4, 5).astype(fX)
# test that bias is 0 initially
np.testing.assert_allclose(fn(x)[0], x)
# set bias_var value to new value
bias_var.value = y
# test that adding works
np.testing.assert_allclose(fn(x)[0], x + y)
def test_monitor_update_ratio_node():
network = tn.WeightDecayNode(
"decay",
monitor_update_ratio.MonitorUpdateRatioNode(
"mur",
tn.SequentialNode(
"s",
[tn.InputNode("i", shape=(None, 3)),
tn.LinearMappingNode("linear", output_dim=10),
tn.AddBiasNode("bias")])),
weight_decay=1
).network()
network.build()
mur_net = network["mur"]
vws = mur_net.find_vws_in_subtree(tags={"monitor"})
assert len(vws) == 1
vw, = vws
assert re.match(".*_2-norm$", vw.name)
assert re.match(".*linear.*", vw.name)
assert not re.match(".*bias.*", vw.name)
def test_dense_node_and_dense_combine_node2():
# testing that summing the output of 2 dense nodes is the same as
# applying a dense combine node with 2 identities (+ bias)
# and the same as multiplying the output of 1 dense node by 2
network0 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseNode("dense1", num_units=6),
tn.MultiplyConstantNode("mul", value=2)
]),
inits=[treeano.inits.ConstantInit(1)]).network()
network1 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.ElementwiseSumNode("sum", [
tn.DenseNode("dense1", num_units=6),
tn.DenseNode("dense2", num_units=6)
])
]),
inits=[treeano.inits.ConstantInit(1)]).network()
network2 = tn.HyperparameterNode(
"hp",
tn.SequentialNode("seq", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.DenseCombineNode("fc",
[tn.IdentityNode("i1"),
tn.IdentityNode("i2")],
num_units=6),
tn.AddBiasNode("bias")
]),
inits=[treeano.inits.ConstantInit(1)]).network()
x = np.random.randn(3, 4, 5).astype(fX)
fn0 = network0.function(["in"], ["hp"])
fn1 = network1.function(["in"], ["hp"])
fn2 = network2.function(["in"], ["hp"])
np.testing.assert_allclose(fn0(x), fn1(x))
np.testing.assert_allclose(fn0(x), fn2(x))
i, o = binary_toy_data(lag, length)
inputs.append(i)
outputs.append(o)
return np.array(inputs)[..., np.newaxis], np.array(outputs)[..., np.newaxis]
# ############################## prepare model ##############################
model = tn.HyperparameterNode(
"model",
tn.SequentialNode(
"seq",
[tn.InputNode("x", shape=(None, None, 1)),
recurrent_hc.GRUNode("gru1"),
tn.LinearMappingNode("y_linear", output_dim=1),
tn.AddBiasNode("y_bias", broadcastable_axes=(0, 1)),
tn.SigmoidNode("sigmoid"),
]),
inits=[treeano.inits.OrthogonalInit()],
num_units=HIDDEN_STATE_SIZE,
learn_init=True,
grad_clip=1,
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam",
{"subtree": model,
"cost": tn.TotalCostNode("cost", {
"pred": tn.ReferenceNode("pred_ref", reference="model"),
def test_add_bias_node_broadcastable_incorrect_size2():
tn.SequentialNode("s", [
tn.InputNode("in", shape=(3, 4, 5)),
tn.AddBiasNode("b", broadcastable=(True, False, True, False))
]).network().build()
def get_bias_shape(broadcastable):
return tn.SequentialNode("s", [
tn.InputNode("in", shape=(3, 4, 5)),
(tn.AddBiasNode("b", broadcastable=broadcastable)
if broadcastable is not None else tn.AddBiasNode("b"))
]).network()["b"].get_vw("bias").shape