num_filters_5x5=32,
num_filters_poolproj=32,
pool_size=(2, 2),
num_units=32,
inits=[treeano.inits.XavierNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam", {
"subtree":
model,
"cost":
tn.TotalCostNode(
"cost",
{
"pred": tn.ReferenceNode("pred_ref", reference="model"),
"target": tn.InputNode("y", shape=(None, ), dtype="int32")
},
)
}),
cost_function=treeano.utils.categorical_crossentropy_i32,
)
network = with_updates.network()
network.build() # build eagerly to share weights
BATCH_SIZE = 500
valid_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="valid_time"),
},
cost_function=treeano.utils.categorical_crossentropy_i32),
tn.InputElementwiseSumNode("total_cost")
]),
num_units=512,
sparsity=0.1,
cost_reference="total_cost",
dropout_probability=0.5,
inits=[treeano.inits.XavierNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam", {
"subtree": model,
"cost": tn.ReferenceNode("cost_ref", reference="total_cost")
}),
)
network = with_updates.network()
network.build() # build eagerly to share weights
BATCH_SIZE = 500
valid_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="valid_time"),
canopy.handlers.override_hyperparameters(dropout_probability=0),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])
], {
"x": "x",
]),
num_units=512,
dropout_probability=0.5,
inits=[treeano.inits.XavierNormalInit()],
)
model = tn.L2PenaltyNode(
"l2_cost",
model,
l2_weight=0.0001,
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam",
{"subtree": model,
"cost": tn.InputElementwiseSumNode("cost")}),
cost_function=treeano.utils.categorical_crossentropy_i32,
cost_reference="cost",
)
network = with_updates.network()
network.build() # build eagerly to share weights
BATCH_SIZE = 500
valid_fn = canopy.handled_fn(
network,
[canopy.handlers.time_call(key="valid_time"),
canopy.handlers.override_hyperparameters(dropout_probability=0),
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])],
tn.SoftmaxNode("pred"),
]),
num_units=512,
dropout_probability=0.5,
inits=[treeano.inits.XavierNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam", {
"subtree":
MonitorUpdateRatioNode("updateratio", model),
"cost":
tn.TotalCostNode(
"cost",
{
"pred": tn.ReferenceNode("pred_ref", reference="model"),
"target": tn.InputNode("y", shape=(None, ), dtype="int32")
},
)
}),
cost_function=treeano.utils.categorical_crossentropy_i32,
)
network = with_updates.network()
network.build() # build eagerly to share weights
BATCH_SIZE = 500
valid_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="valid_time"),
def load_network(update_scale_factor):
localization_network = tn.HyperparameterNode(
"loc",
tn.SequentialNode(
"loc_seq",
[tn.DnnMaxPoolNode("loc_pool1"),
tn.DnnConv2DWithBiasNode("loc_conv1"),
tn.DnnMaxPoolNode("loc_pool2"),
bn.NoScaleBatchNormalizationNode("loc_bn1"),
tn.ReLUNode("loc_relu1"),
tn.DnnConv2DWithBiasNode("loc_conv2"),
bn.NoScaleBatchNormalizationNode("loc_bn2"),
tn.ReLUNode("loc_relu2"),
tn.DenseNode("loc_fc1", num_units=50),
bn.NoScaleBatchNormalizationNode("loc_bn3"),
tn.ReLUNode("loc_relu3"),
tn.DenseNode("loc_fc2",
num_units=6,
inits=[treeano.inits.NormalWeightInit(std=0.001)])]),
num_filters=20,
filter_size=(5, 5),
pool_size=(2, 2),
)
st_node = st.AffineSpatialTransformerNode(
"st",
localization_network,
output_shape=(20, 20))
model = tn.HyperparameterNode(
"model",
tn.SequentialNode(
"seq",
[tn.InputNode("x", shape=(None, 1, 60, 60)),
# scaling the updates of the spatial transformer
# seems to be very helpful, to allow the clasification
# net to learn what to look for, before prematurely
# looking
tn.UpdateScaleNode(
"st_update_scale",
st_node,
update_scale_factor=update_scale_factor),
tn.Conv2DWithBiasNode("conv1"),
tn.MaxPool2DNode("mp1"),
bn.NoScaleBatchNormalizationNode("bn1"),
tn.ReLUNode("relu1"),
tn.Conv2DWithBiasNode("conv2"),
tn.MaxPool2DNode("mp2"),
bn.NoScaleBatchNormalizationNode("bn2"),
tn.ReLUNode("relu2"),
tn.GaussianDropoutNode("do1"),
tn.DenseNode("fc1"),
bn.NoScaleBatchNormalizationNode("bn3"),
tn.ReLUNode("relu3"),
tn.DenseNode("fc2", num_units=10),
tn.SoftmaxNode("pred"),
]),
num_filters=32,
filter_size=(3, 3),
pool_size=(2, 2),
num_units=256,
dropout_probability=0.5,
inits=[treeano.inits.HeUniformInit()],
bn_update_moving_stats=True,
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam",
{"subtree": model,
"cost": tn.TotalCostNode("cost", {
"pred": tn.ReferenceNode("pred_ref", reference="model"),
"target": tn.InputNode("y", shape=(None,), dtype="int32")},
)}),
cost_function=treeano.utils.categorical_crossentropy_i32,
learning_rate=2e-3,
)
network = with_updates.network()
network.build() # build eagerly to share weights
return network
}, {
"from": "sigma",
"to": "REINFORCE",
"to_key": "sigma"
}, {
"from": "reward",
"to": "REINFORCE",
"to_key": "reward"
}, {
"from": "sampled",
"to": "REINFORCE",
"to_key": "sampled"
}, {
"from": "REINFORCE"
}]])
network = tn.AdamNode("adam", {
"subtree": graph,
"cost": tn.ReferenceNode("cost", reference="REINFORCE")
},
learning_rate=0.1).network()
fn = network.function([], ["graph", "mu"], include_updates=True)
mus = []
for i in range(1000):
_, mu = fn()
print("Iter:", i, "Predicted constant:", mu)
mus.append(mu)
print("MSE from optimal constant:", np.mean((np.array(mus) - 3.5)**2))
tn.SoftmaxNode("pred"),
]),
num_units=512,
dropout_probability=0.5,
inits=[treeano.inits.XavierNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam", {
"subtree":
model,
"cost":
anrat.ANRATNode(
"cost",
{
"pred": tn.ReferenceNode("pred_ref", reference="model"),
"target": tn.InputNode("y", shape=(None, ), dtype="int32")
},
i32_target=True,
)
}),
)
network = with_updates.network()
network.build() # build eagerly to share weights
BATCH_SIZE = 500
train_fn = canopy.handled_fn(network, [
canopy.handlers.chunk_variables(batch_size=BATCH_SIZE,
variables=["x", "y"])
], {
num_filters=32,
filter_size=(5, 5),
pool_size=(2, 2),
num_units=256,
inits=[treeano.inits.XavierNormalInit()],
)
with_updates = tn.HyperparameterNode(
"with_updates",
tn.AdamNode(
"adam", {
"subtree":
model,
"cost":
tn.TotalCostNode(
"cost",
{
"pred": tn.ReferenceNode("pred_ref", reference="model"),
"target": tn.InputNode("y", shape=(None, ), dtype="int32")
},
cost_function=treeano.utils.categorical_crossentropy_i32,
)
}),
)
network = with_updates.network()
network.build() # build eagerly to share weights
BATCH_SIZE = 500
valid_fn = canopy.handled_fn(network, [
canopy.handlers.time_call(key="valid_time"),
canopy.handlers.override_hyperparameters(bn_use_moving_stats=True),
