def run(be, fake_dilation, fsz, stride, pad, dilation):
K = 8
strides = stride
padding = pad
be.rng = be.gen_rng(be.rng_seed)
in_shape = 16
while out_shape(in_shape, fsz, stride, dilation, pad) < 3:
in_shape *= 2
train_shape = (1, in_shape, in_shape)
inp = be.array(be.rng.randn(np.prod(train_shape), be.bsz))
init = Gaussian()
layers = [
Conv((5, 5, K), init=init),
Conv((fsz, fsz, K),
strides=strides,
padding=padding,
init=init,
dilation=dict(dil_d=1, dil_h=dilation, dil_w=dilation)),
Conv((3, 3, K), init=init),
Affine(nout=1, init=init)
]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
model.initialize(train_shape, cost)
if fake_dilation:
# Perform regular convolution with an expanded filter.
weights = save(model)
new_layers = layers
# Replace the middle layers.
new_fsz = dilated_fsz(fsz, dilation)
new_layers[1] = Conv((new_fsz, new_fsz, K),
strides=strides,
padding=padding,
init=init)
model = Model(layers=new_layers)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
model.initialize(train_shape, cost)
load(weights, model, K, fsz, dilation)
print(model)
model.optimizer = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9)
outputs = fprop(model, inp)
weights = bprop(model, outputs)
model.optimizer.optimize(model.layers_to_optimize, epoch=0)
return outputs.get(), weights.get()
def run(be, fake_dilation, fsz, stride, pad, dilation):
K = 8
strides = stride
padding = pad
be.rng = be.gen_rng(be.rng_seed)
in_shape = 16
while out_shape(in_shape, fsz, stride, dilation, pad) < 3:
in_shape *= 2
train_shape = (1, in_shape, in_shape)
inp = be.array(be.rng.randn(np.prod(train_shape), be.bsz))
init = Gaussian()
layers = [Conv((5, 5, K), init=init),
Conv((fsz, fsz, K), strides=strides, padding=padding, init=init,
dilation=dict(dil_d=1, dil_h=dilation, dil_w=dilation)),
Conv((3, 3, K), init=init),
Affine(nout=1, init=init)]
model = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
model.initialize(train_shape, cost)
if fake_dilation:
# Perform regular convolution with an expanded filter.
weights = save(model)
new_layers = layers
# Replace the middle layers.
new_fsz = dilated_fsz(fsz, dilation)
new_layers[1] = Conv((new_fsz, new_fsz, K), strides=strides, padding=padding, init=init)
model = Model(layers=new_layers)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
model.initialize(train_shape, cost)
load(weights, model, K, fsz, dilation)
print(model)
model.optimizer = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9)
outputs = fprop(model, inp)
weights = bprop(model, outputs)
model.optimizer.optimize(model.layers_to_optimize, epoch=0)
return outputs.get(), weights.get()
def test_model_serialize(backend_default, data):
dataset = MNIST(path=data)
(X_train, y_train), (X_test, y_test), nclass = dataset.load_data()
train_set = ArrayIterator(
[X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = Sequential([Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())])
path2 = Sequential([Affine(nout=100, init=init_norm, bias=Constant(0), activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())])
layers = [MergeMultistream(layers=[path1, path2], merge="stack"),
Affine(nout=20, init=init_norm, batch_norm=True, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp.initialize(train_set, cost=mlp.cost)
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
mlp.save_params(tmp_save, keep_states=True)
# Load model
mlp = Model(tmp_save)
mlp.initialize(train_set)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert allclose_with_out(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert allclose_with_out(_s, _s_e)
else:
assert allclose_with_out(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
assert type(p) == type(p_e)
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert allclose_with_out(_p, _p_e)
elif isinstance(p, np.ndarray):
assert allclose_with_out(p, p_e)
else:
assert p == p_e
os.remove(tmp_save)
def test_model_serialize(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator([X_train, X_train],
y_train,
nclass=nclass,
lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = [
Conv((5, 5, 16),
init=init_norm,
bias=Constant(0),
activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
path2 = [
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
layers = [
MergeConcat([path1, path2]),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
BatchNorm(),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1,
momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
else:
assert np.allclose(p, p_e)
os.remove(tmp_save)
# run fprop and bprop on this minibatch save the results
out_fprop = model.fprop(im)
out_fprop_save = [x.get() for x in out_fprop]
im.set(im_save)
out_fprop = model.fprop(im)
out_fprop_save2 = [x.get() for x in out_fprop]
for x, y in zip(out_fprop_save, out_fprop_save2):
assert np.max(np.abs(x-y)) == 0.0, '2 fprop iterations do not match'
# run fit fot 1 minibatch
# have to do this by hand
delta = model.cost.get_errors(im, l)
model.bprop(delta)
if args.resume:
model.optimizer = opt
model.optimizer.optimize(model.layers_to_optimize, epoch=model.epoch_index)
# run fprop again as a measure of the model state
out_fprop = model.fprop(im)
out_fprop_save2 = [x.get() for x in out_fprop]
if not args.resume:
with open('serial_test_out1.pkl', 'w') as fid:
pickle.dump([out_fprop_save, out_fprop_save2], fid)
else:
# load up the saved file and compare
with open('serial_test_out1.pkl', 'r') as fid:
run1 = pickle.load(fid)
# compare the initial fprops
# run fprop and bprop on this minibatch save the results
out_fprop = model.fprop(im)
out_fprop_save = [x.get() for x in out_fprop]
im.set(im_save)
out_fprop = model.fprop(im)
out_fprop_save2 = [x.get() for x in out_fprop]
for x, y in zip(out_fprop_save, out_fprop_save2):
assert np.max(np.abs(x - y)) == 0.0, '2 fprop iterations do not match'
# run fit fot 1 minibatch
# have to do this by hand
delta = model.cost.get_errors(im, l)
model.bprop(delta)
if args.resume:
model.optimizer = opt
model.optimizer.optimize(model.layers_to_optimize, epoch=model.epoch_index)
# run fprop again as a measure of the model state
out_fprop = model.fprop(im)
out_fprop_save2 = [x.get() for x in out_fprop]
if not args.resume:
save_obj([out_fprop_save, out_fprop_save2], 'serial_test_out1.pkl')
else:
# load up the saved file and compare
run1 = load_obj('serial_test_out1.pkl')
# compare the initial fprops
for x, y in zip(run1[0], out_fprop_save):
assert np.max(np.abs(
def test_model_serialize(backend_default, data):
dataset = MNIST(path=data)
(X_train, y_train), (X_test, y_test), nclass = dataset.load_data()
train_set = ArrayIterator([X_train, X_train],
y_train,
nclass=nclass,
lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = Sequential([
Conv((5, 5, 16),
init=init_norm,
bias=Constant(0),
activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
])
path2 = Sequential([
Affine(nout=100,
init=init_norm,
bias=Constant(0),
activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
])
layers = [
MergeMultistream(layers=[path1, path2], merge="stack"),
Affine(nout=20, init=init_norm, batch_norm=True, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1,
momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp.initialize(train_set, cost=mlp.cost)
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
mlp.save_params(tmp_save, keep_states=True)
# Load model
mlp = Model(tmp_save)
mlp.initialize(train_set)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert allclose_with_out(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert allclose_with_out(_s, _s_e)
else:
assert allclose_with_out(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
assert type(p) == type(p_e)
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert allclose_with_out(_p, _p_e)
elif isinstance(p, np.ndarray):
assert allclose_with_out(p, p_e)
else:
assert p == p_e
os.remove(tmp_save)
def test_model_serialize(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator([X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = [Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())]
path2 = [Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())]
layers = [MergeConcat([path1, path2]),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
BatchNorm(),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
else:
assert np.allclose(p, p_e)
os.remove(tmp_save)
