def apply(self, input_lb, input_un, target):
batch_size = input_lb.shape[0]
get_labeled = lambda x: x[:batch_size] if x is not None else x
input = T.concatenate([input_lb, input_un], axis=0)
self.layer_dims = {0: self.input_dim}
self.lr = self.shared(self.default_lr, 'learning_rate', role=None)
top = len(self.layers) - 1
clean = self.encoder(input, noise_std=[0])
corr = self.encoder(input, noise_std=self.noise_std)
ests, costs = self.decoder(clean, corr, batch_size)
# Costs
y = target.flatten()
costs.class_clean = CategoricalCrossEntropy().apply(
y, get_labeled(clean.h[top]))
costs.class_clean.name = 'CE_clean'
costs.class_corr = CategoricalCrossEntropy().apply(
y, get_labeled(corr.h[top]))
costs.class_corr.name = 'CE_corr'
costs.total = costs.class_corr * 1.0
for i in range(len(self.layers)):
costs.total += costs.denois[i] * self.denoising_cost_x[i]
costs.total.name = 'Total_cost'
self.costs = costs
# Classification error
mr = MisclassificationRate()
self.error = mr.apply(y, get_labeled(clean.h[top])) * np.float32(100.)
self.error.name = 'Error_rate'
def __init__(self, config):
self.X = T.tensor4("features")
c = config
seq = BrickSequence(
input_dim=(3, 32, 32),
bricks=[
conv3(c['n_l1']),
conv3(c['n_l2']),
max_pool(),
conv3(c['n_l3']),
conv3(c['n_l4']),
max_pool(),
#conv3(10),
#conv3(10),
Flattener(),
linear(c['n_l5']),
Softmax()
])
seq.initialize()
self.pred = seq.apply(self.X)
self.Y = T.imatrix("targets")
self.cost = CategoricalCrossEntropy().apply(self.Y.flatten(),
self.pred)
self.cost.name = "cost"
self.accur = 1.0 - MisclassificationRate().apply(
self.Y.flatten(), self.pred)
self.accur.name = "accur"
def main(save_to, num_epochs):
mlp = MLP([Tanh(), Softmax()], [784, 100, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
x = tensor.matrix('features')
y = tensor.lmatrix('targets')
probs = mlp.apply(tensor.flatten(x, outdim=2))
cost = CategoricalCrossEntropy().apply(y.flatten(), probs)
error_rate = MisclassificationRate().apply(y.flatten(), probs)
cg = ComputationGraph([cost])
W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables)
cost = cost + .00005 * (W1**2).sum() + .00005 * (W2**2).sum()
cost.name = 'final_cost'
mnist_train = MNIST(("train", ))
mnist_test = MNIST(("test", ))
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=0.1))
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate],
Flatten(DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, 500)),
which_sources=('features', )),
prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
Printing()
]
if BLOCKS_EXTRAS_AVAILABLE:
extensions.append(
Plot('MNIST example',
channels=[[
'test_final_cost',
'test_misclassificationrate_apply_error_rate'
], ['train_total_gradient_norm']]))
main_loop = MainLoop(algorithm,
Flatten(DataStream.default_stream(
mnist_train,
iteration_scheme=SequentialScheme(
mnist_train.num_examples, 50)),
which_sources=('features', )),
model=Model(cost),
extensions=extensions)
main_loop.run()
def setup_model():
# shape: T x B x F
input_ = T.tensor3('features')
# shape: B
target = T.lvector('targets')
model = LSTMAttention(input_dim=10000,
dim=500,
mlp_hidden_dims=[2000, 500, 4],
batch_size=100,
image_shape=(100, 100),
patch_shape=(28, 28),
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
model.initialize()
h, c = model.apply(input_)
classifier = MLP([Rectifier(), Softmax()], [500, 100, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
classifier.initialize()
probabilities = classifier.apply(h[-1])
cost = CategoricalCrossEntropy().apply(target, probabilities)
error_rate = MisclassificationRate().apply(target, probabilities)
return cost, error_rate
def build_training(lr=0.002, model=None):
x = T.tensor4('x')
y = T.imatrix()
if model is None:
model = build_model()
y_prev = model.apply(x)
y_softmax =Softmax().apply(y_prev)
##### prediction #####
#cost = CategoricalCrossEntropy().apply(y.flatten(), y_prev).mean()
cost = Softmax().categorical_cross_entropy(y.flatten(), y_prev).mean()
error = MisclassificationRate().apply(y.flatten(), y_softmax).mean()
W, B = get_Params(y_prev)
params = W + B
regulizer_full = sum([w.norm(2) for w in W[0:2]])
regulizer_conv = sum([w.norm(2) for w in W[2:]])
cost = cost #+ 0.01*regulizer_conv #+ 0.001*regulizer_conv
updates, updates_init = RMSProp(cost, params, lr)
#updates, updates_init = Adam(cost, params, lr)
#updates = Sgd(cost, params, lr)
train_function = theano.function([x,y], cost, updates=updates,
allow_input_downcast=True)
valid_function = theano.function([x,y], cost,
allow_input_downcast=True)
test_function = theano.function([x,y], error,
allow_input_downcast=True)
reinit = theano.function([], T.zeros((1,)), updates=updates_init)
observation = theano.function([], [w.norm(2) for w in W])
"""
reg_function = theano.function([], T.zeros((1,)), updates=clip(W),
allow_input_downcast=True)
observation = theano.function([], [w.norm(2) for w in W])
"""
return train_function, valid_function, test_function, model, reinit
def pretrain(model,
hyper_params,
full_hdf5,
full_meta,
train_selectors,
valid_selectors=None):
"""
generic training method for siamese networks;
works with any network structure
:return:
"""
from theano import tensor
from deepthought.datasets.triplet import TripletsIndexDataset
from blocks.bricks.cost import MisclassificationRate
train_data = TripletsIndexDataset(
full_hdf5,
full_meta,
train_selectors,
targets_source=hyper_params['pretrain_target_source'],
group_attribute=hyper_params['group_attribute'])
if valid_selectors is not None:
if hyper_params['use_ext_dataset_for_validation']:
ext_selectors = train_selectors
else:
ext_selectors = None
valid_data = TripletsIndexDataset(
full_hdf5,
full_meta,
valid_selectors,
ext_selectors=ext_selectors,
targets_source=hyper_params['pretrain_target_source'],
group_attribute=hyper_params['group_attribute'])
else:
valid_data = None
# Note: this has to match the sources defined in the dataset
#y = tensor.lvector('targets')
y = tensor.lmatrix('targets')
# Note: this requires a one-hot encoding of the targets
probs = model.outputs[0]
# cost = HingeLoss().apply(y, probs) # cost for SCE
cost = (probs[:, 0]**2).sum() # triplet network cost = const * d+**2
# Note: this requires just the class labels, not in a one-hot encoding
# error_rate = MisclassificationRate().apply(y.argmax(axis=1), probs) # SCE version
error_rate = 1. - MisclassificationRate().apply(
y.argmax(axis=1), probs) # flipped for triplet net
error_rate.name = 'error_rate'
return GenericNNEncoderExperiment.run_pretrain(model, hyper_params,
cost, train_data,
valid_data,
[error_rate])
def test_misclassification_rate():
y = tensor.vector(dtype='int32')
yhat = tensor.matrix(theano.config.floatX)
top1_brick = MisclassificationRate()
top2_brick = MisclassificationRate(top_k=2)
top3_brick = MisclassificationRate(top_k=3)
f = theano.function([y, yhat], [
top1_brick.apply(y, yhat),
top2_brick.apply(y, yhat),
top3_brick.apply(y, yhat)
])
y_ = numpy.array([2, 1, 0, 1, 2], dtype='int32')
yhat_ = numpy.array(
[[3, 2, 1, 0], [1, 8, 2, 1], [3, 8, 1, 2], [1, 6, 4, 2], [9, 7, 5, 5]],
dtype='float32')
top1_error = 0.6
top2_error = 0.4
top3_error = 0.2
assert_allclose([top1_error, top2_error, top3_error], f(y_, yhat_))
def create_act_table(self, save_to, act_table):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(
y.flatten(), probs).copy(name='error_rate'))
max_activation_table = (MaxActivationTable().apply(outs).copy(
name='max_activation_table'))
max_activation_table.tag.aggregation_scheme = (
Concatenate(max_activation_table))
model = Model([error_rate, max_activation_table])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test_stream = DataStream.default_stream(
self.mnist_test,
iteration_scheme=SequentialScheme(self.mnist_test.num_examples,
batch_size))
evaluator = DatasetEvaluator([error_rate, max_activation_table])
results = evaluator.evaluate(mnist_test_stream)
table = results['max_activation_table']
pickle.dump(table, open(act_table, 'wb'))
return table
def main(save_to, num_epochs):
mlp = MLP([Tanh(), Softmax()], [784, 100, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
x = tensor.matrix('features')
y = tensor.lmatrix('targets')
probs = mlp.apply(x)
cost = CategoricalCrossEntropy().apply(y.flatten(), probs)
error_rate = MisclassificationRate().apply(y.flatten(), probs)
cg = ComputationGraph([cost])
W1, W2 = VariableFilter(roles=[WEIGHTS])(cg.variables)
cost = cost + .00005 * (W1**2).sum() + .00005 * (W2**2).sum()
cost.name = 'final_cost'
mnist_train = MNIST("train")
mnist_test = MNIST("test")
algorithm = GradientDescent(cost=cost,
step_rule=SteepestDescent(learning_rate=0.1))
main_loop = MainLoop(
mlp,
DataStream(mnist_train,
iteration_scheme=SequentialScheme(mnist_train.num_examples,
50)),
algorithm,
extensions=[
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate],
DataStream(mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, 500)),
prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_every_epoch=True),
SerializeMainLoop(save_to),
Plot('MNIST example',
channels=[[
'test_final_cost',
'test_misclassificationrate_apply_error_rate'
], ['train_total_gradient_norm']]),
Printing()
])
main_loop.run()
def setup_model():
# shape: T x B x F
input_ = T.tensor3('features')
# shape: B
target = T.lvector('targets')
model = LSTMAttention(dim=256,
mlp_hidden_dims=[256, 4],
batch_size=100,
image_shape=(64, 64),
patch_shape=(16, 16),
weights_init=Glorot(),
biases_init=Constant(0))
model.initialize()
h, c, location, scale = model.apply(input_)
classifier = MLP([Rectifier(), Softmax()], [256 * 2, 200, 10],
weights_init=Glorot(),
biases_init=Constant(0))
model.h = h
model.c = c
model.location = location
model.scale = scale
classifier.initialize()
probabilities = classifier.apply(T.concatenate([h[-1], c[-1]], axis=1))
cost = CategoricalCrossEntropy().apply(target, probabilities)
error_rate = MisclassificationRate().apply(target, probabilities)
model.cost = cost
location_x_0_avg = T.mean(location[0, :, 0])
location_x_0_avg.name = 'location_x_0_avg'
location_x_10_avg = T.mean(location[10, :, 0])
location_x_10_avg.name = 'location_x_10_avg'
location_x_20_avg = T.mean(location[-1, :, 0])
location_x_20_avg.name = 'location_x_20_avg'
scale_x_0_avg = T.mean(scale[0, :, 0])
scale_x_0_avg.name = 'scale_x_0_avg'
scale_x_10_avg = T.mean(scale[10, :, 0])
scale_x_10_avg.name = 'scale_x_10_avg'
scale_x_20_avg = T.mean(scale[-1, :, 0])
scale_x_20_avg.name = 'scale_x_20_avg'
monitorings = [
error_rate, location_x_0_avg, location_x_10_avg, location_x_20_avg,
scale_x_0_avg, scale_x_10_avg, scale_x_20_avg
]
model.monitorings = monitorings
return model
def main(save_to, num_epochs, batch_size):
mlp = MLP([Tanh(), Tanh(), Tanh(), Softmax()], [3072, 4096, 1024, 512, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
x = tt.tensor4('features', dtype='float32')
y = tt.vector('label', dtype='int32')
probs = mlp.apply(x.reshape((-1, 3072)))
cost = CategoricalCrossEntropy().apply(y, probs)
error_rate = MisclassificationRate().apply(y, probs)
cg = ComputationGraph([cost])
ws = VariableFilter(roles=[WEIGHT])(cg.variables)
cost = cost + .00005 * sum(([(w**2).sum() for w in ws]))
cost.name = 'final_cost'
train_dataset = Cifar10Dataset(data_dir='/home/belohlavek/data/cifar10',
is_train=True)
valid_dataset = Cifar10Dataset(data_dir='/home/belohlavek/data/cifar10',
is_train=False)
train_stream = train_dataset.get_stream(batch_size)
valid_stream = valid_dataset.get_stream(batch_size)
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Adam(learning_rate=0.001))
extensions = [
Timing(),
LogExtension('/home/belohlavek/ALI/mlp.log'),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate], valid_stream, prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
Printing()
]
main_loop = MainLoop(algorithm,
train_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def __init__(self, dwin, n_mot, vect_size, n_hidden, n_out=2, **kwargs):
self.dwin = dwin
self.n_mot = n_mot
self.vect_size = vect_size
if isinstance(n_hidden, int):
self.n_hidden = [n_hidden]
else:
self.n_hidden = n_hidden
self.n_out = n_out
self.window = Window(self.dwin,
self.n_mot,
self.vect_size,
self.n_hidden,
self.n_out,
weights_init=IsotropicGaussian(0.001))
super(LookUpTrain, self).__init__(**kwargs)
self.softmax = Softmax()
self.error = MisclassificationRate()
self.children = [self.window, self.softmax, self.error]
train_monitor = None
test_monitor = None
for ex in main_loop.extensions:
if isinstance(ex, DataStreamMonitoring) and ex.prefix == 'train':
train_monitor = ex
if isinstance(ex, DataStreamMonitoring) and ex.prefix == 'test':
test_monitor = ex
model = Model(test_monitor._evaluator.theano_variables[0])
loss = model.outputs[0]
y, x = model.inputs
out = VariableFilter(theano_name='mlp_inference_output')(model.variables)[0]
out2 = VariableFilter(theano_name='linear_apply_output')(model.variables)[0]
pred = Softmax().apply(out2)
misclass = MisclassificationRate().apply(tensor.flatten(y, outdim=1), out)
##############
### Noise
##############
import theano
import theano.tensor as tensor
import numpy
from PIL import Image
#from mnist import MNIST
from fuel.datasets import MNIST
from fuel.streams import DataStream
from fuel.schemes import ShuffledScheme
grad = tensor.grad(loss, x)
def main(save_to, num_epochs,
weight_decay=0.0001, noise_pressure=0, subset=None, num_batches=None,
batch_size=None, histogram=None, resume=False):
output_size = 10
prior_noise_level = -10
noise_step_rule = Scale(1e-6)
noise_rate = theano.shared(numpy.asarray(1e-5, dtype=theano.config.floatX))
convnet = create_res_net(out_noise=True, tied_noise=True, tied_sigma=True,
noise_rate=noise_rate,
prior_noise_level=prior_noise_level)
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
test_probs = convnet.apply(x)
test_cost = (CategoricalCrossEntropy().apply(y.flatten(), test_probs)
.copy(name='cost'))
test_error_rate = (MisclassificationRate().apply(y.flatten(), test_probs)
.copy(name='error_rate'))
test_confusion = (ConfusionMatrix().apply(y.flatten(), test_probs)
.copy(name='confusion'))
test_confusion.tag.aggregation_scheme = Sum(test_confusion)
test_cg = ComputationGraph([test_cost, test_error_rate])
# Apply dropout to all layer outputs except final softmax
# dropout_vars = VariableFilter(
# roles=[OUTPUT], bricks=[Convolutional],
# theano_name_regex="^conv_[25]_apply_output$")(test_cg.variables)
# drop_cg = apply_dropout(test_cg, dropout_vars, 0.5)
# Apply 0.2 dropout to the pre-averaging layer
# dropout_vars_2 = VariableFilter(
# roles=[OUTPUT], bricks=[Convolutional],
# theano_name_regex="^conv_8_apply_output$")(test_cg.variables)
# train_cg = apply_dropout(test_cg, dropout_vars_2, 0.2)
# Apply 0.2 dropout to the input, as in the paper
# train_cg = apply_dropout(test_cg, [x], 0.2)
# train_cg = drop_cg
# train_cg = apply_batch_normalization(test_cg)
# train_cost, train_error_rate, train_components = train_cg.outputs
with batch_normalization(convnet):
with training_noise(convnet):
train_probs = convnet.apply(x)
train_cost = (CategoricalCrossEntropy().apply(y.flatten(), train_probs)
.copy(name='cost'))
train_components = (ComponentwiseCrossEntropy().apply(y.flatten(),
train_probs).copy(name='components'))
train_error_rate = (MisclassificationRate().apply(y.flatten(),
train_probs).copy(name='error_rate'))
train_cg = ComputationGraph([train_cost,
train_error_rate, train_components])
population_updates = get_batch_normalization_updates(train_cg)
bn_alpha = 0.9
extra_updates = [(p, p * bn_alpha + m * (1 - bn_alpha))
for p, m in population_updates]
# for annealing
nit_penalty = theano.shared(numpy.asarray(noise_pressure, dtype=theano.config.floatX))
nit_penalty.name = 'nit_penalty'
# Compute noise rates for training graph
train_logsigma = VariableFilter(roles=[LOG_SIGMA])(train_cg.variables)
train_mean_log_sigma = tensor.concatenate([n.flatten() for n in train_logsigma]).mean()
train_mean_log_sigma.name = 'mean_log_sigma'
train_nits = VariableFilter(roles=[NITS])(train_cg.auxiliary_variables)
train_nit_rate = tensor.concatenate([n.flatten() for n in train_nits]).mean()
train_nit_rate.name = 'nit_rate'
train_nit_regularization = nit_penalty * train_nit_rate
train_nit_regularization.name = 'nit_regularization'
# Apply regularization to the cost
trainable_parameters = VariableFilter(roles=[WEIGHT, BIAS])(
train_cg.parameters)
mask_parameters = [p for p in trainable_parameters
if get_brick(p).name == 'mask']
noise_parameters = VariableFilter(roles=[NOISE])(train_cg.parameters)
biases = VariableFilter(roles=[BIAS])(train_cg.parameters)
weights = VariableFilter(roles=[WEIGHT])(train_cg.variables)
nonmask_weights = [p for p in weights if get_brick(p).name != 'mask']
l2_norm = sum([(W ** 2).sum() for W in nonmask_weights])
l2_norm.name = 'l2_norm'
l2_regularization = weight_decay * l2_norm
l2_regularization.name = 'l2_regularization'
# testversion
test_cost = test_cost + l2_regularization
test_cost.name = 'cost_with_regularization'
# Training version of cost
train_cost_without_regularization = train_cost
train_cost_without_regularization.name = 'cost_without_regularization'
train_cost = train_cost + l2_regularization + train_nit_regularization
train_cost.name = 'cost_with_regularization'
cifar10_train = CIFAR10(("train",))
cifar10_train_stream = RandomPadCropFlip(
NormalizeBatchLevels(DataStream.default_stream(
cifar10_train, iteration_scheme=ShuffledScheme(
cifar10_train.num_examples, batch_size)),
which_sources=('features',)),
(32, 32), pad=4, which_sources=('features',))
test_batch_size = 128
cifar10_test = CIFAR10(("test",))
cifar10_test_stream = NormalizeBatchLevels(DataStream.default_stream(
cifar10_test,
iteration_scheme=ShuffledScheme(
cifar10_test.num_examples, test_batch_size)),
which_sources=('features',))
momentum = Momentum(0.01, 0.9)
# Create a step rule that doubles the learning rate of biases, like Caffe.
# scale_bias = Restrict(Scale(2), biases)
# step_rule = CompositeRule([scale_bias, momentum])
# Create a step rule that reduces the learning rate of noise
scale_mask = Restrict(noise_step_rule, mask_parameters)
step_rule = CompositeRule([scale_mask, momentum])
# from theano.compile.nanguardmode import NanGuardMode
# Train with simple SGD
algorithm = GradientDescent(
cost=train_cost, parameters=trainable_parameters,
step_rule=step_rule)
algorithm.add_updates(extra_updates)
#,
# theano_func_kwargs={
# 'mode': NanGuardMode(
# nan_is_error=True, inf_is_error=True, big_is_error=True)})
exp_name = save_to.replace('.%d', '')
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
EpochSchedule(momentum.learning_rate, [
(0, 0.01), # Warm up with 0.01 learning rate
(50, 0.1), # Then go back to 0.1
(100, 0.01),
(150, 0.001)
# (83, 0.01), # Follow the schedule in the paper
# (125, 0.001)
]),
EpochSchedule(noise_step_rule.learning_rate, [
(0, 1e-2),
(2, 1e-1),
(4, 1)
# (0, 1e-6),
# (2, 1e-5),
# (4, 1e-4)
]),
EpochSchedule(noise_rate, [
(0, 1e-2),
(2, 1e-1),
(4, 1)
# (0, 1e-6),
# (2, 1e-5),
# (4, 1e-4),
# (6, 3e-4),
# (8, 1e-3), # Causes nit rate to jump
# (10, 3e-3),
# (12, 1e-2),
# (15, 3e-2),
# (19, 1e-1),
# (24, 3e-1),
# (30, 1)
]),
NoiseExtension(
noise_parameters=noise_parameters),
NoisyDataStreamMonitoring(
[test_cost, test_error_rate, test_confusion],
cifar10_test_stream,
noise_parameters=noise_parameters,
prefix="test"),
TrainingDataMonitoring(
[train_cost, train_error_rate, train_nit_rate,
train_cost_without_regularization,
l2_regularization,
train_nit_regularization,
momentum.learning_rate,
train_mean_log_sigma,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
every_n_batches=17),
# after_epoch=True),
Plot('Training performance for ' + exp_name,
channels=[
['train_cost_with_regularization',
'train_cost_without_regularization',
'train_nit_regularization',
'train_l2_regularization'],
['train_error_rate'],
['train_total_gradient_norm'],
['train_mean_log_sigma'],
],
every_n_batches=17),
Plot('Test performance for ' + exp_name,
channels=[[
'train_error_rate',
'test_error_rate',
]],
after_epoch=True),
EpochCheckpoint(save_to, use_cpickle=True, after_epoch=True),
ProgressBar(),
Printing()]
if histogram:
attribution = AttributionExtension(
components=train_components,
parameters=cg.parameters,
components_size=output_size,
after_batch=True)
extensions.insert(0, attribution)
if resume:
extensions.append(Load(exp_name, True, True))
model = Model(train_cost)
main_loop = MainLoop(
algorithm,
cifar10_train_stream,
model=model,
extensions=extensions)
main_loop.run()
if histogram:
save_attributions(attribution, filename=histogram)
with open('execution-log.json', 'w') as outfile:
json.dump(main_loop.log, outfile, cls=NumpyEncoder)
def main(job_id, params):
config = ConfigParser.ConfigParser()
config.readfp(open('./params'))
max_epoch = int(config.get('hyperparams', 'max_iter', 100))
base_lr = float(config.get('hyperparams', 'base_lr', 0.01))
train_batch = int(config.get('hyperparams', 'train_batch', 256))
valid_batch = int(config.get('hyperparams', 'valid_batch', 512))
test_batch = int(config.get('hyperparams', 'valid_batch', 512))
W_sd = float(config.get('hyperparams', 'W_sd', 0.01))
W_mu = float(config.get('hyperparams', 'W_mu', 0.0))
b_sd = float(config.get('hyperparams', 'b_sd', 0.01))
b_mu = float(config.get('hyperparams', 'b_mu', 0.0))
hidden_units = int(config.get('hyperparams', 'hidden_units', 32))
input_dropout_ratio = float(
config.get('hyperparams', 'input_dropout_ratio', 0.2))
dropout_ratio = float(config.get('hyperparams', 'dropout_ratio', 0.2))
weight_decay = float(config.get('hyperparams', 'weight_decay', 0.001))
max_norm = float(config.get('hyperparams', 'max_norm', 100.0))
solver = config.get('hyperparams', 'solver_type', 'rmsprop')
data_file = config.get('hyperparams', 'data_file')
side = config.get('hyperparams', 'side', 'b')
# Spearmint optimization parameters:
if params:
base_lr = float(params['base_lr'][0])
dropout_ratio = float(params['dropout_ratio'][0])
hidden_units = params['hidden_units'][0]
weight_decay = params['weight_decay'][0]
if 'adagrad' in solver:
solver_type = CompositeRule([
AdaGrad(learning_rate=base_lr),
VariableClipping(threshold=max_norm)
])
else:
solver_type = CompositeRule([
RMSProp(learning_rate=base_lr),
VariableClipping(threshold=max_norm)
])
input_dim = {'l': 11427, 'r': 10519, 'b': 10519 + 11427}
data_file = config.get('hyperparams', 'data_file')
if 'b' in side:
train = H5PYDataset(data_file, which_set='train')
valid = H5PYDataset(data_file, which_set='valid')
test = H5PYDataset(data_file, which_set='test')
x_l = tensor.matrix('l_features')
x_r = tensor.matrix('r_features')
x = tensor.concatenate([x_l, x_r], axis=1)
else:
train = H5PYDataset(data_file,
which_set='train',
sources=['{}_features'.format(side), 'targets'])
valid = H5PYDataset(data_file,
which_set='valid',
sources=['{}_features'.format(side), 'targets'])
test = H5PYDataset(data_file,
which_set='test',
sources=['{}_features'.format(side), 'targets'])
x = tensor.matrix('{}_features'.format(side))
y = tensor.lmatrix('targets')
# Define a feed-forward net with an input, two hidden layers, and a softmax output:
model = MLP(activations=[
Rectifier(name='h1'),
Rectifier(name='h2'),
Softmax(name='output'),
],
dims=[input_dim[side], hidden_units, hidden_units, 2],
weights_init=IsotropicGaussian(std=W_sd, mean=W_mu),
biases_init=IsotropicGaussian(b_sd, b_mu))
# Don't forget to initialize params:
model.initialize()
# y_hat is the output of the neural net with x as its inputs
y_hat = model.apply(x)
# Define a cost function to optimize, and a classification error rate.
# Also apply the outputs from the net and corresponding targets:
cost = CategoricalCrossEntropy().apply(y.flatten(), y_hat)
error = MisclassificationRate().apply(y.flatten(), y_hat)
error.name = 'error'
# This is the model: before applying dropout
model = Model(cost)
# Need to define the computation graph for the cost func:
cost_graph = ComputationGraph([cost])
# This returns a list of weight vectors for each layer
W = VariableFilter(roles=[WEIGHT])(cost_graph.variables)
# Add some regularization to this model:
cost += weight_decay * l2_norm(W)
cost.name = 'entropy'
# computational graph with l2 reg
cost_graph = ComputationGraph([cost])
# Apply dropout to inputs:
inputs = VariableFilter([INPUT])(cost_graph.variables)
dropout_inputs = [
input for input in inputs if input.name.startswith('linear_')
]
dropout_graph = apply_dropout(cost_graph, [dropout_inputs[0]],
input_dropout_ratio)
dropout_graph = apply_dropout(dropout_graph, dropout_inputs[1:],
dropout_ratio)
dropout_cost = dropout_graph.outputs[0]
dropout_cost.name = 'dropout_entropy'
# Learning Algorithm (notice: we use the dropout cost for learning):
algo = GradientDescent(step_rule=solver_type,
params=dropout_graph.parameters,
cost=dropout_cost)
# algo.step_rule.learning_rate.name = 'learning_rate'
# Data stream used for training model:
training_stream = Flatten(
DataStream.default_stream(dataset=train,
iteration_scheme=ShuffledScheme(
train.num_examples,
batch_size=train_batch)))
training_monitor = TrainingDataMonitoring([
dropout_cost,
aggregation.mean(error),
aggregation.mean(algo.total_gradient_norm)
],
after_batch=True)
# Use the 'valid' set for validation during training:
validation_stream = Flatten(
DataStream.default_stream(dataset=valid,
iteration_scheme=ShuffledScheme(
valid.num_examples,
batch_size=valid_batch)))
validation_monitor = DataStreamMonitoring(variables=[cost, error],
data_stream=validation_stream,
prefix='validation',
after_epoch=True)
test_stream = Flatten(
DataStream.default_stream(
dataset=test,
iteration_scheme=ShuffledScheme(test.num_examples,
batch_size=test_batch)))
test_monitor = DataStreamMonitoring(variables=[error],
data_stream=test_stream,
prefix='test',
after_training=True)
plotting = Plot('AdniNet_{}'.format(side),
channels=[
['dropout_entropy', 'validation_entropy'],
['error', 'validation_error'],
],
after_batch=False)
# Checkpoint class used to save model and log:
stamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H:%M')
checkpoint = Checkpoint('./models/{}net/{}'.format(side, stamp),
save_separately=['model', 'log'],
every_n_epochs=1)
# Home-brewed class for early stopping when we detect we have started to overfit
early_stopper = FinishIfOverfitting(error_name='error',
validation_name='validation_error',
threshold=0.1,
epochs=5,
burn_in=100)
# The main loop will train the network and output reports, etc
main_loop = MainLoop(data_stream=training_stream,
model=model,
algorithm=algo,
extensions=[
validation_monitor,
training_monitor,
plotting,
FinishAfter(after_n_epochs=max_epoch),
early_stopper,
Printing(),
ProgressBar(),
checkpoint,
test_monitor,
])
main_loop.run()
ve = float(main_loop.log.last_epoch_row['validation_error'])
te = float(main_loop.log.last_epoch_row['error'])
spearmint_loss = ve + abs(te - ve)
print 'Spearmint Loss: {}'.format(spearmint_loss)
return spearmint_loss
l = Linear(input_dim=l.get_dim("output"),
output_dim=10,
weights_init=IsotropicGaussian(std=0.01),
biases_init=IsotropicGaussian(std=0.01))
l.initialize()
o = l.apply(o)
o = Softmax().apply(o)
Y = T.imatrix(name="targets")
cost = CategoricalCrossEntropy().apply(Y.flatten(), o)
cost.name = "cost"
miss_class = 1.0 - MisclassificationRate().apply(Y.flatten(), o)
miss_class.name = "accuracy"
cg = ComputationGraph(cost)
print cg.shared_variables
bricks = [get_brick(var) for var in cg.variables if get_brick(var)]
for i, b in enumerate(bricks):
b.name += str(i)
step_rule = AdaM()
algorithm = GradientDescent(cost=cost, step_rule=step_rule)
print "Loading data"
mnist_train = MNIST("train")
train_stream = DataStream(dataset=mnist_train,
def __init__(self, save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test",), sources=['features', 'targets'])
basis = create_fair_basis(mnist_test, 10, 10)
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
cg.variables))
# Normalize input and apply the convnet
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
confusion_image = (ConfusionImage().apply(y.flatten(), probs, x)
.copy(name='confusion_image'))
confusion_image.tag.aggregation_scheme = Sum(confusion_image)
model = Model(
[error_rate, confusion, confusion_image] + list(outs.values()))
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))
self.model = model
self.mnist_test_stream = mnist_test_stream
self.evaluator = DatasetEvaluator(
[error_rate, confusion, confusion_image])
self.base_results = self.evaluator.evaluate(mnist_test_stream)
# TODO: allow target layer to be parameterized
self.target_layer = '/lenet/mlp/linear_0'
self.next_layer_param = '/lenet/mlp/linear_1.W'
self.base_sample = extract_sample(
outs[self.target_layer], mnist_test_stream)
self.base_param_value = (
model.get_parameter_dict()[
self.next_layer_param].get_value().copy())
def main(num_epochs,
feature_maps=None,
mlp_hiddens=None,
conv_sizes=None,
pool_sizes=None,
batch_size=500,
num_batches=None):
############# Architecture #############
if feature_maps is None:
feature_maps = [20, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5]
if pool_sizes is None:
pool_sizes = [2, 2]
image_size = (32, 32)
batch_size = 50
output_size = 2
learningRate = 0.1
num_epochs = 10
num_batches = None
delta = 0.01
drop_prob = 0.5
weight_noise = 0.75
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
3,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
x = tensor.tensor4('image_features')
y = tensor.lmatrix('targets')
probs = (convnet.apply(x)).copy(name='probs')
# Computational Graph just for cost for drop_out and noise application
cg_probs = ComputationGraph([probs])
inputs = VariableFilter(roles=[INPUT])(cg_probs.variables)
weights = VariableFilter(roles=[FILTER, WEIGHT])(cg_probs.variables)
############# Regularization #############
#regularization = 0
logger.info('Applying regularization')
regularization = delta * sum([(W**2).mean() for W in weights])
probs.name = "reg_probs"
############# Guaussian Noise #############
logger.info('Applying Gaussian noise')
cg_train = apply_noise(cg_probs, weights, weight_noise)
############# Dropout #############
logger.info('Applying dropout')
cg_probs = apply_dropout(cg_probs, inputs, drop_prob)
dropped_out = VariableFilter(roles=[DROPOUT])(cg_probs.variables)
inputs_referenced = [var.tag.replacement_of for var in dropped_out]
set(inputs) == set(inputs_referenced)
############# Batch normalization #############
# recalculate probs after dropout and noise and regularization:
probs = cg_probs.outputs[0] + regularization
cost = (CategoricalCrossEntropy().apply(y.flatten(),
probs).copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
cg = ComputationGraph([probs, cost, error_rate])
cg = apply_batch_normalization(cg)
########### Loading images #####################
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream, ServerDataStream
from fuel.schemes import ShuffledScheme
from fuel.transformers.image import RandomFixedSizeCrop, MinimumImageDimensions, Random2DRotation
from fuel.transformers import Flatten, Cast, ScaleAndShift
def create_data(data):
stream = DataStream(data,
iteration_scheme=ShuffledScheme(
data.num_examples, batch_size))
stream_downscale = MinimumImageDimensions(
stream, image_size, which_sources=('image_features', ))
stream_rotate = Random2DRotation(stream_downscale,
which_sources=('image_features', ))
stream_max = ScikitResize(stream_rotate,
image_size,
which_sources=('image_features', ))
stream_scale = ScaleAndShift(stream_max,
1. / 255,
0,
which_sources=('image_features', ))
stream_cast = Cast(stream_scale,
dtype='float32',
which_sources=('image_features', ))
#stream_flat = Flatten(stream_scale, which_sources=('image_features',))
return stream_cast
stream_data_train = create_data(
DogsVsCats(('train', ), subset=slice(0, 20)))
stream_data_test = create_data(
DogsVsCats(('train', ), subset=slice(20, 30)))
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=learningRate))
#algorithm = GradientDescent(cost=cost, parameters=cg.parameters,step_rule=Adam(0.001))
#algorithm.add_updates(extra_updates)
# `Timing` extension reports time for reading data, aggregating a batch and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = []
extensions.append(Timing())
extensions.append(
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches))
extensions.append(
DataStreamMonitoring([cost, error_rate],
stream_data_test,
prefix="valid"))
extensions.append(
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True))
#extensions.append(Checkpoint(save_to))
extensions.append(ProgressBar())
extensions.append(Printing())
logger.info("Building the model")
model = Model(cost)
main_loop = MainLoop(algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
from fuel.schemes import SequentialScheme
from fuel.transformers import Flatten
# Construct the model
mlp = MLP(activations=[Tanh(), Softmax()],
dims=[784, 100, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
# Calculate the loss function
x = T.matrix('features')
y = T.lmatrix('targets')
y_hat = mlp.apply(x)
cost = CategoricalCrossEntropy().apply(y.flatten(), y_hat)
error_rate = MisclassificationRate().apply(y.flatten(), y_hat)
# load training data using Fuel
mnist_train = MNIST("train")
train_stream = Flatten(
DataStream.default_stream(dataset=mnist_train,
iteration_scheme=SequentialScheme(
mnist_train.num_examples, 128)), )
# load testing data
mnist_test = MNIST("test")
test_stream = Flatten(
DataStream.default_stream(dataset=mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, 1024)), )
def main(save_to, model, train, test, num_epochs, input_size = (150,150), learning_rate=0.01,
batch_size=50, num_batches=None, flatten_stream=False):
"""
save_to : where to save trained model
model : model given in input must be already initialised (works with convnet and mlp)
input_size : the shape of the reshaped image in input (before flattening is applied if flatten_stream is True)
"""
if flatten_stream :
x = tensor.matrix('image_features')
else :
x = tensor.tensor4('image_features')
y = tensor.lmatrix('targets')
#Data augmentation
#insert data augmentation here
#Generating stream
train_stream = DataStream.default_stream(
train,
iteration_scheme=ShuffledScheme(train.num_examples, batch_size)
)
test_stream = DataStream.default_stream(
test,
iteration_scheme=ShuffledScheme(test.num_examples, batch_size)
)
#Reshaping procedure
#Add a crop option in scikitresize so that the image is not deformed
#Resize to desired square shape
train_stream = ScikitResize(train_stream, input_size, which_sources=('image_features',))
test_stream = ScikitResize(test_stream, input_size, which_sources=('image_features',))
#Flattening the stream
if flatten_stream is True:
train_stream = Flatten(train_stream, which_sources=('image_features',))
test_stream = Flatten(test_stream, which_sources=('image_features',))
# Apply input to model
probs = model.apply(x)
#Defining cost and various indices to watch
#print(probs)
#cost = SquaredError().apply(y.flatten(),probs)
cost = CategoricalCrossEntropy().apply(y.flatten(), probs).copy(name='cost')
error_rate = MisclassificationRate().apply(y.flatten(), probs).copy(
name='error_rate')
#Building Computation Graph
cg = ComputationGraph([cost, error_rate])
# Train with simple SGD
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=Scale(learning_rate=learning_rate))
#Defining extensions
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
TrainingDataMonitoring([cost, error_rate,aggregation.mean(algorithm.total_gradient_norm)], prefix="train", every_n_batches=5),
DataStreamMonitoring([cost, error_rate],test_stream,prefix="test", every_n_batches=25),
Checkpoint(save_to),
ProgressBar(),
Printing(every_n_batches=5)]
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
model = Model(cost)
main_loop = MainLoop(
algorithm,
train_stream,
model=model,
extensions=extensions)
main_loop.run()
def build_and_run(save_to,modelconfig,experimentconfig):
""" part of this is adapted from lasagne tutorial"""
n, num_filters, image_size, num_blockstack = modelconfig['depth'], modelconfig['num_filters'], modelconfig['image_size'], modelconfig['num_blockstack']
print("Amount of bottlenecks: %d" % n)
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('image_features')
#target_value = T.ivector('targets')
target_var = T.lmatrix('targets')
target_vec = T.extra_ops.to_one_hot(target_var[:,0],2)
#target_var = T.matrix('targets')
# Create residual net model
print("Building model...")
network = build_cnn(input_var, image_size, n, num_blockstack, num_filters)
get_info(network)
prediction = lasagne.utils.as_theano_expression(lasagne.layers.get_output(network))
test_prediction = lasagne.utils.as_theano_expression(lasagne.layers.get_output(network,deterministic=True))
# Loss function -> The objective to minimize
print("Instanciation of loss function...")
#loss = CategoricalCrossEntropy().apply(target_var.flatten(), prediction)
#test_loss = CategoricalCrossEntropy().apply(target_var.flatten(), test_prediction)
# loss = lasagne.objectives.categorical_crossentropy(prediction, target_var.flatten()).mean()
# test_loss = lasagne.objectives.categorical_crossentropy(test_prediction, target_var.flatten()).mean()
loss = lasagne.objectives.squared_error(prediction,target_vec).mean()
test_loss = lasagne.objectives.squared_error(test_prediction,target_vec).mean()
# loss = tensor.nnet.binary_crossentropy(prediction, target_var).mean()
# test_loss = tensor.nnet.binary_crossentropy(test_prediction, target_var).mean()
test_loss.name = "loss"
# loss.name = 'x-ent_error'
# loss.name = 'sqr_error'
layers = lasagne.layers.get_all_layers(network)
#l1 and l2 regularization
#pondlayers = {x:0.000025 for i,x in enumerate(layers)}
#l1_penality = lasagne.regularization.regularize_layer_params_weighted(pondlayers, lasagne.regularization.l2)
#l2_penality = lasagne.regularization.regularize_layer_params(layers[len(layers)/4:], lasagne.regularization.l1) * 25e-6
#reg_penalty = l1_penality + l2_penality
#reg_penalty.name = 'reg_penalty'
#loss = loss + reg_penalty
loss.name = 'reg_loss'
error_rate = MisclassificationRate().apply(target_var.flatten(), test_prediction).copy(
name='error_rate')
# Load the dataset
print("Loading data...")
istest = 'test' in experimentconfig.keys()
if istest:
print("Using test stream")
train_stream, valid_stream, test_stream = get_stream(experimentconfig['batch_size'],image_size,test=istest)
# Defining step rule and algorithm
if 'step_rule' in experimentconfig.keys() and not experimentconfig['step_rule'] is None :
step_rule = experimentconfig['step_rule'](learning_rate=experimentconfig['learning_rate'])
else :
step_rule=Scale(learning_rate=experimentconfig['learning_rate'])
params = map(lasagne.utils.as_theano_expression,lasagne.layers.get_all_params(network, trainable=True))
print("Initializing algorithm")
algorithm = GradientDescent(
cost=loss, gradients={var:T.grad(loss,var) for var in params},#parameters=cg.parameters, #params
step_rule=step_rule)
#algorithm.add_updates(extra_updates)
grad_norm = aggregation.mean(algorithm.total_gradient_norm)
grad_norm.name = "grad_norm"
print("Initializing extensions...")
plot = Plot(save_to, channels=[['train_loss','valid_loss'],
['train_grad_norm'],
#['train_grad_norm','train_reg_penalty'],
['train_error_rate','valid_error_rate']], server_url='http://hades.calculquebec.ca:5042')
checkpoint = Checkpoint('models/best_'+save_to+'.tar')
# checkpoint.add_condition(['after_n_batches=25'],
checkpoint.add_condition(['after_epoch'],
predicate=OnLogRecord('valid_error_rate_best_so_far'))
#Defining extensions
extensions = [Timing(),
FinishAfter(after_n_epochs=experimentconfig['num_epochs'],
after_n_batches=experimentconfig['num_batches']),
TrainingDataMonitoring([test_loss, error_rate, grad_norm], # reg_penalty],
prefix="train", after_epoch=True), #after_n_epochs=1
DataStreamMonitoring([test_loss, error_rate],valid_stream,prefix="valid", after_epoch=True), #after_n_epochs=1
plot,
#Checkpoint(save_to,after_n_epochs=5),
#ProgressBar(),
# Plot(save_to, channels=[['train_loss','valid_loss'], ['train_error_rate','valid_error_rate']], server_url='http://hades.calculquebec.ca:5042'), #'grad_norm'
# after_batch=True),
Printing(after_epoch=True),
TrackTheBest('valid_error_rate',min), #Keep best
checkpoint, #Save best
FinishIfNoImprovementAfter('valid_error_rate_best_so_far', epochs=5)] # Early-stopping
# model = Model(loss)
# print("Model",model)
main_loop = MainLoop(
algorithm,
train_stream,
# model=model,
extensions=extensions)
print("Starting main loop...")
main_loop.run()
def main():
feature_maps = [20, 50]
mlp_hiddens = [50]
conv_sizes = [5, 5]
pool_sizes = [3, 3]
save_to = "DvC.pkl"
batch_size = 500
image_size = (32, 32)
output_size = 2
learningRate = 0.1
num_epochs = 10
num_batches = None
host_plot = 'http://*****:*****@ %s' %
('CNN ', datetime.datetime.now(), socket.gethostname()),
channels=[['valid_cost', 'valid_error_rate'],
['train_total_gradient_norm']],
after_epoch=True,
server_url=host_plot))
model = Model(cost)
main_loop = MainLoop(algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
def main():
mlp_hiddens = [1000]
filter_sizes = [(9, 9), (5, 5), (5, 5)]
feature_maps = [80, 50, 20]
pooling_sizes = [(3, 3), (2, 2), (2, 2)]
save_to = "DvC.pkl"
image_size = (128, 128)
output_size = 2
learningRate = 0.1
num_epochs = 300
num_batches = None
if socket.gethostname() == 'tim-X550JX':
host_plot = 'http://*****:*****@ %s' %
('CNN ', datetime.datetime.now(), socket.gethostname()),
channels=[['train_error_rate', 'valid_error_rate'],
['train_total_gradient_norm']],
after_epoch=True,
server_url=host_plot))
model = Model(cost)
main_loop = MainLoop(algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
x_to_h1 = Linear(name='x_to_h1', input_dim=x_dim, output_dim=h_dim)
pre_rnn = x_to_h1.apply(x)
rnn = SimpleRecurrent(activation=Rectifier(), dim=h_dim, name="rnn")
h1 = rnn.apply(pre_rnn)
h1_to_o = Linear(name='h1_to_o', input_dim=h_dim, output_dim=o_dim)
pre_softmax = h1_to_o.apply(h1)
softmax = Softmax()
shape = pre_softmax.shape
softmax_out = softmax.apply(pre_softmax.reshape((-1, o_dim)))
softmax_out = softmax_out.reshape(shape)
softmax_out.name = 'softmax_out'
# comparing only last time-step
cost = CategoricalCrossEntropy().apply(y[-1, :, 0], softmax_out[-1])
cost.name = 'CrossEntropy'
error_rate = MisclassificationRate().apply(y[-1, :, 0], softmax_out[-1])
error_rate.name = 'error_rate'
# Initialization
for brick in (x_to_h1, h1_to_o):
brick.weights_init = IsotropicGaussian(0.01)
brick.biases_init = Constant(0)
brick.initialize()
rnn.weights_init = Identity()
rnn.biases_init = Constant(0)
rnn.initialize()
print 'Bulding training process...'
algorithm = GradientDescent(cost=cost,
parameters=ComputationGraph(cost).parameters,
step_rule=learning_algorithm(
########## hyper parameters###########################################
# We push initialization config to set different initialization schemes
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=0.2)
convnet.layers[1].weights_init = Uniform(width=0.2)
convnet.layers[2].weights_init = Uniform(width=0.2)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=0.2)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=0.2)
convnet.initialize()
#########################################################
#Generate output and error signal
predict = convnet.apply(x)
cost = CategoricalCrossEntropy().apply(y.flatten(), predict).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), predict)
#Little trick to plot the error rate in two different plots (We can't use two time the same data in the plot for a unknow reason)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
cg = ComputationGraph([cost, error_rate])
########### ALGORITHM of training#############
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Adam(learning_rate=0.0005))
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate, error_rate2],
stream_valid,
prefix="valid"),
TrainingDataMonitoring(
def main(feature_maps=None, mlp_hiddens=None,
conv_sizes=None, pool_sizes=None, batch_size=None,
num_batches=None):
if feature_maps is None:
feature_maps = [32, 48, 64, 96, 96, 128]
if mlp_hiddens is None:
mlp_hiddens = [1000]
if conv_sizes is None:
conv_sizes = [9, 7, 5, 3, 2, 1]
if pool_sizes is None:
pool_sizes = [2, 2, 2, 2, 1, 1]
if batch_size is None:
batch_size = 64
conv_steps=[2, 1, 1, 1, 1, 1] #same as stride
image_size = (128, 128)
output_size = 2
learningRate = 0.001
drop_prob = 0.4
weight_noise = 0.75
num_epochs = 150
num_batches = None
host_plot='http://*****:*****@ %s' % (graph_name, datetime.datetime.now(), socket.gethostname()),
channels=[['train_error_rate', 'valid_error_rate'],
['train_total_gradient_norm']], after_epoch=True, server_url=host_plot))
PLOT_AVAILABLE = True
except ImportError:
PLOT_AVAILABLE = False
extensions.append(Checkpoint(save_to, after_epoch=True, after_training=True, save_separately=['log']))
logger.info("Building the model")
model = Model(cost)
########### Loading images #####################
main_loop = MainLoop(
algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
def main(save_to, num_epochs,
regularization=0.0003, subset=None, num_batches=None,
histogram=None, resume=False):
batch_size = 500
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
.copy(name='cost'))
components = (ComponentwiseCrossEntropy().apply(y.flatten(), probs)
.copy(name='components'))
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
cg = ComputationGraph([cost, error_rate, components])
# Apply regularization to the cost
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
l2_norm = sum([(W ** 2).sum() for W in weights])
l2_norm.name = 'l2_norm'
cost = cost + regularization * l2_norm
cost.name = 'cost_with_regularization'
if subset:
start = 30000 - subset // 2
mnist_train = MNIST(("train",), subset=slice(start, start+subset))
else:
mnist_train = MNIST(("train",))
mnist_train_stream = DataStream.default_stream(
mnist_train, iteration_scheme=ShuffledScheme(
mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(
mnist_test.num_examples, batch_size))
# Train with simple SGD
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=AdaDelta(decay_rate=0.99))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
DataStreamMonitoring(
[cost, error_rate, confusion],
mnist_test_stream,
prefix="test"),
TrainingDataMonitoring(
[cost, error_rate, l2_norm,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()]
if histogram:
attribution = AttributionExtension(
components=components,
parameters=cg.parameters,
components_size=output_size,
after_batch=True)
extensions.insert(0, attribution)
if resume:
extensions.append(Load(save_to, True, True))
model = Model(cost)
main_loop = MainLoop(
algorithm,
mnist_train_stream,
model=model,
extensions=extensions)
main_loop.run()
if histogram:
save_attributions(attribution, filename=histogram)
with open('execution-log.json', 'w') as outfile:
json.dump(main_loop.log, outfile, cls=NumpyEncoder)
def apply(self, input_labeled, target_labeled, input_unlabeled):
self.layer_counter = 0
input_dim = self.p.encoder_layers[0]
# Store the dimension tuples in the same order as layers.
layers = self.layers
self.layer_dims = {0: input_dim}
self.lr = self.shared(self.default_lr, 'learning_rate', role=None)
self.costs = costs = AttributeDict()
self.costs.denois = AttributeDict()
self.act = AttributeDict()
self.error = AttributeDict()
top = len(layers) - 1
N = input_labeled.shape[0]
self.join = lambda l, u: T.concatenate([l, u], axis=0)
self.labeled = lambda x: x[:N] if x is not None else x
self.unlabeled = lambda x: x[N:] if x is not None else x
self.split_lu = lambda x: (self.labeled(x), self.unlabeled(x))
input_concat = self.join(input_labeled, input_unlabeled)
def encoder(input_, path_name, input_noise_std=0, noise_std=[]):
h = input_
logger.info(' 0: noise %g' % input_noise_std)
if input_noise_std > 0.:
h = h + self.noise_like(h) * input_noise_std
d = AttributeDict()
d.unlabeled = self.new_activation_dict()
d.labeled = self.new_activation_dict()
d.labeled.z[0] = self.labeled(h)
d.unlabeled.z[0] = self.unlabeled(h)
prev_dim = input_dim
for i, (spec, _, act_f) in layers[1:]:
d.labeled.h[i - 1], d.unlabeled.h[i - 1] = self.split_lu(h)
noise = noise_std[i] if i < len(noise_std) else 0.
curr_dim, z, m, s, h = self.f(h,
prev_dim,
spec,
i,
act_f,
path_name=path_name,
noise_std=noise)
assert self.layer_dims.get(i) in (None, curr_dim)
self.layer_dims[i] = curr_dim
d.labeled.z[i], d.unlabeled.z[i] = self.split_lu(z)
d.unlabeled.s[i] = s
d.unlabeled.m[i] = m
prev_dim = curr_dim
d.labeled.h[i], d.unlabeled.h[i] = self.split_lu(h)
return d
# Clean, supervised
logger.info('Encoder: clean, labeled')
clean = self.act.clean = encoder(input_concat, 'clean')
# Corrupted, supervised
logger.info('Encoder: corr, labeled')
corr = self.act.corr = encoder(input_concat,
'corr',
input_noise_std=self.p.super_noise_std,
noise_std=self.p.f_local_noise_std)
est = self.act.est = self.new_activation_dict()
# Decoder path in opposite order
logger.info('Decoder: z_corr -> z_est')
for i, ((_, spec), l_type, act_f) in layers[::-1]:
z_corr = corr.unlabeled.z[i]
z_clean = clean.unlabeled.z[i]
z_clean_s = clean.unlabeled.s.get(i)
z_clean_m = clean.unlabeled.m.get(i)
fspec = layers[i + 1][1][0] if len(layers) > i + 1 else (None,
None)
if i == top:
ver = corr.unlabeled.h[i]
ver_dim = self.layer_dims[i]
top_g = True
else:
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
l_type=l_type,
num=i,
fspec=fspec,
top_g=top_g)
if z_est is not None:
# Denoising cost
if z_clean_s:
z_est_norm = (z_est - z_clean_m) / z_clean_s
else:
z_est_norm = z_est
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
denois_print = 'denois %.2f' % self.p.denoising_cost_x[i]
else:
denois_print = ''
# Store references for later use
est.h[i] = self.apply_act(z_est, act_f)
est.z[i] = z_est
est.s[i] = None
est.m[i] = None
logger.info(' g%d: %10s, %s, dim %s -> %s' %
(i, l_type, denois_print, self.layer_dims.get(i + 1),
self.layer_dims.get(i)))
# Costs
y = target_labeled.flatten()
costs.class_clean = CategoricalCrossEntropy().apply(
y, clean.labeled.h[top])
costs.class_clean.name = 'cost_class_clean'
costs.class_corr = CategoricalCrossEntropy().apply(
y, corr.labeled.h[top])
costs.class_corr.name = 'cost_class_corr'
# This will be used for training
costs.total = costs.class_corr * 1.0
for i in range(top + 1):
if costs.denois.get(i) and self.p.denoising_cost_x[i] > 0:
costs.total += costs.denois[i] * self.p.denoising_cost_x[i]
costs.total.name = 'cost_total'
# Classification error
mr = MisclassificationRate()
self.error.clean = mr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.error.clean.name = 'error_rate_clean'
def main(save_to,
num_epochs,
feature_maps=None,
mlp_hiddens=None,
conv_sizes=None,
pool_sizes=None,
batch_size=500):
if feature_maps is None:
feature_maps = [20, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5]
if pool_sizes is None:
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
1,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
logging.info("Layer {} dim: {} {} {}".format(i,
*layer.get_dim('output')))
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = named_copy(CategoricalCrossEntropy().apply(y.flatten(), probs),
'cost')
error_rate = named_copy(MisclassificationRate().apply(y.flatten(), probs),
'error_rate')
cg = ComputationGraph([cost, error_rate])
mnist_train = MNIST(("train", ))
mnist_train_stream = DataStream.default_stream(
mnist_train,
iteration_scheme=ShuffledScheme(mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(mnist_test.num_examples, batch_size))
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=0.1))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate],
mnist_test_stream,
prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()
]
model = Model(cost)
main_loop = MainLoop(algorithm,
mnist_train_stream,
model=model,
extensions=extensions)
main_loop.run()
def create_main_loop(save_to,
num_epochs,
unit_order=None,
batch_size=500,
num_batches=None):
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
case_costs = CasewiseCrossEntropy().apply(y.flatten(), probs)
cost = case_costs.mean().copy(name='cost')
# cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
# .copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
cg = ComputationGraph([cost, error_rate])
# Apply regularization to the cost
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
cost = cost + sum([0.0003 * (W**2).sum() for W in weights])
cost.name = 'cost_with_regularization'
mnist_train = MNIST(("train", ))
mnist_train_stream = DataStream.default_stream(
mnist_train,
iteration_scheme=ShuffledScheme(mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(mnist_test.num_examples, batch_size))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=AdaDelta())
# Find layer outputs to probe
outs = OrderedDict(
reversed(
list((get_brick(out).name, out)
for out in VariableFilter(roles=[OUTPUT],
bricks=[Convolutional, Linear])(
cg.variables))))
actpic_extension = ActpicExtension(actpic_variables=outs,
case_labels=y,
pics=x,
label_count=output_size,
rectify=-1,
data_stream=mnist_test_stream,
after_batch=True)
synpic_extension = SynpicExtension(synpic_parameters=biases,
case_costs=case_costs,
case_labels=y,
pics=x,
batch_size=batch_size,
pic_size=image_size,
label_count=output_size,
after_batch=True)
# Impose an orderint for the SaveImages extension
if unit_order is not None:
with open(unit_order, 'rb') as handle:
histograms = pickle.load(handle)
unit_order = compute_unit_order(histograms)
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches),
actpic_extension, synpic_extension,
SaveImages(picsources=[synpic_extension, actpic_extension],
title="LeNet-5: batch {i}, " +
"cost {cost_with_regularization:.2f}, " +
"trainerr {error_rate:.3f}",
data=[cost, error_rate],
graph='error_rate',
graph_len=500,
unit_order=unit_order,
after_batch=True),
DataStreamMonitoring([cost, error_rate],
mnist_test_stream,
prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()
]
model = Model(cost)
main_loop = MainLoop(algorithm,
mnist_train_stream,
model=model,
extensions=extensions)
return main_loop
