convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=0.2)
convnet.layers[1].weights_init = Uniform(width=0.2)
convnet.layers[2].weights_init = Uniform(width=0.2)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=0.2)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=0.2)
convnet.initialize()
#########################################################
#Generate output and error signal
predict = convnet.apply(x)
cost = CategoricalCrossEntropy().apply(y.flatten(), predict).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), predict)
#Little trick to plot the error rate in two different plots (We can't use two time the same data in the plot for a unknow reason)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
cg = ComputationGraph([cost, error_rate])
########### ALGORITHM of training#############
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Adam(learning_rate=0.0005))
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate, error_rate2],
stream_valid,
prefix="valid"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
data_train_stream = create_data(DogsVsCats(('train',), subset=slice(0, 100)))
data_valid_stream = create_data(DogsVsCats(('train',), subset=slice(100, 110)))
if mode is "GPU_run":
data_train_stream = create_data(DogsVsCats(('train',), subset=slice(0, 22500)))
data_valid_stream = create_data(DogsVsCats(('train',), subset=slice(22500, 25000)))
if mode is "data_server":
data_train_stream = ServerDataStream(('image_features','targets'), False, port=5560)
data_valid_stream = ServerDataStream(('image_features','targets'), False, port=5561)
### Setting up the model
probs = top_mlp.apply(conv_out)
cost = CategoricalCrossEntropy().apply(y.flatten(), probs).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), probs)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
cg = ComputationGraph([cost, error_rate])
### Gradient Descent
algorithm = GradientDescent(cost=cost, parameters=cg.parameters, step_rule=Scale(learning_rate=learning_rate))
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring(
[cost, error_rate, error_rate2],
data_valid_stream,
prefix="valid"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
def build_and_run(label, config):
############## CREATE THE NETWORK ###############
#Define the parameters
num_epochs, num_batches, num_channels, image_shape, filter_size, num_filter, pooling_sizes, mlp_hiddens, output_size, batch_size, activation, mlp_activation = config[
'num_epochs'], config['num_batches'], config['num_channels'], config[
'image_shape'], config['filter_size'], config[
'num_filter'], config['pooling_sizes'], config[
'mlp_hiddens'], config['output_size'], config[
'batch_size'], config['activation'], config[
'mlp_activation']
# print(num_epochs, num_channels, image_shape, filter_size, num_filter, pooling_sizes, mlp_hiddens, output_size, batch_size, activation, mlp_activation)
lambda_l1 = 0.000025
lambda_l2 = 0.000025
print("Building model")
#Create the symbolics variable
x = T.tensor4('image_features')
y = T.lmatrix('targets')
#Get the parameters
conv_parameters = zip(filter_size, num_filter)
#Create the convolutions layers
conv_layers = list(
interleave([(Convolutional(filter_size=filter_size,
num_filters=num_filter,
name='conv_{}'.format(i))
for i, (filter_size,
num_filter) in enumerate(conv_parameters)),
(activation),
(MaxPooling(size, name='pool_{}'.format(i))
for i, size in enumerate(pooling_sizes))]))
# (AveragePooling(size, name='pool_{}'.format(i)) for i, size in enumerate(pooling_sizes))]))
#Create the sequence
conv_sequence = ConvolutionalSequence(conv_layers,
num_channels,
image_size=image_shape,
weights_init=Uniform(width=0.2),
biases_init=Constant(0.))
#Initialize the convnet
conv_sequence.initialize()
#Add the MLP
top_mlp_dims = [np.prod(conv_sequence.get_dim('output'))
] + mlp_hiddens + [output_size]
out = Flattener().apply(conv_sequence.apply(x))
mlp = MLP(mlp_activation,
top_mlp_dims,
weights_init=Uniform(0, 0.2),
biases_init=Constant(0.))
#Initialisze the MLP
mlp.initialize()
#Get the output
predict = mlp.apply(out)
cost = CategoricalCrossEntropy().apply(y.flatten(),
predict).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), predict)
#Little trick to plot the error rate in two different plots (We can't use two time the same data in the plot for a unknow reason)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
########### REGULARIZATION ##################
cg = ComputationGraph([cost])
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
biases = VariableFilter(roles=[BIAS])(cg.variables)
# # l2_penalty_weights = T.sum([i*lambda_l2/len(weights) * (W ** 2).sum() for i,W in enumerate(weights)]) # Gradually increase penalty for layer
l2_penalty = T.sum([
lambda_l2 * (W**2).sum() for i, W in enumerate(weights + biases)
]) # Gradually increase penalty for layer
# # #l2_penalty_bias = T.sum([lambda_l2*(B **2).sum() for B in biases])
# # #l2_penalty = l2_penalty_weights + l2_penalty_bias
l2_penalty.name = 'l2_penalty'
l1_penalty = T.sum([lambda_l1 * T.abs_(z).sum() for z in weights + biases])
# l1_penalty_weights = T.sum([i*lambda_l1/len(weights) * T.abs_(W).sum() for i,W in enumerate(weights)]) # Gradually increase penalty for layer
# l1_penalty_biases = T.sum([lambda_l1 * T.abs_(B).sum() for B in biases])
# l1_penalty = l1_penalty_biases + l1_penalty_weights
l1_penalty.name = 'l1_penalty'
costreg = cost + l2_penalty + l1_penalty
costreg.name = 'costreg'
########### DEFINE THE ALGORITHM #############
# algorithm = GradientDescent(cost=cost, parameters=cg.parameters, step_rule=Momentum())
algorithm = GradientDescent(cost=costreg,
parameters=cg.parameters,
step_rule=Adam())
########### GET THE DATA #####################
istest = 'test' in config.keys()
train_stream, valid_stream, test_stream = get_stream(batch_size,
image_shape,
test=istest)
########### INITIALIZING EXTENSIONS ##########
checkpoint = Checkpoint('models/best_' + label + '.tar')
checkpoint.add_condition(
['after_epoch'], predicate=OnLogRecord('valid_error_rate_best_so_far'))
#Adding a live plot with the bokeh server
plot = Plot(
label,
channels=[
['train_error_rate', 'valid_error_rate'],
['valid_cost', 'valid_error_rate2'],
# ['train_costreg','train_grad_norm']], #
[
'train_costreg', 'train_total_gradient_norm',
'train_l2_penalty', 'train_l1_penalty'
]
],
server_url="http://hades.calculquebec.ca:5042")
grad_norm = aggregation.mean(algorithm.total_gradient_norm)
grad_norm.name = 'grad_norm'
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches),
DataStreamMonitoring([cost, error_rate, error_rate2],
valid_stream,
prefix="valid"),
TrainingDataMonitoring([
costreg, error_rate, error_rate2, grad_norm, l2_penalty, l1_penalty
],
prefix="train",
after_epoch=True),
plot,
ProgressBar(),
Printing(),
TrackTheBest('valid_error_rate', min), #Keep best
checkpoint, #Save best
FinishIfNoImprovementAfter('valid_error_rate_best_so_far', epochs=4)
] # Early-stopping
model = Model(cost)
main_loop = MainLoop(algorithm,
data_stream=train_stream,
model=model,
extensions=extensions)
main_loop.run()
def main(feature_maps=None, mlp_hiddens=None, conv_sizes=None, pool_sizes=None, batch_size=None, num_batches=None):
if feature_maps is None:
feature_maps = [32, 48, 64, 80, 128, 128]
if mlp_hiddens is None:
mlp_hiddens = [1000]
if conv_sizes is None:
conv_sizes = [7, 5, 5, 5, 5, 4]
if pool_sizes is None:
pool_sizes = [3, 2, 2, 2, 2, 1]
if batch_size is None:
batch_size = 64
conv_steps = [2, 1, 1, 1, 1, 1] # same as stride
image_size = (256, 256)
output_size = 2
learningRate = 0.001
drop_prob = 0.5
weight_noise = 0.75
num_epochs = 250
num_batches = None
host_plot = "http://*****:*****@ %s" % (graph_name, datetime.datetime.now(), socket.gethostname()),
channels=[["train_error_rate", "valid_error_rate"], ["train_total_gradient_norm"]],
after_epoch=True,
server_url=host_plot,
)
)
PLOT_AVAILABLE = True
except ImportError:
PLOT_AVAILABLE = False
extensions.append(Checkpoint(save_to, after_epoch=True, after_training=True, save_separately=["log"]))
logger.info("Building the model")
model = Model(cost)
########### Loading images #####################
main_loop = MainLoop(algorithm, stream_data_train, model=model, extensions=extensions)
main_loop.run()
def build_and_run(label, config):
############## CREATE THE NETWORK ###############
#Define the parameters
num_epochs, num_batches, num_channels, image_shape, filter_size, num_filter, pooling_sizes, mlp_hiddens, output_size, batch_size, activation, mlp_activation = config['num_epochs'], config['num_batches'], config['num_channels'], config['image_shape'], config['filter_size'], config['num_filter'], config['pooling_sizes'], config['mlp_hiddens'], config['output_size'], config['batch_size'], config['activation'], config['mlp_activation']
# print(num_epochs, num_channels, image_shape, filter_size, num_filter, pooling_sizes, mlp_hiddens, output_size, batch_size, activation, mlp_activation)
lambda_l1 = 0.000025
lambda_l2 = 0.000025
print("Building model")
#Create the symbolics variable
x = T.tensor4('image_features')
y = T.lmatrix('targets')
#Get the parameters
conv_parameters = zip(filter_size, num_filter)
#Create the convolutions layers
conv_layers = list(interleave([(Convolutional(
filter_size=filter_size,
num_filters=num_filter,
name='conv_{}'.format(i))
for i, (filter_size, num_filter)
in enumerate(conv_parameters)),
(activation),
(MaxPooling(size, name='pool_{}'.format(i)) for i, size in enumerate(pooling_sizes))]))
# (AveragePooling(size, name='pool_{}'.format(i)) for i, size in enumerate(pooling_sizes))]))
#Create the sequence
conv_sequence = ConvolutionalSequence(conv_layers, num_channels, image_size=image_shape, weights_init=Uniform(width=0.2), biases_init=Constant(0.))
#Initialize the convnet
conv_sequence.initialize()
#Add the MLP
top_mlp_dims = [np.prod(conv_sequence.get_dim('output'))] + mlp_hiddens + [output_size]
out = Flattener().apply(conv_sequence.apply(x))
mlp = MLP(mlp_activation, top_mlp_dims, weights_init=Uniform(0, 0.2),
biases_init=Constant(0.))
#Initialisze the MLP
mlp.initialize()
#Get the output
predict = mlp.apply(out)
cost = CategoricalCrossEntropy().apply(y.flatten(), predict).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), predict)
#Little trick to plot the error rate in two different plots (We can't use two time the same data in the plot for a unknow reason)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
########### REGULARIZATION ##################
cg = ComputationGraph([cost])
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
biases = VariableFilter(roles=[BIAS])(cg.variables)
# # l2_penalty_weights = T.sum([i*lambda_l2/len(weights) * (W ** 2).sum() for i,W in enumerate(weights)]) # Gradually increase penalty for layer
l2_penalty = T.sum([lambda_l2 * (W ** 2).sum() for i,W in enumerate(weights+biases)]) # Gradually increase penalty for layer
# # #l2_penalty_bias = T.sum([lambda_l2*(B **2).sum() for B in biases])
# # #l2_penalty = l2_penalty_weights + l2_penalty_bias
l2_penalty.name = 'l2_penalty'
l1_penalty = T.sum([lambda_l1*T.abs_(z).sum() for z in weights+biases])
# l1_penalty_weights = T.sum([i*lambda_l1/len(weights) * T.abs_(W).sum() for i,W in enumerate(weights)]) # Gradually increase penalty for layer
# l1_penalty_biases = T.sum([lambda_l1 * T.abs_(B).sum() for B in biases])
# l1_penalty = l1_penalty_biases + l1_penalty_weights
l1_penalty.name = 'l1_penalty'
costreg = cost + l2_penalty + l1_penalty
costreg.name = 'costreg'
########### DEFINE THE ALGORITHM #############
# algorithm = GradientDescent(cost=cost, parameters=cg.parameters, step_rule=Momentum())
algorithm = GradientDescent(cost=costreg, parameters=cg.parameters, step_rule=Adam())
########### GET THE DATA #####################
istest = 'test' in config.keys()
train_stream, valid_stream, test_stream = get_stream(batch_size,image_shape,test=istest)
########### INITIALIZING EXTENSIONS ##########
checkpoint = Checkpoint('models/best_'+label+'.tar')
checkpoint.add_condition(['after_epoch'],
predicate=OnLogRecord('valid_error_rate_best_so_far'))
#Adding a live plot with the bokeh server
plot = Plot(label,
channels=[['train_error_rate', 'valid_error_rate'],
['valid_cost', 'valid_error_rate2'],
# ['train_costreg','train_grad_norm']], #
['train_costreg','train_total_gradient_norm','train_l2_penalty','train_l1_penalty']],
server_url="http://hades.calculquebec.ca:5042")
grad_norm = aggregation.mean(algorithm.total_gradient_norm)
grad_norm.name = 'grad_norm'
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
DataStreamMonitoring([cost, error_rate, error_rate2], valid_stream, prefix="valid"),
TrainingDataMonitoring([costreg, error_rate, error_rate2,
grad_norm,l2_penalty,l1_penalty],
prefix="train", after_epoch=True),
plot,
ProgressBar(),
Printing(),
TrackTheBest('valid_error_rate',min), #Keep best
checkpoint, #Save best
FinishIfNoImprovementAfter('valid_error_rate_best_so_far', epochs=4)] # Early-stopping
model = Model(cost)
main_loop = MainLoop(algorithm,data_stream=train_stream,model=model,extensions=extensions)
main_loop.run()
