def create_data(data, size, batch_size, _port):
if data == "train":
cats = DogsVsCats(('train', ), subset=slice(0, 20000))
port = _port + 2
elif data == "valid":
cats = DogsVsCats(('train', ), subset=slice(20000, 25000))
port = _port + 3
print 'port', port
stream = DataStream.default_stream(cats,
iteration_scheme=ShuffledScheme(
cats.num_examples, batch_size))
stream_downscale = MinimumImageDimensions(
stream, size, which_sources=('image_features', ))
stream_rotate = FlipAsYouCan(stream_downscale, )
stream_max = ScikitResize(stream_rotate,
image_size,
which_sources=('image_features', ))
stream_scale = ScaleAndShift(stream_max,
1. / 255,
0,
which_sources=('image_features', ))
stream_data = Cast(stream_scale,
dtype='float32',
which_sources=('image_features', ))
start_server(stream_data, port=port)
def _test_dataset():
train = DogsVsCats(('train', ))
assert train.num_examples == 25000
assert_raises(ValueError, DogsVsCats, ('valid', ))
test = DogsVsCats(('test', ))
stream = DataStream.default_stream(test,
iteration_scheme=SequentialScheme(
10, 10))
data = next(stream.get_epoch_iterator())[0][0]
assert data.dtype.kind == 'f'
def load_dataset(self, source):
# Splitting the dataset
if self.division == "leaderboard":
index_train, index_valid = dataset_division_leaderboard()
else:
index_train, index_valid, index_test = dataset_division()
for i in range(self.bagging_iterator):
np.random.shuffle(index_train)
index_train = index_train[0:int(self.bagging * index_train.shape[0])]
if self.mode == "train":
dataset = DogsVsCats(('train', ))
if self.shuffle:
scheme = ShuffledScheme(index_train, self.batch_size)
else:
scheme = SequentialScheme(index_train, self.batch_size)
elif self.mode == "valid":
dataset = DogsVsCats(('train', ))
if self.shuffle:
scheme = ShuffledScheme(index_valid, self.batch_size)
else:
scheme = SequentialScheme(index_valid, self.batch_size)
elif self.mode == "test":
if self.division == "leaderboard":
self.shuffle = False
dataset = DogsVsCats(('test', ))
scheme = SequentialScheme(range(12500), self.batch_size)
else:
dataset = DogsVsCats(('train', ))
if self.shuffle:
scheme = ShuffledScheme(index_test, self.batch_size)
else:
scheme = SequentialScheme(index_test, self.batch_size)
else:
raise Exception(
"Mode not understood. Use : train, valid or test. Here : %s" %
self.mode)
stream = DataStream(dataset, iteration_scheme=scheme)
if self.tmp_size[2] == 1:
downscaled_stream = ResizeAndGrayscaleImage(
stream,
self.tmp_size[0:2],
resample="bicubic",
which_sources=(source, ))
elif self.tmp_size[2] == 3:
downscaled_stream = ResizeImage(stream,
self.tmp_size[0:2],
resample="bicubic",
which_sources=(source, ))
else:
raise Exception("tmp_size[2] = 1 or 3. Here : %d" %
self.tmp_size[2])
return downscaled_stream
def get_stream(batch_size, input_size, test=False):
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream
from fuel.schemes import ShuffledScheme, SequentialScheme, SequentialExampleScheme
from fuel.transformers.image import RandomFixedSizeCrop
from fuel.transformers import Flatten #, ForceFloatX
from ScikitResize import ScikitResize
from fuel.transformers import Cast
# Load the training set
if test:
train = DogsVsCats(('train', ), subset=slice(0, 30))
valid = DogsVsCats(('train', ), subset=slice(19980, 20000))
test = DogsVsCats(('test', ), subset=slice(0, 4))
else:
train = DogsVsCats(('train', ), subset=slice(0, 22000))
valid = DogsVsCats(('train', ), subset=slice(22000, 25000))
test = DogsVsCats(('test', ))
#Generating stream
train_stream = DataStream.default_stream(train,
iteration_scheme=ShuffledScheme(
train.num_examples,
batch_size))
valid_stream = DataStream.default_stream(valid,
iteration_scheme=ShuffledScheme(
valid.num_examples,
batch_size))
test_stream = DataStream.default_stream(
test,
iteration_scheme=SequentialScheme(test.num_examples, 1)
# iteration_scheme=SequentialExampleScheme(test.num_examples)
)
#Reshaping procedure
#Apply crop and resize to desired square shape
train_stream = ScikitResize(train_stream,
input_size,
which_sources=('image_features', ))
valid_stream = ScikitResize(valid_stream,
input_size,
which_sources=('image_features', ))
test_stream = ScikitResize(test_stream,
input_size,
which_sources=('image_features', ))
#ForceFloatX, to spare you from possible bugs
#train_stream = ForceFloatX(train_stream)
#valid_stream = ForceFloatX(valid_stream)
#test_stream = ForceFloatX(test_stream)
#Cast instead of forcefloatX
train_stream = Cast(train_stream,
dtype='float32',
which_sources=('image_features', ))
valid_stream = Cast(valid_stream,
dtype='float32',
which_sources=('image_features', ))
test_stream = Cast(test_stream,
dtype='float32',
which_sources=('image_features', ))
return train_stream, valid_stream, test_stream
if __name__ == '__main__':
# Let's load and process the dataset
import numpy as np
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream
from fuel.schemes import ShuffledScheme
from fuel.transformers.image import RandomFixedSizeCrop
from fuel.transformers import Flatten
# Load the training set
train = DogsVsCats(('train',),subset=slice(0, 20)) #subset=slice(0, 20000)
test = DogsVsCats(('test',),subset=slice(0,20))
input_size = (150,150)
from models import mlp,convnet
#main(None,mlp(input_size[0]*input_size[1]*3), train, test, num_epochs=1, input_size=input_size, batch_size=5, num_batches=20, flatten_stream=True)
main("test1.txt", convnet(input_size), train, test, num_epochs=1, input_size=input_size, batch_size=64, num_batches=100)
# from deep_res import build_cnn
# model = build_cnn(x,3,64)
#
# THEANO_FLAGS='cuda.root=/usr/lib/nvidia-cuda-toolkit/', THEANO_FLAGS=cuda.root=/usr/lib/nvidia-cuda-toolkit/,device=gpu,floatX=float32 python dogs_cats.py
# THEANO_FLAGS=device=gpu
stream = DataStream(data, iteration_scheme=ShuffledScheme(data.num_examples, batch_size))
# Data Augmentation
stream = MinimumImageDimensions(stream, image_size, which_sources=('image_features',))
stream = MaximumImageDimensions(stream, image_size, which_sources=('image_features',))
stream = RandomHorizontalSwap(stream, which_sources=('image_features',))
stream = Random2DRotation(stream, which_sources=('image_features',))
# Data Transformation
stream = ScaleAndShift(stream, 1./255, 0, which_sources=('image_features',))
stream = Cast(stream, dtype='float32', which_sources=('image_features',))
return stream
if mode is "CPU_test":
data_train_stream = create_data(DogsVsCats(('train',), subset=slice(0, 100)))
data_valid_stream = create_data(DogsVsCats(('train',), subset=slice(100, 110)))
if mode is "GPU_run":
data_train_stream = create_data(DogsVsCats(('train',), subset=slice(0, 22500)))
data_valid_stream = create_data(DogsVsCats(('train',), subset=slice(22500, 25000)))
if mode is "data_server":
data_train_stream = ServerDataStream(('image_features','targets'), False, port=5560)
data_valid_stream = ServerDataStream(('image_features','targets'), False, port=5561)
### Setting up the model
probs = top_mlp.apply(conv_out)
cost = CategoricalCrossEntropy().apply(y.flatten(), probs).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), probs)
error_rate = error.copy(name='error_rate')
def main(feature_maps=None, mlp_hiddens=None,
conv_sizes=None, pool_sizes=None, batch_size=None,
num_batches=None):
if feature_maps is None:
feature_maps = [32, 48, 64, 96, 96, 128]
if mlp_hiddens is None:
mlp_hiddens = [1000]
if conv_sizes is None:
conv_sizes = [9, 7, 5, 3, 2, 1]
if pool_sizes is None:
pool_sizes = [2, 2, 2, 2, 1, 1]
if batch_size is None:
batch_size = 64
conv_steps=[2, 1, 1, 1, 1, 1] #same as stride
image_size = (128, 128)
output_size = 2
learningRate = 0.001
drop_prob = 0.4
weight_noise = 0.75
num_epochs = 150
num_batches = None
host_plot='http://*****:*****@ %s' % (graph_name, datetime.datetime.now(), socket.gethostname()),
channels=[['train_error_rate', 'valid_error_rate'],
['train_total_gradient_norm']], after_epoch=True, server_url=host_plot))
PLOT_AVAILABLE = True
except ImportError:
PLOT_AVAILABLE = False
extensions.append(Checkpoint(save_to, after_epoch=True, after_training=True, save_separately=['log']))
logger.info("Building the model")
model = Model(cost)
########### Loading images #####################
main_loop = MainLoop(
algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
def main(num_epochs,
feature_maps=None,
mlp_hiddens=None,
conv_sizes=None,
pool_sizes=None,
batch_size=500,
num_batches=None):
############# Architecture #############
if feature_maps is None:
feature_maps = [20, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5]
if pool_sizes is None:
pool_sizes = [2, 2]
image_size = (32, 32)
batch_size = 50
output_size = 2
learningRate = 0.1
num_epochs = 10
num_batches = None
delta = 0.01
drop_prob = 0.5
weight_noise = 0.75
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
3,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
x = tensor.tensor4('image_features')
y = tensor.lmatrix('targets')
probs = (convnet.apply(x)).copy(name='probs')
# Computational Graph just for cost for drop_out and noise application
cg_probs = ComputationGraph([probs])
inputs = VariableFilter(roles=[INPUT])(cg_probs.variables)
weights = VariableFilter(roles=[FILTER, WEIGHT])(cg_probs.variables)
############# Regularization #############
#regularization = 0
logger.info('Applying regularization')
regularization = delta * sum([(W**2).mean() for W in weights])
probs.name = "reg_probs"
############# Guaussian Noise #############
logger.info('Applying Gaussian noise')
cg_train = apply_noise(cg_probs, weights, weight_noise)
############# Dropout #############
logger.info('Applying dropout')
cg_probs = apply_dropout(cg_probs, inputs, drop_prob)
dropped_out = VariableFilter(roles=[DROPOUT])(cg_probs.variables)
inputs_referenced = [var.tag.replacement_of for var in dropped_out]
set(inputs) == set(inputs_referenced)
############# Batch normalization #############
# recalculate probs after dropout and noise and regularization:
probs = cg_probs.outputs[0] + regularization
cost = (CategoricalCrossEntropy().apply(y.flatten(),
probs).copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
cg = ComputationGraph([probs, cost, error_rate])
cg = apply_batch_normalization(cg)
########### Loading images #####################
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream, ServerDataStream
from fuel.schemes import ShuffledScheme
from fuel.transformers.image import RandomFixedSizeCrop, MinimumImageDimensions, Random2DRotation
from fuel.transformers import Flatten, Cast, ScaleAndShift
def create_data(data):
stream = DataStream(data,
iteration_scheme=ShuffledScheme(
data.num_examples, batch_size))
stream_downscale = MinimumImageDimensions(
stream, image_size, which_sources=('image_features', ))
stream_rotate = Random2DRotation(stream_downscale,
which_sources=('image_features', ))
stream_max = ScikitResize(stream_rotate,
image_size,
which_sources=('image_features', ))
stream_scale = ScaleAndShift(stream_max,
1. / 255,
0,
which_sources=('image_features', ))
stream_cast = Cast(stream_scale,
dtype='float32',
which_sources=('image_features', ))
#stream_flat = Flatten(stream_scale, which_sources=('image_features',))
return stream_cast
stream_data_train = create_data(
DogsVsCats(('train', ), subset=slice(0, 20)))
stream_data_test = create_data(
DogsVsCats(('train', ), subset=slice(20, 30)))
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=learningRate))
#algorithm = GradientDescent(cost=cost, parameters=cg.parameters,step_rule=Adam(0.001))
#algorithm.add_updates(extra_updates)
# `Timing` extension reports time for reading data, aggregating a batch and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = []
extensions.append(Timing())
extensions.append(
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches))
extensions.append(
DataStreamMonitoring([cost, error_rate],
stream_data_test,
prefix="valid"))
extensions.append(
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True))
#extensions.append(Checkpoint(save_to))
extensions.append(ProgressBar())
extensions.append(Printing())
logger.info("Building the model")
model = Model(cost)
main_loop = MainLoop(algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
def main(save_to,
num_epochs,
feature_maps=None,
mlp_hiddens=None,
conv_sizes=None,
pool_sizes=None,
batch_size=500,
num_batches=None):
if feature_maps is None:
feature_maps = [20, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5]
if pool_sizes is None:
pool_sizes = [2, 2]
image_size = (32, 23)
batch_size = 50
output_size = 2
learningRate = 0.1
num_epochs = 10
num_batches = None
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
3,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
x = tensor.tensor4('image_features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = (CategoricalCrossEntropy().apply(y.flatten(),
probs).copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
cg = ComputationGraph([cost, error_rate])
########### Loading images #####################
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream, ServerDataStream
from fuel.schemes import ShuffledScheme
from fuel.transformers.image import RandomFixedSizeCrop, MinimumImageDimensions, Random2DRotation
from fuel.transformers import Flatten, Cast, ScaleAndShift
def create_data(data):
stream = DataStream.default_stream(data,
iteration_scheme=ShuffledScheme(
data.num_examples, batch_size))
stream_downscale = MinimumImageDimensions(
stream, image_size, which_sources=('image_features', ))
#stream_rotate = Random2DRotation(stream_downscale, which_sources=('image_features',))
stream_max = ScikitResize(stream_downscale,
image_size,
which_sources=('image_features', ))
stream_scale = ScaleAndShift(stream_max,
1. / 255,
0,
which_sources=('image_features', ))
stream_cast = Cast(stream_scale,
dtype='float32',
which_sources=('image_features', ))
#stream_flat = Flatten(stream_scale, which_sources=('image_features',))
return stream_cast
stream_data_train = create_data(
DogsVsCats(('train', ), subset=slice(0, 20000)))
stream_data_test = create_data(
DogsVsCats(('train', ), subset=slice(20000, 25000)))
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=learningRate))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = []
extensions.append(Timing())
extensions.append(
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches))
extensions.append(
DataStreamMonitoring([cost, error_rate],
stream_data_test,
prefix="valid"))
extensions.append(
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True))
extensions.append(Checkpoint(save_to))
extensions.append(ProgressBar())
extensions.append(Printing())
model = Model(cost)
########### Loading images #####################
main_loop = MainLoop(algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()
from fuel.transformers.image import RandomFixedSizeCrop
from fuel.transformers import Flatten
from fuel.transformers.image import MinimumImageDimensions
import theano
from theano import tensor
import numpy as np
# Use Blocks to train this network
from blocks.algorithms import GradientDescent, Momentum
from blocks.extensions import Printing, FinishAfter
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.main_loop import MainLoop
# Load the training set
train = DogsVsCats(('train', ), subset=slice(0, 20000))
# We now create a "stream" over the dataset which will return shuffled batches
# of size 128. Using the DataStream.default_stream constructor will turn our
# 8-bit images into floating-point decimals in [0, 1].
stream = DataStream.default_stream(train,
iteration_scheme=ShuffledScheme(
train.num_examples, 128))
# Enlarge images that are too small
downnscale_stream = MinimumImageDimensions(stream, (64, 64),
which_sources=('image_features', ))
# Our images are of different sizes, so we'll use a Fuel transformer
# to take random crops of size (32 x 32) from each image
cropped_stream = RandomFixedSizeCrop(downnscale_stream, (32, 32),
#stream = ScikitResize(stream, image_size, which_sources=('image_features',))
# Data Preprocessing
# Data Transformation
stream = ScaleAndShift(stream,
1. / 255,
0,
which_sources=('image_features', ))
stream = Cast(stream, dtype='float32', which_sources=('image_features', ))
return stream
if mode is "CPU_test":
stream_data_train = create_data(
DogsVsCats(('train', ), subset=slice(0, 100)))
stream_data_valid = create_data(
DogsVsCats(('train', ), subset=slice(100, 110)))
if mode is "GPU_run":
stream_data_train = create_data(
DogsVsCats(('train', ), subset=slice(0, 22500)))
stream_data_valid = create_data(
DogsVsCats(('train', ), subset=slice(22500, 25000)))
if mode is "data_server":
stream_data_train = ServerDataStream(('image_features', 'targets'),
False,
port=5560)
stream_data_valid = ServerDataStream(('image_features', 'targets'),
False,
port=5561)
conv_sequence = ConvolutionalSequence(conv_layers,
num_channels,
image_size=image_shape,
use_bias=False)
#Add the Softmax function
out = Flattener().apply(conv_sequence.apply(x))
predict = NDimensionalSoftmax().apply(out)
#get the test stream
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream, ServerDataStream
from fuel.schemes import ShuffledScheme, SequentialExampleScheme
from fuel.transformers.image import RandomFixedSizeCrop, MinimumImageDimensions, MaximumImageDimensions, Random2DRotation
from fuel.transformers import Flatten, Cast, ScaleAndShift
size = (128, 128)
cats = DogsVsCats(('test', ))
stream = DataStream.default_stream(cats,
iteration_scheme=SequentialExampleScheme(
cats.num_examples))
stream_upscale = MaximumImageDimensions(stream,
size,
which_sources=('image_features', ))
stream_scale = ScaleAndShift(stream_upscale,
1. / 255,
0,
which_sources=('image_features', ))
stream_data = Cast(stream_scale,
dtype='float32',
which_sources=('image_features', ))
#Load the parameters of the model
from transformers import RandomHorizontalFlip, DownscaleMinDimension
if socket.gethostname() == 'yop':
sub = slice(0, 1500)
batch_size = 10
else:
sub = slice(0, 15000)
batch_size = 25
if len(sys.argv) > 1:
port = int(sys.argv[1])
else:
port = 5557
# Load the training set
train = DogsVsCats(('train', ), subset=sub)
# We now create a "stream" over the dataset which will return shuffled batches
# of size 128. Using the DataStream.default_stream constructor will turn our
# 8-bit images into floating-point decimals in [0, 1].
stream = DataStream.default_stream(train,
iteration_scheme=SequentialScheme(
train.num_examples, batch_size))
# upscaled_stream = MinimumImageDimensions(stream, (100, 100), which_sources=('image_features',))
downscaled_stream = DownscaleMinDimension(stream,
100,
which_sources=('image_features', ))
# Our images are of different sizes, so we'll use a Fuel transformer
# to take random crops of size (32 x 32) from each image
def main(save_to,
num_epochs,
feature_maps=None,
mlp_hiddens=None,
conv_sizes=None,
pool_sizes=None,
batch_size=200,
num_batches=None):
if feature_maps is None:
feature_maps = [32, 32, 64, 64, 128, 128]
if mlp_hiddens is None:
mlp_hiddens = [1000]
if conv_sizes is None:
conv_sizes = [7, 5, 5, 5, 3, 3]
if pool_sizes is None:
pool_sizes = [2, 2, 2, 2, 2, 2]
image_size = (128, 128)
batch_size = 64
output_size = 2
learningRate = 0.01
drop_prob = 0.4
weight_noise = 0.75
num_epochs = 150
num_batches = None
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
3,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
x = tensor.tensor4('image_features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
# Save on csv
# numpy.save(probs)
cost = (CategoricalCrossEntropy().apply(y.flatten(),
probs).copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
error_rate2 = error_rate.copy(name='error_rate2')
cg = ComputationGraph([cost, error_rate])
weights = VariableFilter(roles=[FILTER, WEIGHT])(cg.variables)
############# Dropout #############
logger.info('Applying dropout')
cg = apply_dropout(cg, weights[0:3], drop_prob)
dropped_out = VariableFilter(roles=[DROPOUT])(cg.variables)
############# Guaussian Noise #############
logger.info('Applying Gaussian noise')
cg = apply_noise(cg, weights, weight_noise)
########### Loading images #####################
from fuel.datasets.dogs_vs_cats import DogsVsCats
from fuel.streams import DataStream, ServerDataStream
from fuel.schemes import ShuffledScheme
from fuel.transformers.image import RandomFixedSizeCrop, MinimumImageDimensions, Random2DRotation
from fuel.transformers import Flatten, Cast, ScaleAndShift
def create_data(data):
stream = DataStream(data,
iteration_scheme=ShuffledScheme(
data.num_examples, batch_size))
stream = MinimumImageDimensions(stream,
image_size,
which_sources=('image_features', ))
stream = MaximumImageDimensions(stream,
image_size,
which_sources=('image_features', ))
stream = RandomHorizontalSwap(stream,
which_sources=('image_features', ))
stream = Random2DRotation(stream, which_sources=('image_features', ))
#stream = ScikitResize(stream, image_size, which_sources=('image_features',))
stream = ScaleAndShift(stream,
1. / 255,
0,
which_sources=('image_features', ))
stream = Cast(stream,
dtype='float32',
which_sources=('image_features', ))
return stream
stream_data_train = create_data(
DogsVsCats(('train', ), subset=slice(0, 22500)))
stream_data_test = create_data(
DogsVsCats(('train', ), subset=slice(22500, 25000)))
#stream_data_train = create_data(DogsVsCats(('train',), subset=slice(0, 10)))
#stream_data_test = create_data(DogsVsCats(('train',), subset=slice(10, 12)))
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Momentum(learning_rate=learningRate,
momentum=0.7))
#algorithm = GradientDescent(cost=cost, parameters=cg.parameters,step_rule=Scale(learning_rate=learningRate))
#algorithm = GradientDescent(cost=cost, parameters=cg.parameters,step_rule=Adam(0.001))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = []
extensions.append(Timing())
extensions.append(
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches))
extensions.append(
DataStreamMonitoring([cost, error_rate],
stream_data_test,
prefix="valid"))
extensions.append(
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True))
extensions.append(
Checkpoint("Model1_uniform_init.pkl",
after_epoch=True,
after_training=True,
save_separately=['log']))
extensions.append(ProgressBar())
extensions.append(Printing())
host_plot = 'http://hades.calculquebec.ca:5090'
extensions.append(
Plot('5C 3*3C 2*2P 204080...F 004LR 09Mom %s %s @ %s' %
('CNN ', datetime.datetime.now(), socket.gethostname()),
channels=[['train_error_rate', 'valid_error_rate'],
['train_total_gradient_norm']],
after_epoch=True,
server_url=host_plot))
logger.info("Building the model")
model = Model(cost)
########### Loading images #####################
main_loop = MainLoop(algorithm,
stream_data_train,
model=model,
extensions=extensions)
main_loop.run()