def test_force_floatx():
x = [numpy.array(d, dtype="float64") for d in [[1, 2], [3, 4], [5, 6]]]
y = [numpy.array(d, dtype="int64") for d in [1, 2, 3]]
dataset = IterableDataset(OrderedDict([("x", x), ("y", y)]))
wrapper = ForceFloatX(DataStream(dataset))
data = next(wrapper.get_epoch_iterator())
assert str(data[0].dtype) == config.floatX
assert str(data[1].dtype) == "int64"
def test_force_floatx():
x = [numpy.array(d, dtype="float64") for d in [[1, 2], [3, 4], [5, 6]]]
y = [numpy.array(d, dtype="int64") for d in [1, 2, 3]]
dataset = IterableDataset(OrderedDict([("x", x), ("y", y)]))
wrapper = ForceFloatX(DataStream(dataset))
data = next(wrapper.get_epoch_iterator())
assert str(data[0].dtype) == config.floatX
assert str(data[1].dtype) == "int64"
def open_stream(which_sets=('train', ), port=5557, num_examples=None):
dataset = Blizzard(which_sets=which_sets)
if num_examples == None:
num_examples = dataset.num_examples
data_stream = DataStream.default_stream(dataset,
iteration_scheme=SequentialScheme(
num_examples, batch_size))
data_stream = ScaleAndShift(data_stream,
scale=1 / data_std,
shift=-data_mean / data_std)
data_stream = Mapping(data_stream,
_downsample_and_upsample,
add_sources=('upsampled', ))
data_stream = Mapping(data_stream, _equalize_size)
data_stream = Mapping(data_stream,
_get_residual,
add_sources=('residual', ))
data_stream = FilterSources(data_stream,
sources=(
'upsampled',
'residual',
))
data_stream = Mapping(data_stream, _segment_axis)
data_stream = Mapping(data_stream, _transpose)
data_stream = ForceFloatX(data_stream)
start_server(data_stream, port=port)
def construct_stream(dataset, rng, pool_size, maximum_frames, window_features,
**kwargs):
"""Construct data stream.
Parameters:
-----------
dataset : Dataset
Dataset to use.
rng : numpy.random.RandomState
Random number generator.
pool_size : int
Pool size for TIMIT dataset.
maximum_frames : int
Maximum frames for TIMIT datset.
subsample : bool, optional
Subsample features.
pretrain_alignment : bool, optional
Use phoneme alignment for pretraining.
uniform_alignment : bool, optional
Use uniform alignment for pretraining.
"""
kwargs.setdefault('subsample', False)
kwargs.setdefault('pretrain_alignment', False)
kwargs.setdefault('uniform_alignment', False)
stream = DataStream(dataset,
iteration_scheme=SequentialShuffledScheme(
dataset.num_examples, pool_size, rng))
if kwargs['pretrain_alignment'] and kwargs['uniform_alignment']:
stream = AddUniformAlignmentMask(stream)
stream = Reshape('features', 'features_shapes', data_stream=stream)
means, stds = dataset.get_normalization_factors()
stream = Normalize(stream, means, stds)
if not window_features == 1:
stream = WindowFeatures(stream, 'features', window_features)
if kwargs['pretrain_alignment']:
stream = Reshape('alignments', 'alignments_shapes', data_stream=stream)
stream = Mapping(stream, SortMapping(key=key))
stream = MaximumFrameCache(max_frames=maximum_frames,
data_stream=stream,
rng=rng)
stream = Padding(data_stream=stream, mask_sources=['features', 'phonemes'])
if kwargs['pretrain_alignment']:
stream = AlignmentPadding(stream, 'alignments')
stream = Transpose(stream, [(1, 0, 2), (1, 0), (1, 0), (1, 0),
(2, 1, 0)])
else:
stream = Transpose(stream, [(1, 0, 2), (1, 0), (1, 0), (1, 0)])
stream = ForceFloatX(stream)
if kwargs['subsample']:
stream = Subsample(stream, 'features', 5)
stream = Subsample(stream, 'features_mask', 5)
return stream
def get_stream(self, part, batches=True, shuffle=True, add_sources=(),
num_examples=None, rng=None, seed=None):
dataset = self.get_dataset(part, add_sources=add_sources)
if num_examples is None:
num_examples = dataset.num_examples
if shuffle:
iteration_scheme = ShuffledExampleScheme(num_examples, rng=rng)
else:
iteration_scheme = SequentialExampleScheme(num_examples)
stream = DataStream(
dataset, iteration_scheme=iteration_scheme)
stream = FilterSources(stream, (self.recordings_source,
self.labels_source)+tuple(add_sources))
if self.add_eos:
stream = Mapping(stream, _AddLabel(self.eos_label))
if self.add_bos:
stream = Mapping(stream, _AddLabel(self.bos_label, append=False,
times=self.add_bos))
if self.preprocess_text:
stream = Mapping(stream, lvsr.datasets.wsj.preprocess_text)
stream = Filter(stream, self.length_filter)
if self.sort_k_batches and batches:
stream = Batch(stream,
iteration_scheme=ConstantScheme(
self.batch_size * self.sort_k_batches))
stream = Mapping(stream, SortMapping(_length))
stream = Unpack(stream)
if self.preprocess_features == 'log_spectrogram':
stream = Mapping(
stream, functools.partial(apply_preprocessing,
log_spectrogram))
if self.normalization:
stream = self.normalization.wrap_stream(stream)
stream = ForceFloatX(stream)
if not batches:
return stream
stream = Batch(
stream,
iteration_scheme=ConstantScheme(self.batch_size if part == 'train'
else self.validation_batch_size))
stream = Padding(stream)
stream = Mapping(stream, switch_first_two_axes)
stream = ForceCContiguous(stream)
return stream
def timit_datastream(path, which_set, local_copy, pool_size, maximum_frames):
# load dataset
timit_dataset = Timit(which_set=which_set,
path=path,
local_copy=local_copy)
# get statistics
data_means, data_stds = timit_dataset.get_normalization_factors()
# set shuffle range
shuffle_rng = numpy.random.RandomState(123)
# set iterator scheme
iterator_scheme = SequentialShuffledScheme(
num_examples=timit_dataset.num_examples,
batch_size=pool_size,
rng=shuffle_rng)
# base data stream
base_stream = DataStream(dataset=timit_dataset,
iteration_scheme=iterator_scheme)
# reshape stream
reshape_stream = Reshape(data_source='features',
shape_source='features_shapes',
data_stream=base_stream)
# normalize data stream
normalize_stream = Normalize(data_stream=reshape_stream,
means=data_means,
stds=data_stds)
# sort data stream
sort_stream = Mapping(data_stream=normalize_stream,
mapping=SortMapping(key=lambda x: x[0].shape[0]))
# max frame stream
max_frame_stream = MaximumFrameCache(max_frames=maximum_frames,
data_stream=sort_stream,
rng=shuffle_rng)
# padding data stream
padded_stream = Padding(data_stream=max_frame_stream,
mask_sources=['features', 'phonemes'])
# floatX stream
data_stream = ForceFloatX(padded_stream)
return timit_dataset, data_stream
def get_ucf_streams(batch_size):
train_dataset = JpegHDF5Dataset("train")
valid_dataset = JpegHDF5Dataset("valid")
test_dataset = JpegHDF5Dataset("test")
train_datastream = ForceFloatX(
DataStream(dataset=train_dataset,
iteration_scheme=HDF5ShuffledScheme(
train_dataset.video_indexes,
examples=train_dataset.num_video_examples,
batch_size=batch_size,
frames_per_video=20)))
valid_datastream = ForceFloatX(
DataStream(dataset=valid_dataset,
iteration_scheme=HDF5ShuffledScheme(
valid_dataset.video_indexes,
examples=valid_dataset.num_video_examples,
batch_size=batch_size,
frames_per_video=20)))
test_datastream = ForceFloatX(
DataStream(dataset=test_dataset,
iteration_scheme=HDF5ShuffledScheme(
test_dataset.video_indexes,
examples=test_dataset.num_video_examples,
batch_size=batch_size,
frames_per_video=20)))
train_datastream = JpegHDF5Transformer(data_stream=train_datastream)
valid_datastream = JpegHDF5Transformer(data_stream=valid_datastream)
test_datastream = JpegHDF5Transformer(data_stream=test_datastream)
train_datastream.sources = ('features', 'targets')
valid_datastream.sources = ('features', 'targets')
test_datastream.sources = ('features', 'targets')
return train_datastream, valid_datastream
def stream_handwriting(which_sets, batch_size, seq_size, tbptt=True):
dataset = Handwriting(which_sets)
data_stream = DataStream.default_stream(
dataset,
iteration_scheme=ShuffledScheme(
batch_size * (dataset.num_examples / batch_size), batch_size))
data_stream = FilterSources(data_stream, sources=('features', ))
data_stream = Padding(data_stream)
data_stream = Mapping(data_stream, _transpose)
if tbptt:
data_stream = SegmentSequence(data_stream,
add_flag=True,
seq_size=seq_size)
data_stream = ForceFloatX(data_stream)
return data_stream
def get_stream(self, part, batches=True, shuffle=True,
add_sources=()):
dataset = self.get_dataset(part, add_sources=add_sources)
stream = (DataStream(dataset,
iteration_scheme=ShuffledExampleScheme(dataset.num_examples))
if shuffle
else dataset.get_example_stream())
stream = FilterSources(stream, (self.recordings_source,
self.labels_source)+tuple(add_sources))
if self.add_eos:
if self.prepend_eos:
stream = Mapping(stream, _AddEosLabelBeginEnd(self.eos_label))
else:
stream = Mapping(stream, _AddEosLabelEnd(self.eos_label))
if self.preprocess_text:
stream = Mapping(stream, lvsr.datasets.wsj.preprocess_text)
stream = Filter(stream, self.length_filter)
if self.sort_k_batches and batches:
stream = Batch(stream,
iteration_scheme=ConstantScheme(
self.batch_size * self.sort_k_batches))
stream = Mapping(stream, SortMapping(_length))
stream = Unpack(stream)
if self.preprocess_features == 'log_spectrogram':
stream = Mapping(
stream, functools.partial(apply_preprocessing,
log_spectrogram))
if self.normalization:
stream = self.normalization.wrap_stream(stream)
stream = ForceFloatX(stream)
if not batches:
return stream
stream = Batch(stream, iteration_scheme=ConstantScheme(self.batch_size))
stream = Padding(stream)
stream = Mapping(stream, switch_first_two_axes)
stream = ForceCContiguous(stream)
return stream
def monk_music_stream(which_sets=('train', ),
batch_size=64,
seq_size=128,
frame_size=160,
num_examples=None,
which_sources=('features', )):
"""
This function generates the stream for the monk_music dataset.
It doesn't compute incremental windows and instead simply separates the
dataset into sequences
"""
dataset = MonkMusic(which_sets=which_sets,
filename="dataset.hdf5",
load_in_memory=True)
large_batch_size = batch_size * frame_size * seq_size
if not num_examples:
num_examples = large_batch_size * (dataset.num_examples /
large_batch_size)
# If there are memory problems revert to SequentialScheme
data_stream = DataStream.default_stream(dataset,
iteration_scheme=SequentialScheme(
num_examples,
large_batch_size))
data_stream = ScaleAndShift(data_stream,
scale=1. / data_stats["std"],
shift=-data_stats["mean"] / data_stats["std"])
data_stream = Mapping(
data_stream,
lambda data: _get_subsequences(data, batch_size, seq_size, frame_size))
data_stream = ForceFloatX(data_stream)
return data_stream
def get_stream(batch_size,
source_window=4000,
target_window=1000,
num_examples=5000):
from fuel.datasets.youtube_audio import YouTubeAudio
data = YouTubeAudio('XqaJ2Ol5cC4')
train_stream = data.get_example_stream()
train_stream = ForceFloatX(train_stream)
window_stream = Window(0,
source_window,
target_window,
overlapping=False,
data_stream=train_stream)
source_stream = FilterSources(window_stream, sources=('features', ))
feats_stream = Mapping(source_stream, mfcc)
targets_stream = FilterSources(window_stream, sources=('targets', ))
targets_stream = Flatten(targets_stream)
stream = Merge((feats_stream, targets_stream),
sources=('features', 'targets'))
#Add a random Scheme?
it_scheme = ConstantScheme(batch_size, num_examples)
batched_stream = Batch(stream, it_scheme, strictness=1)
return batched_stream
def get_ucf_streams(batch_size):
train_dataset = JpegHDF5Dataset("train")
valid_dataset = JpegHDF5Dataset("valid")
test_dataset = JpegHDF5Dataset("test")
train_datastream = ForceFloatX(DataStream(
dataset=train_dataset,
iteration_scheme=HDF5ShuffledScheme(
train_dataset.video_indexes,
examples=train_dataset.num_video_examples,
batch_size=batch_size,
frames_per_video=20)))
valid_datastream = ForceFloatX(DataStream(
dataset=valid_dataset,
iteration_scheme=HDF5ShuffledScheme(
valid_dataset.video_indexes,
examples=valid_dataset.num_video_examples,
batch_size=batch_size,
frames_per_video=20)))
test_datastream = ForceFloatX(DataStream(
dataset=test_dataset,
iteration_scheme=HDF5ShuffledScheme(
test_dataset.video_indexes,
examples=test_dataset.num_video_examples,
batch_size=batch_size,
frames_per_video=20)))
train_datastream = JpegHDF5Transformer(data_stream=train_datastream)
valid_datastream = JpegHDF5Transformer(data_stream=valid_datastream)
test_datastream = JpegHDF5Transformer(data_stream=test_datastream)
train_datastream.sources = ('features', 'targets')
valid_datastream.sources = ('features', 'targets')
test_datastream.sources = ('features', 'targets')
return train_datastream, valid_datastream
# input_mat = np.random.rand(batch_size, input_dim0, input_dim1).astype(theano.config.floatX)
# input_ratings = np.zeros((batch_size, input_dim0, input_dim1), dtype=theano.config.floatX)
# input_ratings[:, :, 0] = 1
trainset = MovieLens1M(which_set=['train'],
sources=('input_ratings', 'output_ratings',
'input_masks', 'output_masks'))
validset = MovieLens1M(which_set=['valid'],
sources=('input_ratings', 'output_ratings',
'input_masks', 'output_masks'))
testset = MovieLens1M(which_set=['test'],
sources=('input_ratings', 'output_ratings',
'input_masks', 'output_masks'))
train_loop_stream = ForceFloatX(data_stream=MovieLensTransformer(
data_stream=Trainer_MovieLensTransformer(
data_stream=DataStream(dataset=trainset,
iteration_scheme=ShuffledScheme(
trainset.num_examples, batch_size)))))
trainval_loop_stream = ForceFloatX(data_stream=MovieLensTransformer(
data_stream=Trainer_MovieLensTransformer(
data_stream=DataStream(dataset=testset,
iteration_scheme=ShuffledScheme(
trainset.num_examples, batch_size)))))
# train_monitor_stream = ForceFloatX(
# data_stream=MovieLensTransformer(
# data_stream=DataStream(
# dataset=trainset,
# iteration_scheme=ShuffledScheme(
# validset.num_examples,
def test_force_floatx_axis_labels():
x = numpy.eye(2).astype('float64')
axis_labels = {'x': ('batch', 'feature')}
dataset = IterableDataset({'x': x}, axis_labels=axis_labels)
stream = ForceFloatX(DataStream(dataset))
assert stream.axis_labels == axis_labels
def main(num_epochs=1000):
x = tensor.matrix('features')
y = tensor.lmatrix('targets')
softmax_regressor = SoftmaxRegressor(input_dim=784, n_classes=10)
probs = softmax_regressor.get_probs(features=x)
params = softmax_regressor.get_params()
weights = softmax_regressor.get_weights()
cost = softmax_regressor.get_cost(probs=probs, targets=y).mean()
cost.name = 'cost'
misclassification = softmax_regressor.get_misclassification(
probs=probs, targets=y).mean()
misclassification.name = 'misclassification'
train_dataset = MNIST('train')
test_dataset = MNIST('test')
algorithm = GradientDescent(cost=cost,
params=params,
step_rule=Momentum(learning_rate=0.1,
momentum=0.1))
train_data_stream = ForceFloatX(
data_stream=DataStream(dataset=train_dataset,
iteration_scheme=ShuffledScheme(
examples=train_dataset.num_examples,
batch_size=100,
)))
test_data_stream = ForceFloatX(
data_stream=DataStream(dataset=test_dataset,
iteration_scheme=SequentialScheme(
examples=test_dataset.num_examples,
batch_size=1000,
)))
model = Model(cost)
extensions = []
extensions.append(Timing())
extensions.append(FinishAfter(after_n_epochs=num_epochs))
extensions.append(
DataStreamMonitoring([cost, misclassification],
test_data_stream,
prefix='test'))
extensions.append(
TrainingDataMonitoring([cost, misclassification],
prefix='train',
after_epoch=True))
plotters = []
plotters.append(
Plotter(channels=[[
'test_cost', 'test_misclassification', 'train_cost',
'train_misclassification'
]],
titles=['Costs']))
display_train = ImageDataStreamDisplay(
data_stream=copy.deepcopy(train_data_stream),
image_shape=(28, 28, 1),
axes=(0, 1, 'c'),
shift=-0.5,
rescale=2.,
)
weight_display = WeightDisplay(weights=weights,
transpose=(1, 0),
image_shape=(28, 28, 1),
axes=(0, 1, 'c'),
shift=-0.5,
rescale=2.,
grid_shape=(1, 10))
images_displayer = DisplayImage(
image_getters=[display_train, weight_display],
titles=['Training examples', 'Softmax weights'])
plotters.append(images_displayer)
extensions.append(
PlotManager('MNIST softmax examples',
plotters=plotters,
after_epoch=False,
every_n_epochs=10,
after_training=True))
extensions.append(Printing())
main_loop = MainLoop(model=model,
data_stream=train_data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def main(num_epochs=100):
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
num_filters = 64
layers = [ConvolutionalLayer(filter_size=(8, 8), num_filters=num_filters,
num_channels=3, step=(1, 1),
border_mode='valid'),
MaxPooling(pooling_size=(2, 2)),
ConvolutionalLayer(filter_size=(5, 5), num_filters=num_filters,
num_channels=num_filters, step=(1, 1),
border_mode='valid')]
convnet = ConvolutionalNetwork(layers=layers)
output_shape = convnet.get_output_shape((32, 32))
convnet.set_input_shape((32, 32))
fc_net = NeuralSoftmax(input_dim=numpy.prod(output_shape) * num_filters,
n_classes=10, n_hidden=[500])
convnet_features = convnet.apply(x)
probs = fc_net.get_probs(features=convnet_features.flatten(2))
params = convnet.get_params() + fc_net.get_params()
weights = convnet.get_weights()
cost = fc_net.get_cost(probs=probs, targets=y).mean()
cost.name = 'cost'
misclassification = fc_net.get_misclassification(
probs=probs, targets=y
).mean()
misclassification.name = 'misclassification'
train_dataset = CIFAR10('train', flatten=False)
test_dataset = CIFAR10('test', flatten=False)
algorithm = GradientDescent(
cost=cost,
params=params,
step_rule=Momentum(learning_rate=.01,
momentum=0.1))
train_data_stream = Mapping(
data_stream=ForceFloatX(
data_stream=DataStream(
dataset=train_dataset,
iteration_scheme=ShuffledScheme(
examples=range(50000),
batch_size=100,
)
)
),
mapping=Rescale(scale=1/127.5, shift=-127.5)
)
valid_data_stream = Mapping(
data_stream=ForceFloatX(
data_stream=DataStream(
dataset=train_dataset,
iteration_scheme=SequentialScheme(
examples=range(40000, 50000),
batch_size=1000,
)
)
),
mapping=Rescale(scale=1/127.5, shift=-127.5)
)
test_data_stream = Mapping(
data_stream=ForceFloatX(
data_stream=DataStream(
dataset=test_dataset,
iteration_scheme=SequentialScheme(
examples=test_dataset.num_examples,
batch_size=100,
)
)
),
mapping=Rescale(scale=1/127.5, shift=-127.5)
)
model = Model(cost)
extensions = []
extensions.append(Timing())
extensions.append(FinishAfter(after_n_epochs=num_epochs))
extensions.append(DataStreamMonitoring(
[cost, misclassification],
test_data_stream,
prefix='test'))
extensions.append(DataStreamMonitoring(
[cost, misclassification],
valid_data_stream,
prefix='valid'))
extensions.append(TrainingDataMonitoring(
[cost, misclassification],
prefix='train',
after_epoch=True))
plotters = []
plotters.append(Plotter(
channels=[['test_cost', 'test_misclassification',
'train_cost', 'train_misclassification']],
titles=['Costs']))
display_train = ImageDataStreamDisplay(
data_stream=copy.deepcopy(train_data_stream),
image_shape=(3, 32, 32),
axes=('c', 0, 1),
shift=0,
rescale=1.,
)
weight_display = WeightDisplay(
weights=weights,
transpose=(0, 1, 2, 3),
image_shape=(3, 8, 8),
axes=('c', 0, 1),
shift=-0.5,
rescale=2.,
)
# Feature maps
one_example_train_data_stream = Mapping(
data_stream=ForceFloatX(
data_stream=DataStream(
dataset=train_dataset,
iteration_scheme=ShuffledScheme(
examples=train_dataset.num_examples,
batch_size=1,
)
)
),
mapping=Rescale(scale=1/127.5, shift=-127.5)
)
displayable_convnet_features = convnet_features.dimshuffle((1, 0, 2, 3))
convnet_features_normalizer = abs(
displayable_convnet_features
).max(axis=(1, 2, 3))
displayable_convnet_features = displayable_convnet_features \
/ convnet_features_normalizer.dimshuffle((0, 'x', 'x', 'x'))
get_displayable_convnet_features = theano.function(
[x], displayable_convnet_features
)
display_feature_maps_data_stream = Mapping(
data_stream=one_example_train_data_stream,
mapping=TupleMapping(get_displayable_convnet_features,
same_len_out=True)
)
display_feature_maps = ImageDataStreamDisplay(
data_stream=display_feature_maps_data_stream,
image_shape=(1, ) + output_shape,
axes=('c', 0, 1),
shift=0,
rescale=1.,
)
# Saliency map
displayable_saliency_map = tensor.grad(cost, x)
saliency_map_normalizer = abs(
displayable_saliency_map
).max(axis=(1, 2, 3))
displayable_saliency_map = displayable_saliency_map \
/ saliency_map_normalizer.dimshuffle((0, 'x', 'x', 'x'))
get_displayable_saliency_map = theano.function(
[x, y], displayable_saliency_map
)
display_saliency_map_data_stream = Mapping(
data_stream=copy.deepcopy(train_data_stream),
mapping=TupleMapping(get_displayable_saliency_map,
same_len_out=True,
same_len_in=True)
)
display_saliency_map = ImageDataStreamDisplay(
data_stream=display_saliency_map_data_stream,
image_shape=(3, 32, 32),
axes=('c', 0, 1),
shift=0,
rescale=1.,
)
# Deconvolution
x_repeated = x.repeat(num_filters, axis=0)
convnet_features_repeated = convnet.apply(x_repeated)
convnet_features_selected = convnet_features_repeated \
* tensor.eye(num_filters).repeat(
x.shape[0], axis=0
).dimshuffle((0, 1, 'x', 'x'))
displayable_deconvolution = tensor.grad(misclassification, x_repeated,
known_grads={
convnet_features_selected:
convnet_features_selected
})
deconvolution_normalizer = abs(
displayable_deconvolution
).max(axis=(1, 2, 3))
displayable_deconvolution = displayable_deconvolution \
/ deconvolution_normalizer.dimshuffle((0, 'x', 'x', 'x'))
get_displayable_deconvolution = theano.function(
[x], displayable_deconvolution
)
display_deconvolution_data_stream = Mapping(
data_stream=one_example_train_data_stream,
mapping=TupleMapping(get_displayable_deconvolution,
same_len_out=True)
)
display_deconvolution = ImageDataStreamDisplay(
data_stream=display_deconvolution_data_stream,
image_shape=(3, 32, 32),
axes=('c', 0, 1),
shift=0,
rescale=1.,
)
images_displayer = DisplayImage(
image_getters=[display_train, weight_display, display_feature_maps,
display_saliency_map, display_deconvolution],
titles=['Training examples', 'Convolutional weights', 'Feature maps',
'Sensitivity', 'Deconvolution']
)
plotters.append(images_displayer)
extensions.append(PlotManager('CIFAR-10 convnet examples',
plotters=plotters,
after_epoch=False,
every_n_epochs=10,
after_training=True))
extensions.append(Printing())
main_loop = MainLoop(model=model,
data_stream=train_data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def test_force_floatx(self):
transformer = ForceFloatX(DataStream(self.dataset))
data = next(transformer.get_epoch_iterator())
assert_equal(str(data[0].dtype), config.floatX)
assert_equal(str(data[1].dtype), 'int64')
def get_one_stream(self, part, lang=None, batches=True, shuffle=True, add_sources=(),
num_examples=None, rng=None, seed=None, num_result=None,
soften_distributions=None, only_stream=False):
assert lang in self.langs
dataset = self.get_dataset(part, lang, add_sources=add_sources)
if num_examples is None:
num_examples = dataset.num_examples
if shuffle:
iteration_scheme = ShuffledExampleScheme(num_examples, rng=rng)
else:
iteration_scheme = SequentialExampleScheme(num_examples)
if num_result is None:
num_result = num_examples
if lang != self.langs[0] and not only_stream:
iteration_scheme = RandomExampleScheme(num_examples, num_result=num_result, rng=rng)
stream = DataStream(
dataset, iteration_scheme=iteration_scheme)
if soften_distributions:
stream = Mapping(stream, SoftenResult(self.default_sources, soften_distributions))
for bconv in self._binary_convertable_data:
if bconv in self.default_sources:
stream = Mapping(stream, ConvertToMask(self.default_sources,
bconv,
self.num_features(bconv)))
if self.add_eos:
stream = Mapping(stream, _AddLabel(
self.eos_label,
index=stream.sources.index(self.sources_map['labels'])))
if self.add_bos:
if self.bos_label is None:
raise Exception('No bos label given')
stream = Mapping(stream, _AddLabel(
self.bos_label, append=False, times=self.add_bos,
index=stream.sources.index(self.sources_map['labels'])))
if self.max_length:
stream = Filter(stream, self.length_filter)
if self.sort_k_batches and batches:
stream = Batch(stream,
iteration_scheme=ConstantScheme(
self.batch_size * self.sort_k_batches))
#
# Hardcode 0 for source on which to sort. This will be good, as
# most source lengths are correlated and, furthermore, the
# labels will typically be the last source, thus in a single-input
# case this sorts on input lengths
#
stream = Mapping(stream, SortMapping(_Length(
index=0)))
stream = Unpack(stream)
if self.normalization:
stream = self.normalization.wrap_stream(stream)
stream = ForceFloatX(stream)
stream = Rename(stream,
names=dict_subset({v: k for (k, v)
in self.sources_map.items()},
stream.sources,
must_have=False))
if not batches:
return stream, num_examples
stream = Batch(
stream,
iteration_scheme=ConstantScheme(self.batch_size if part == 'train'
else self.validation_batch_size))
stream._produces_examples = False
return stream, num_examples
def preprocessing(data_stream):
return ForceFloatX(ScaleAndShift(data_stream,
1 / 255.0,
0.0,
which_sources=('features', )),
which_sources=('features', ))
def test_force_floatx(self):
transformer = ForceFloatX(DataStream(self.dataset))
data = next(transformer.get_epoch_iterator())
assert_equal(str(data[0].dtype), config.floatX)
assert_equal(str(data[1].dtype), 'int64')
def main(dataset_name='sklearn', num_epochs=100):
x = tensor.matrix('features')
y = tensor.lmatrix('targets')
logistic_regressor = LogisticRegressor(input_dim=2)
probs = logistic_regressor.get_probs(features=x)
params = logistic_regressor.get_params()
cost = logistic_regressor.get_cost(probs=probs, targets=y).mean()
cost.name = 'cost'
misclassification = logistic_regressor.get_misclassification(
probs=probs, targets=y).mean()
misclassification.name = 'misclassification'
train_dataset, test_dataset = build_2d_datasets(dataset_name=dataset_name)
algorithm = GradientDescent(cost=cost,
params=params,
step_rule=Scale(learning_rate=0.1))
train_data_stream = ForceFloatX(
data_stream=DataStream(dataset=train_dataset,
iteration_scheme=SequentialScheme(
examples=train_dataset.num_examples,
batch_size=40,
)))
test_data_stream = ForceFloatX(
data_stream=DataStream(dataset=test_dataset,
iteration_scheme=SequentialScheme(
examples=test_dataset.num_examples,
batch_size=400,
)))
model = Model(cost)
extensions = []
extensions.append(Timing())
extensions.append(FinishAfter(after_n_epochs=num_epochs))
extensions.append(
DataStreamMonitoring([cost, misclassification],
test_data_stream,
prefix='test'))
extensions.append(
TrainingDataMonitoring([cost, misclassification],
prefix='train',
after_epoch=True))
scoring_function = theano.function([x], probs)
plotters = []
plotters.append(
Plotter(channels=[[
'test_cost', 'test_misclassification', 'train_cost',
'train_misclassification'
]],
titles=['Costs']))
score_train_stream = Mapping(data_stream=copy.deepcopy(train_data_stream),
mapping=TupleMapping(scoring_function),
add_sources=('scores', ))
score_test_stream = Mapping(data_stream=copy.deepcopy(test_data_stream),
mapping=TupleMapping(scoring_function),
add_sources=('scores', ))
plotters.append(
Display2DData(data_streams=[
score_train_stream,
copy.deepcopy(train_data_stream), score_test_stream,
copy.deepcopy(test_data_stream)
],
radius=0.01))
extensions.append(
PlotManager('2D examples',
plotters=plotters,
after_epoch=False,
every_n_epochs=50,
after_training=True))
extensions.append(Printing())
main_loop = MainLoop(model=model,
data_stream=train_data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def get_stream(self,
part,
batches=True,
shuffle=True,
add_sources=(),
num_examples=None,
rng=None,
seed=None):
dataset = self.get_dataset(part, add_sources=add_sources)
iteration_scheme = None
if self.use_iteration_scheme:
if num_examples is None:
num_examples = dataset.num_examples
if shuffle:
iteration_scheme = ShuffledExampleScheme(num_examples, rng=rng)
else:
iteration_scheme = SequentialExampleScheme(num_examples)
stream = DataStream(dataset, iteration_scheme=iteration_scheme)
# Transformations before rearrangement
labels_source = self.sources_map['labels']
if self.add_eos:
stream = _AddLabel(stream,
self.eos_label,
which_sources=[labels_source])
if self.add_bos:
if self.bos_label is None:
raise Exception('No bos label given')
stream = _AddLabel(stream,
self.bos_label,
append=False,
times=self.add_bos,
which_sources=[labels_source])
if self.clip_length:
stream = _Clip(stream,
self.clip_length,
force_eos=self.eos_label
if self.force_eos_when_clipping else None,
which_sources=[labels_source])
# More efficient packing of examples in batches
if self.sort_k_batches and batches:
stream = Batch(stream,
iteration_scheme=ConstantScheme(
self.batch_size * self.sort_k_batches))
stream = Mapping(stream, SortMapping(_Length(index=0)))
stream = Unpack(stream)
stream = Rearrange(
stream,
dict_subset(self.sources_map,
self.default_sources + list(add_sources)))
# Tranformations after rearrangement
if self.corrupt_sources:
# Can only corrupt sources with the same alphabet
# as labels
for source, prob in zip(self.corrupt_sources['names'],
self.corrupt_sources['probs']):
stream = _Corrupt(stream,
prob,
self.token_map(source),
self.eos_label,
which_sources=[source])
if self.max_length and part == 'train':
# Filtering by the maximum length is only done
# for the training set.
self.length_filter = _LengthFilter(indices=[
i for i, source in enumerate(stream.sources)
if source in self.filter_by
],
max_length=self.max_length)
stream = Filter(stream, self.length_filter)
stream = ForceFloatX(stream)
if not batches:
return stream
stream = Batch(
stream,
iteration_scheme=ConstantScheme(self.batch_size if part == 'train'
else self.validation_batch_size))
stream = Padding(stream)
stream = Mapping(stream, switch_first_two_axes)
stream = ForceCContiguous(stream)
return stream
def test_axis_labels(self):
axis_labels = {'x': ('batch', 'feature'), 'y': ('batch', 'index')}
self.dataset.axis_labels = axis_labels
transformer = ForceFloatX(DataStream(self.dataset))
assert_equal(transformer.axis_labels, axis_labels)
def _create_main_loop(self):
# hyper parameters
hp = self.params
batch_size = hp['batch_size']
biases_init = Constant(0)
batch_normalize = hp['batch_normalize']
### Build fprop
tensor5 = T.TensorType(config.floatX, (False, ) * 5)
X = tensor5("images")
#X = T.tensor4("images")
y = T.lvector('targets')
gnet_params = OrderedDict()
#X_shuffled = X[:, :, :, :, [2, 1, 0]]
#X_shuffled = gpu_contiguous(X.dimshuffle(0, 1, 4, 2, 3)) * 255
X = X[:, :, :, :, [2, 1, 0]]
X_shuffled = X.dimshuffle((0, 1, 4, 2, 3)) * 255
X_r = X_shuffled.reshape(
(X_shuffled.shape[0], X_shuffled.shape[1] * X_shuffled.shape[2],
X_shuffled.shape[3], X_shuffled.shape[4]))
X_r = X_r - (np.array([104, 117, 123])[None, :, None,
None]).astype('float32')
expressions, input_data, param = stream_layer_exp(inputs=('data', X_r),
mode='rgb')
res = expressions['outloss']
y_hat = res.flatten(ndim=2)
import pdb
pdb.set_trace()
### Build Cost
cost = CategoricalCrossEntropy().apply(y, y_hat)
cost = T.cast(cost, theano.config.floatX)
cost.name = 'cross_entropy'
y_pred = T.argmax(y_hat, axis=1)
misclass = T.cast(T.mean(T.neq(y_pred, y)), theano.config.floatX)
misclass.name = 'misclass'
monitored_channels = []
monitored_quantities = [cost, misclass, y_hat, y_pred]
model = Model(cost)
training_cg = ComputationGraph(monitored_quantities)
inference_cg = ComputationGraph(monitored_quantities)
### Get evaluation function
#training_eval = training_cg.get_theano_function(additional_updates=bn_updates)
training_eval = training_cg.get_theano_function()
#inference_eval = inference_cg.get_theano_function()
# Dataset
test = JpegHDF5Dataset(
'test',
#name='jpeg_data_flows.hdf5',
load_in_memory=True)
#mean = np.load(os.path.join(os.environ['UCF101'], 'mean.npy'))
import pdb
pdb.set_trace()
### Eval
labels = np.zeros(test.num_video_examples)
y_hat = np.zeros((test.num_video_examples, 101))
labels_flip = np.zeros(test.num_video_examples)
y_hat_flip = np.zeros((test.num_video_examples, 101))
### Important to shuffle list for batch normalization statistic
#rng = np.random.RandomState()
#examples_list = range(test.num_video_examples)
#import pdb; pdb.set_trace()
#rng.shuffle(examples_list)
nb_frames = 1
for i in xrange(24):
scheme = HDF5SeqScheme(test.video_indexes,
examples=test.num_video_examples,
batch_size=batch_size,
f_subsample=i,
nb_subsample=25,
frames_per_video=nb_frames)
#for crop in ['upleft', 'upright', 'downleft', 'downright', 'center']:
for crop in ['center']:
stream = JpegHDF5Transformer(
input_size=(240, 320),
crop_size=(224, 224),
#input_size=(256, 342), crop_size=(224, 224),
crop_type=crop,
translate_labels=True,
flip='noflip',
nb_frames=nb_frames,
data_stream=ForceFloatX(
DataStream(dataset=test, iteration_scheme=scheme)))
stream_flip = JpegHDF5Transformer(
input_size=(240, 320),
crop_size=(224, 224),
#input_size=(256, 342), crop_size=(224, 224),
crop_type=crop,
translate_labels=True,
flip='flip',
nb_frames=nb_frames,
data_stream=ForceFloatX(
DataStream(dataset=test, iteration_scheme=scheme)))
## Do the evaluation
epoch = stream.get_epoch_iterator()
for j, batch in enumerate(epoch):
output = training_eval(batch[0], batch[1])
# import cv2
# cv2.imshow('img', batch[0][0, 0, :, :, :])
# cv2.waitKey(160)
# cv2.destroyAllWindows()
#import pdb; pdb.set_trace()
labels_flip[batch_size * j:batch_size * (j + 1)] = batch[1]
y_hat_flip[batch_size * j:batch_size *
(j + 1), :] += output[2]
preds = y_hat_flip.argmax(axis=1)
misclass = np.sum(labels_flip != preds) / float(len(preds))
print i, crop, "flip Misclass:", misclass
epoch = stream_flip.get_epoch_iterator()
for j, batch in enumerate(epoch):
output = training_eval(batch[0], batch[1])
labels[batch_size * j:batch_size * (j + 1)] = batch[1]
y_hat[batch_size * j:batch_size * (j + 1), :] += output[2]
preds = y_hat.argmax(axis=1)
misclass = np.sum(labels != preds) / float(len(preds))
print i, crop, "noflip Misclass:", misclass
y_merge = y_hat + y_hat_flip
preds = y_merge.argmax(axis=1)
misclass = np.sum(labels != preds) / float(len(preds))
print i, crop, "avg Misclass:", misclass
### Compute misclass
y_hat += y_hat_flip
preds = y_hat.argmax(axis=1)
misclass = np.sum(labels != preds) / float(len(preds))
print "Misclass:", misclass
# input_mat = np.random.rand(batch_size, input_dim0, input_dim1).astype(theano.config.floatX)
# input_ratings = np.zeros((batch_size, input_dim0, input_dim1), dtype=theano.config.floatX)
# input_ratings[:, :, 0] = 1
trainset = MovieLens1M(which_set=['train'], sources=('input_ratings', 'output_ratings', 'input_masks', 'output_masks'))
validset = MovieLens1M(which_set=['valid'], sources=('input_ratings', 'output_ratings', 'input_masks', 'output_masks'))
testset = MovieLens1M(which_set=['test'], sources=('input_ratings', 'output_ratings', 'input_masks', 'output_masks'))
train_loop_stream = ForceFloatX(
data_stream=MovieLensTransformer(
data_stream=Trainer_MovieLensTransformer(
data_stream=DataStream(
dataset=trainset,
iteration_scheme=ShuffledScheme(
trainset.num_examples,
batch_size
)
)
)
)
)
trainval_loop_stream = ForceFloatX(
data_stream=MovieLensTransformer(
data_stream=Trainer_MovieLensTransformer(
data_stream=DataStream(
dataset=testset,
iteration_scheme=ShuffledScheme(
trainset.num_examples,
def test_value_error_on_request(self):
transformer = ForceFloatX(DataStream(self.dataset))
assert_raises(ValueError, transformer.get_data, [0, 1])
k = 20
target_size = frame_size * k
hidden_size_recurrent = 400
readout_size = 6 * k + 1
lr = 3e-4
dataset = Handwriting(('train', ))
data_stream = DataStream.default_stream(dataset,
iteration_scheme=SequentialScheme(
dataset.num_examples, batch_size))
data_stream = FilterSources(data_stream, sources=('features', ))
data_stream = Padding(data_stream)
data_stream = Mapping(data_stream, _transpose)
data_stream = ForceFloatX(data_stream)
dataset = Handwriting(('valid', ))
valid_stream = DataStream.default_stream(dataset,
iteration_scheme=SequentialScheme(
dataset.num_examples,
10 * batch_size))
valid_stream = FilterSources(valid_stream, sources=('features', ))
valid_stream = Padding(valid_stream)
valid_stream = Mapping(valid_stream, _transpose)
valid_stream = ForceFloatX(valid_stream)
x_tr = next(data_stream.get_epoch_iterator())
x = tensor.tensor3('features')
x_mask = tensor.matrix('features_mask')
def get_stream(self, part, batches=True, shuffle=True, add_sources=(),
num_examples=None, rng=None, seed=None):
dataset = self.get_dataset(part, add_sources=add_sources)
if num_examples is None:
num_examples = dataset.num_examples
if shuffle:
iteration_scheme = ShuffledExampleScheme(num_examples, rng=rng)
else:
iteration_scheme = SequentialExampleScheme(num_examples)
stream = DataStream(
dataset, iteration_scheme=iteration_scheme)
if self.add_eos:
stream = Mapping(stream, _AddLabel(
self.eos_label,
index=stream.sources.index(self.sources_map['labels'])))
if self.add_bos:
if self.bos_label is None:
raise Exception('No bos label given')
stream = Mapping(stream, _AddLabel(
self.bos_label, append=False, times=self.add_bos,
index=stream.sources.index(self.sources_map['labels'])))
if self.max_length:
stream = Filter(stream, self.length_filter)
if self.sort_k_batches and batches:
stream = Batch(stream,
iteration_scheme=ConstantScheme(
self.batch_size * self.sort_k_batches))
#
# Hardcode 0 for source on which to sort. This will be good, as
# most source lengths are correlated and, furthermore, the
# labels will typically be the last source, thus in a single-input
# case this sorts on input lengths
#
stream = Mapping(stream, SortMapping(_Length(
index=0)))
stream = Unpack(stream)
if self.normalization:
stream = self.normalization.wrap_stream(stream)
stream = ForceFloatX(stream)
stream = Rename(stream,
names=dict_subset({v: k for (k, v)
in self.sources_map.items()},
stream.sources,
must_have=False))
if not batches:
return stream
stream = Batch(
stream,
iteration_scheme=ConstantScheme(self.batch_size if part == 'train'
else self.validation_batch_size))
stream = Padding(stream)
stream = Mapping(stream, switch_first_two_axes)
stream = ForceCContiguous(stream)
stream._produces_examples = False
return stream
def test_axis_labels_are_passed_through(self):
axis_labels = {'x': ('batch', 'feature'), 'y': ('batch', 'index')}
self.dataset.axis_labels = axis_labels
stream = DataStream(self.dataset)
transformer = ForceFloatX(stream)
assert_equal(transformer.axis_labels, stream.axis_labels)
def fit(self, trainset, retrain=True):
batch_size = self.batch_size
n_iter = self.n_iter
look_ahead = self.look_ahead
lr = self.lr
b1 = self.b1
b2 = self.b2
epsilon = self.epsilon
hidden_size = self.hidden_size
activation_function = self.activation_function
drop_rate = self.drop_rate
weight_decay = self.weight_decay
optimizer = self.optimizer
std = self.std
alpha = self.alpha
polyak_mu = self.polyak_mu
rating_category = self.rating_category
item_num = self.item_num
user_num = self.user_num
trainset = self.load_dataset(which_set=['train'],
sources=('input_ratings',
'output_ratings', 'input_masks',
'output_masks'))
validset = self.load_dataset(which_set=['valid'],
sources=('input_ratings',
'output_ratings', 'input_masks',
'output_masks'))
train_loop_stream = ForceFloatX(data_stream=MovieLensTransformer(
data_stream=Trainer_MovieLensTransformer(data_stream=DataStream(
dataset=trainset,
iteration_scheme=ShuffledScheme(trainset.num_examples,
batch_size)))))
valid_monitor_stream = ForceFloatX(data_stream=MovieLensTransformer(
data_stream=DataStream(dataset=validset,
iteration_scheme=ShuffledScheme(
validset.num_examples, batch_size))))
rating_freq = np.zeros((user_num, rating_category))
init_b = np.zeros((user_num, rating_category))
for batch in valid_monitor_stream.get_epoch_iterator():
inp_r, out_r, inp_m, out_m = batch
rating_freq += inp_r.sum(axis=0)
log_rating_freq = np.log(rating_freq + 1e-8)
log_rating_freq_diff = np.diff(log_rating_freq, axis=1)
init_b[:, 1:] = log_rating_freq_diff
init_b[:, 0] = log_rating_freq[:, 0]
# init_b = np.log(rating_freq / (rating_freq.sum(axis=1)[:, None] + 1e-8) +1e-8) * (rating_freq>0)
new_items = np.where(rating_freq.sum(axis=1) == 0)[0]
self.new_items = new_items
input_ratings = T.tensor3(name='input_ratings',
dtype=theano.config.floatX)
output_ratings = T.tensor3(name='output_ratings',
dtype=theano.config.floatX)
input_masks = T.matrix(name='input_masks', dtype=theano.config.floatX)
output_masks = T.matrix(name='output_masks',
dtype=theano.config.floatX)
input_ratings_cum = T.extra_ops.cumsum(input_ratings[:, :, ::-1],
axis=2)[:, :, ::-1]
# hidden_size = [256]
if activation_function == 'reclin':
act = Rectifier
elif activation_function == 'tanh':
act = Tanh
elif activation_function == 'sigmoid':
act = Logistic
else:
act = Softplus
layers_act = [act('layer_%d' % i) for i in range(len(hidden_size))]
NADE_CF_model = tabula_NADE(activations=layers_act,
input_dim0=user_num,
input_dim1=rating_category,
other_dims=hidden_size,
batch_size=batch_size,
weights_init=Uniform(std=0.05),
biases_init=Constant(0.0))
NADE_CF_model.push_initialization_config()
dims = [user_num] + hidden_size + [user_num]
linear_layers = [
layer for layer in NADE_CF_model.children if 'linear' in layer.name
]
assert len(linear_layers) == len(dims) - 1
for i in range(len(linear_layers)):
H1 = dims[i]
H2 = dims[i + 1]
width = 2 * np.sqrt(6) / np.sqrt(H1 + H2)
# std = np.sqrt(2. / dim)
linear_layers[i].weights_init = Uniform(width=width)
NADE_CF_model.initialize()
NADE_CF_model.children[-1].parameters[-1].set_value(
init_b.astype(theano.config.floatX))
y = NADE_CF_model.apply(input_ratings_cum)
y_cum = T.extra_ops.cumsum(y, axis=2)
predicted_ratings = NDimensionalSoftmax().apply(y_cum, extra_ndim=1)
d = input_masks.sum(axis=1)
D = (input_masks + output_masks).sum(axis=1)
cost, nll, nll_item_ratings, cost_ordinal_1N, cost_ordinal_N1, prob_item_ratings = rating_cost(
y,
output_ratings,
input_masks,
output_masks,
D,
d,
alpha=alpha,
std=std)
cost.name = 'cost'
cg = ComputationGraph(cost)
if weight_decay > 0.0:
all_weights = VariableFilter(roles=[WEIGHT])(cg.variables)
l2_weights = T.sum([(W**2).sum() for W in all_weights])
l2_cost = cost + weight_decay * l2_weights
l2_cost.name = 'l2_decay_' + cost.name
cg = ComputationGraph(l2_cost)
if drop_rate > 0.0:
dropped_layer = VariableFilter(roles=[INPUT],
bricks=NADE_CF_model.children)(
cg.variables)
dropped_layer = [
layer for layer in dropped_layer if 'linear' in layer.name
]
dropped_layer = dropped_layer[1:]
cg_dropout = apply_dropout(cg, dropped_layer, drop_rate)
else:
cg_dropout = cg
training_cost = cg_dropout.outputs[0]
lr0 = T.scalar(name='learning_rate', dtype=theano.config.floatX)
input_list = [input_ratings, input_masks, output_ratings, output_masks]
if optimizer == 'Adam':
f_get_grad, f_update_parameters, shared_gradients = Adam_optimizer(
input_list, training_cost, cg_dropout.parameters, lr0, b1, b2,
epsilon)
elif optimizer == 'Adadelta':
f_get_grad, f_update_parameters, shared_gradients = Adadelta_optimizer(
input_list, training_cost, cg_dropout.parameters, lr, epsilon)
else:
f_get_grad, f_update_parameters, shared_gradients = SGD_optimizer(
input_list, training_cost, cg_dropout.parameters, lr0, b1)
param_list = []
[param_list.extend(p.parameters) for p in NADE_CF_model.children]
f_update_polyak, shared_polyak = polyak(param_list, mu=polyak_mu)
f_monitor = theano.function(inputs=[input_ratings],
outputs=[predicted_ratings])
nb_of_epocs_without_improvement = 0
best_valid_error = np.Inf
epoch = 0
best_model = cp.deepcopy(NADE_CF_model)
best_polyak = cp.deepcopy(shared_polyak)
start_training_time = t.time()
lr_tracer = []
rate_score = np.array(list(range(1, rating_category + 1)), np.float32)
best_epoch = -1
while epoch < n_iter and nb_of_epocs_without_improvement < look_ahead:
print('Epoch {0}'.format(epoch))
epoch += 1
start_time_epoch = t.time()
cost_train = []
squared_error_train = []
n_sample_train = []
cntt = 0
train_time = 0
for batch in train_loop_stream.get_epoch_iterator():
inp_r, out_r, inp_m, out_m = batch
train_t = t.time()
cost_value = f_get_grad(inp_r, inp_m, out_r, out_m)
train_time += t.time() - train_t
# pred_ratings = f_monitor(inp_r)
if optimizer == 'Adadelta':
f_update_parameters()
else:
f_update_parameters(lr)
f_update_polyak()
pred_ratings = f_monitor(inp_r)
true_r = out_r.argmax(axis=2) + 1
pred_r = (pred_ratings[0] *
rate_score[np.newaxis, np.newaxis, :]).sum(axis=2)
pred_r[:, new_items] = 3
mask = out_r.sum(axis=2)
se = np.sum(np.square(true_r - pred_r) * mask)
n = np.sum(mask)
squared_error_train.append(se)
n_sample_train.append(n)
cost_train.append(cost_value)
cntt += 1
cost_train = np.array(cost_train).mean()
squared_error_ = np.array(squared_error_train).sum()
n_samples = np.array(n_sample_train).sum()
train_RMSE = np.sqrt(squared_error_ / (n_samples * 1.0 + 1e-8))
print('\tTraining ...')
print('Train :', "RMSE: {0:.6f}".format(train_RMSE),
" Cost Error: {0:.6f}".format(cost_train),
"Train Time: {0:.6f}".format(train_time),
get_done_text(start_time_epoch))
print('\tValidating ...', )
start_time = t.time()
squared_error_valid = []
n_sample_valid = []
valid_time = 0
for batch in valid_monitor_stream.get_epoch_iterator():
inp_r, out_r, inp_m, out_m = batch
valid_t = t.time()
pred_ratings = f_monitor(inp_r)
valid_time += t.time() - valid_t
true_r = out_r.argmax(axis=2) + 1
pred_r = (pred_ratings[0] *
rate_score[np.newaxis, np.newaxis, :]).sum(axis=2)
pred_r[:, new_items] = 3
mask = out_r.sum(axis=2)
se = np.sum(np.square(true_r - pred_r) * mask)
n = np.sum(mask)
squared_error_valid.append(se)
n_sample_valid.append(n)
squared_error_ = np.array(squared_error_valid).sum()
n_samples = np.array(n_sample_valid).sum()
valid_RMSE = np.sqrt(squared_error_ / (n_samples * 1.0 + 1e-8))
print('Validation:', " RMSE: {0:.6f}".format(valid_RMSE),
"Valid Time: {0:.6f}".format(valid_time),
get_done_text(start_time))
if valid_RMSE < best_valid_error:
best_epoch = epoch
nb_of_epocs_without_improvement = 0
best_valid_error = valid_RMSE
del best_model
del best_polyak
gc.collect()
best_model = cp.deepcopy(NADE_CF_model)
best_polyak = cp.deepcopy(shared_polyak)
print('\n\n Got a good one')
else:
nb_of_epocs_without_improvement += 1
if optimizer == 'Adadelta':
pass
elif nb_of_epocs_without_improvement == look_ahead and lr > 1e-5:
nb_of_epocs_without_improvement = 0
lr /= 4
print("learning rate is now %s" % lr)
lr_tracer.append(lr)
print('\n### Training, n_layers=%d' % (len(hidden_size)),
get_done_text(start_training_time))
best_y = best_model.apply(input_ratings_cum)
best_y_cum = T.extra_ops.cumsum(best_y, axis=2)
best_predicted_ratings = NDimensionalSoftmax().apply(best_y_cum,
extra_ndim=1)
self.f_monitor_best = theano.function(inputs=[input_ratings],
outputs=[best_predicted_ratings])
self.best_valid_error = best_valid_error
self.best_epoch = best_epoch
self.best_model = best_model
self.best_polyak = best_polyak
def prediction(self, test_input: list):
"""
:param test_input: (uid, item_id) list
:return: predicted rates
"""
model_manager = self.model_manager
testset = self.load_dataset(which_set=['test'],
sources=('input_ratings', 'output_ratings',
'input_masks', 'output_masks'))
test_monitor_stream = ForceFloatX(data_stream=MovieLensTransformer(
data_stream=DataStream(dataset=testset,
iteration_scheme=SequentialScheme(
testset.num_examples,
self.batch_size))))
f_monitor_best = self.f_monitor_best
best_valid_error = self.best_valid_error
best_model = self.best_model
best_polyak = self.best_polyak
best_epoch = self.best_epoch
rating_category = self.rating_category
new_items = self.new_items
print('\tTesting ...', )
start_time = t.time()
squared_error_test = []
n_sample_test = []
test_time = 0
rate_score = np.array(list(range(1, rating_category + 1)), np.float32)
preds = []
for batch in test_monitor_stream.get_epoch_iterator():
inp_r, out_r, inp_m, _ = batch
test_t = t.time()
pred_ratings = f_monitor_best(inp_r)
test_time += t.time() - test_t
true_r = out_r.argmax(axis=2) + 1
pred_r = (pred_ratings[0] *
rate_score[np.newaxis, np.newaxis, :]).sum(axis=2)
pred_r[:, new_items] = 3
mask = out_r.sum(axis=2)
se = np.sum(np.square(true_r - pred_r) * mask)
n = np.sum(mask)
squared_error_test.append(se)
n_sample_test.append(n)
preds.extend(pred_r)
predictions = list(map(lambda x: preds[x[1]][x[0]], test_input))
squared_error_ = np.array(squared_error_test).sum()
n_samples = np.array(n_sample_test).sum()
test_RMSE = np.sqrt(squared_error_ / (n_samples * 1.0 + 1e-8))
print('Test:', " RMSE: {0:.6f}".format(test_RMSE),
"Test Time: {0:.6f}".format(test_time),
get_done_text(start_time))
f = open(
os.path.join(model_manager.path_name,
'Reco_NADE_masked_directly_itembased.txt'), 'a')
to_write = {
'test_RMSE': test_RMSE,
'best_valid_error': best_valid_error,
'best_epoch': best_epoch
}
to_write.update(
dict(
filter(lambda x: type(x[1]) in [int, float, str],
self.__dict__.items())))
json.dump(to_write, f, ensure_ascii=False)
f.close()
print('\tTesting with polyak parameters...', )
best_param_list = []
[best_param_list.extend(p.parameters) for p in best_model.children]
f_replace = polyak_replace(best_param_list, best_polyak)
f_replace()
cc = 0
for pp in best_polyak:
pp_value = pp.get_value()
np.save(os.path.join(self.model_manager.path_name, str(cc)),
pp_value)
cc += 1
start_time = t.time()
squared_error_test = []
n_sample_test = []
test_time = 0
for batch in test_monitor_stream.get_epoch_iterator():
inp_r, out_r, inp_m, _ = batch
test_t = t.time()
pred_ratings = f_monitor_best(inp_r)
test_time += t.time() - test_t
true_r = out_r.argmax(axis=2) + 1
pred_r = (pred_ratings[0] *
rate_score[np.newaxis, np.newaxis, :]).sum(axis=2)
pred_r[:, new_items] = 3
mask = out_r.sum(axis=2)
se = np.sum(np.square(true_r - pred_r) * mask)
n = np.sum(mask)
squared_error_test.append(se)
n_sample_test.append(n)
squared_error_ = np.array(squared_error_test).sum()
n_samples = np.array(n_sample_test).sum()
test_RMSE = np.sqrt(squared_error_ / (n_samples * 1.0 + 1e-8))
print('Test:', " RMSE: {0:.6f}".format(test_RMSE),
"Test Time: {0:.6f}".format(test_time),
get_done_text(start_time))
f = open(
os.path.join(self.model_manager.path_name,
'Reco_NADE_masked_directly_itembased.txt'), 'a')
to_write = {
'test_RMSE': test_RMSE,
'best_valid_error': best_valid_error,
'best_epoch': best_epoch
}
to_write.update(
dict(
filter(lambda x: type(x[1]) in [int, float, str],
self.__dict__.items())))
json.dump(to_write, f, ensure_ascii=False)
f.close()
return predictions
shift=-mean_data / std_data)
data_stream = Mapping(data_stream,
_downsample_and_upsample,
add_sources=('upsampled', ))
data_stream = Mapping(data_stream, _equalize_size)
data_stream = Mapping(data_stream, _get_residual, add_sources=('residual', ))
data_stream = FilterSources(data_stream, sources=(
'upsampled',
'residual',
))
data_stream = Mapping(data_stream, _segment_axis)
data_stream = Padding(data_stream)
data_stream = FilterSources(data_stream,
sources=('upsampled', 'residual', 'residual_mask'))
data_stream = Mapping(data_stream, _transpose)
data_stream = ForceFloatX(data_stream)
#################
# Model
#################
activations_x = [Rectifier()] * depth_x
dims_x = [frame_size] + [hidden_size_mlp_x]*(depth_x-1) + \
[hidden_size_recurrent]
activations_theta = [Rectifier()] * depth_theta
dims_theta = [hidden_size_recurrent] + \
[hidden_size_mlp_theta]*depth_theta
def main(num_epochs=1000):
# MODEL
# defining the data
x = tensor.matrix('features')
y = tensor.lmatrix('targets')
# defining the model
softmax_regressor = SoftmaxRegressor(input_dim=784, n_classes=10)
# defining the cost to learn on
probs = softmax_regressor.get_probs(features=x)
cost = softmax_regressor.get_cost(probs=probs, targets=y).mean()
cost.name = 'cost'
# defining the cost to monitor
misclassification = softmax_regressor.get_misclassification(
probs=probs, targets=y).mean()
misclassification.name = 'misclassification'
# DATASETS
# defining the datasets
train_dataset = MNIST('train')
test_dataset = MNIST('test')
# TRAINING ALGORITHM
# defining the algorithm
params = softmax_regressor.get_params()
algorithm = GradientDescent(cost=cost,
params=params,
step_rule=Momentum(learning_rate=0.1,
momentum=0.1))
# defining the data stream
# how the dataset is read
train_data_stream = ForceFloatX(
data_stream=DataStream(dataset=train_dataset,
iteration_scheme=ShuffledScheme(
examples=train_dataset.num_examples,
batch_size=100,
)))
test_data_stream = ForceFloatX(
data_stream=DataStream(dataset=test_dataset,
iteration_scheme=SequentialScheme(
examples=test_dataset.num_examples,
batch_size=1000,
)))
# MONITORING
# defining the extensions
extensions = []
# timing the training and each epoch
extensions.append(Timing())
# ending the training after a certain number of epochs
extensions.append(FinishAfter(after_n_epochs=num_epochs))
# monitoring the test set
extensions.append(
DataStreamMonitoring([cost, misclassification],
test_data_stream,
prefix='test'))
# monitoring the training set while training
extensions.append(
TrainingDataMonitoring([cost, misclassification],
prefix='train',
after_every_epoch=True))
# printing quantities
extensions.append(Printing())
# MERGING IT TOGETHER
# defining the model
model = Model(cost)
# defining the training main loop
main_loop = MainLoop(model=model,
data_stream=train_data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
