def test_shared_variable_modifier_two_params():
weights = numpy.array([-1, 1], dtype=floatX)
features = [numpy.array(f, dtype=floatX) for f in [[1, 2], [3, 4], [5, 6]]]
targets = [(weights * f).sum() for f in features]
n_batches = 3
dataset = ContainerDataset(dict(features=features, targets=targets))
x = tensor.vector('features')
y = tensor.scalar('targets')
W = shared_floatx([0, 0], name='W')
cost = ((x * W).sum() - y)**2
cost.name = 'cost'
step_rule = Scale(0.001)
sgd = GradientDescent(cost=cost, params=[W], step_rule=step_rule)
modifier = SharedVariableModifier(
step_rule.learning_rate, lambda _, val: numpy.cast[floatX](val * 0.2))
main_loop = MainLoop(model=None,
data_stream=dataset.get_default_stream(),
algorithm=sgd,
extensions=[FinishAfter(after_n_epochs=1), modifier])
main_loop.run()
new_value = step_rule.learning_rate.get_value()
assert_allclose(new_value, 0.001 * 0.2**n_batches, atol=1e-5)
def test_dataset_evaluators():
X = theano.tensor.matrix('X')
brick = TestBrick(name='test_brick')
Y = brick.apply(X)
graph = ComputationGraph([Y])
monitor_variables = [v for v in graph.auxiliary_variables]
validator = DatasetEvaluator(monitor_variables)
data = [
numpy.arange(1, 5, dtype=floatX).reshape(2, 2),
numpy.arange(10, 16, dtype=floatX).reshape(3, 2)
]
data_stream = ContainerDataset(dict(X=data)).get_default_stream()
values = validator.evaluate(data_stream)
assert values['test_brick_apply_V_squared'] == 4
numpy.testing.assert_allclose(values['test_brick_apply_mean_row_mean'],
numpy.vstack(data).mean())
per_batch_mean = numpy.mean([batch.mean() for batch in data])
numpy.testing.assert_allclose(
values['test_brick_apply_mean_batch_element'], per_batch_mean)
with assert_raises(Exception) as ar:
data_stream = ContainerDataset(dict(X2=data)).get_default_stream()
validator.evaluate(data_stream)
assert "Not all data sources" in ar.exception.args[0]
def test_floatx():
x = [numpy.array(d, dtype="float64") for d in [[1, 2], [3, 4]]]
y = [numpy.array(d, dtype="int64") for d in [1, 2, 3]]
dataset = ContainerDataset(OrderedDict([("x", x), ("y", y)]))
data = next(ForceFloatX(dataset.get_default_stream()).get_epoch_iterator())
assert str(data[0].dtype) == floatX
assert str(data[1].dtype) == "int64"
def get_data_stream(iterable):
dataset = ContainerDataset({'numbers': iterable})
data_stream = DataStreamMapping(dataset.get_default_stream(),
_data_sqrt,
add_sources=('roots', ))
data_stream = DataStreamMapping(data_stream, _array_tuple)
return BatchDataStream(data_stream, ConstantScheme(20))
def get_data_stream(iterable):
dataset = ContainerDataset({'numbers': iterable})
data_stream = DataStreamMapping(dataset.get_default_stream(),
lambda data: (math.sqrt(data[0]), ),
add_sources=('roots', ))
data_stream = DataStreamMapping(
data_stream, lambda data: tuple(
(numpy.asarray(d, dtype=floatX) for d in data)))
return BatchDataStream(data_stream, ConstantScheme(20))
def test_training_data_monitoring():
weights = numpy.array([-1, 1], dtype=floatX)
features = [numpy.array(f, dtype=floatX) for f in [[1, 2], [3, 4], [5, 6]]]
targets = [(weights * f).sum() for f in features]
n_batches = 3
dataset = ContainerDataset(dict(features=features, targets=targets))
x = tensor.vector('features')
y = tensor.scalar('targets')
W = shared_floatx([0, 0], name='W')
V = shared_floatx(7, name='V')
W_sum = named_copy(W.sum(), 'W_sum')
cost = ((x * W).sum() - y)**2
cost.name = 'cost'
class TrueCostExtension(TrainingExtension):
def before_batch(self, data):
self.main_loop.log.current_row.true_cost = ((
(W.get_value() * data["features"]).sum() - data["targets"])**2)
main_loop = MainLoop(model=None,
data_stream=dataset.get_default_stream(),
algorithm=GradientDescent(cost=cost,
params=[W],
step_rule=Scale(0.001)),
extensions=[
FinishAfter(after_n_epochs=1),
TrainingDataMonitoring([W_sum, cost, V],
prefix="train1",
after_every_batch=True),
TrainingDataMonitoring(
[aggregation.mean(W_sum), cost],
prefix="train2",
after_every_epoch=True),
TrueCostExtension()
])
main_loop.run()
# Check monitoring of a shared varible
assert_allclose(main_loop.log.current_row.train1_V, 7.0)
for i in range(n_batches):
# The ground truth is written to the log before the batch is
# processed, where as the extension writes after the batch is
# processed. This is why the iteration numbers differs here.
assert_allclose(main_loop.log[i].true_cost,
main_loop.log[i + 1].train1_cost)
assert_allclose(
main_loop.log[n_batches].train2_cost,
sum([main_loop.log[i].true_cost
for i in range(n_batches)]) / n_batches)
assert_allclose(
main_loop.log[n_batches].train2_W_sum,
sum([main_loop.log[i].train1_W_sum
for i in range(1, n_batches + 1)]) / n_batches)
def test_sources_selection():
features = [5, 6, 7, 1]
targets = [1, 0, 1, 1]
stream = ContainerDataset(OrderedDict(
[('features', features), ('targets', targets)])).get_default_stream()
assert list(stream.get_epoch_iterator()) == list(zip(features, targets))
stream = ContainerDataset({'features': features, 'targets': targets},
sources=('targets',)).get_default_stream()
assert list(stream.get_epoch_iterator()) == list(zip(targets))
def test_dataset():
data = [1, 2, 3]
# The default stream requests an example at a time
stream = ContainerDataset(data).get_default_stream()
epoch = stream.get_epoch_iterator()
assert list(epoch) == list(zip(data))
# Check if iterating over multiple epochs works
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
# Check whether the returning as a dictionary of sources works
assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
def do_test(with_serialization):
data_stream = ContainerDataset(range(10)).get_default_stream()
main_loop = MainLoop(MockAlgorithm(),
data_stream,
extensions=[
WriteBatchExtension(),
FinishAfter(after_n_batches=14)
])
main_loop.run()
assert main_loop.log.status.iterations_done == 14
if with_serialization:
main_loop = cPickle.loads(cPickle.dumps(main_loop))
finish_after = unpack([
ext
for ext in main_loop.extensions if isinstance(ext, FinishAfter)
],
singleton=True)
finish_after.add_condition(
"after_batch",
predicate=lambda log: log.status.iterations_done == 27)
main_loop.run()
assert main_loop.log.status.iterations_done == 27
assert main_loop.log.status.epochs_done == 2
for i in range(27):
assert main_loop.log[i + 1].batch == {"data": i % 10}
def do_test(with_serialization):
data_stream = ContainerDataset(range(10)).get_default_stream()
main_loop = MainLoop(None,
data_stream,
MockAlgorithm(),
extensions=[FinishAfter(after_n_batches=14)])
main_loop.run()
assert main_loop.log.status.iterations_done == 14
if with_serialization:
string_io = BytesIO()
dill.dump(main_loop, string_io, fmode=dill.CONTENTS_FMODE)
string_io.seek(0)
main_loop = dill.load(string_io)
finish_after = unpack([
ext
for ext in main_loop.extensions if isinstance(ext, FinishAfter)
],
singleton=True)
finish_after.add_condition(
"after_batch",
predicate=lambda log: log.status.iterations_done == 27)
main_loop.run()
assert main_loop.log.status.iterations_done == 27
assert main_loop.log.status.epochs_done == 2
for i in range(27):
assert main_loop.log[i].batch == {"data": i % 10}
def test_cache():
dataset = ContainerDataset(range(100))
stream = DataStream(dataset)
batched_stream = BatchDataStream(stream, ConstantScheme(11))
cached_stream = CachedDataStream(batched_stream, ConstantScheme(7))
epoch = cached_stream.get_epoch_iterator()
# Make sure that cache is filled as expected
for (features,), cache_size in zip(epoch, [4, 8, 1, 5, 9, 2,
6, 10, 3, 7, 0, 4]):
assert len(cached_stream.cache[0]) == cache_size
# Make sure that the epoch finishes correctly
for (features,) in cached_stream.get_epoch_iterator():
pass
assert len(features) == 100 % 7
assert not cached_stream.cache[0]
# Ensure that the epoch transition is correct
cached_stream = CachedDataStream(batched_stream,
ConstantScheme(7, times=3))
for _, epoch in zip(range(2), cached_stream.iterate_epochs()):
cache_sizes = [4, 8, 1]
for i, (features,) in enumerate(epoch):
assert len(cached_stream.cache[0]) == cache_sizes[i]
assert len(features) == 7
assert numpy.all(range(100)[i * 7:(i + 1) * 7] == features)
assert i == 2
def test_batch_data_stream():
stream = ContainerDataset([1, 2, 3, 4, 5]).get_default_stream()
batches = list(BatchDataStream(stream, ConstantScheme(2))
.get_epoch_iterator())
expected = [(numpy.array([1, 2]),),
(numpy.array([3, 4]),),
(numpy.array([5]),)]
assert len(batches) == len(expected)
for b, e in zip(batches, expected):
assert (b[0] == e[0]).all()
# Check the `strict` flag
def try_strict():
list(BatchDataStream(stream, ConstantScheme(2), strict=True)
.get_epoch_iterator())
assert_raises(ValueError, try_strict)
stream2 = ContainerDataset([1, 2, 3, 4, 5, 6]).get_default_stream()
assert len(list(BatchDataStream(stream2, ConstantScheme(2), strict=True)
.get_epoch_iterator())) == 3
def test_data_stream_mapping():
data = [1, 2, 3]
data_doubled = [2, 4, 6]
stream = ContainerDataset(data).get_default_stream()
wrapper1 = DataStreamMapping(
stream, lambda d: (2 * d[0],))
assert list(wrapper1.get_epoch_iterator()) == list(zip(data_doubled))
wrapper2 = DataStreamMapping(
stream, lambda d: (2 * d[0],), add_sources=("doubled",))
assert wrapper2.sources == ("data", "doubled")
assert list(wrapper2.get_epoch_iterator()) == list(zip(data, data_doubled))
def test_data_stream_mapping_sort_multisource():
data = OrderedDict()
data['x'] = [[1, 2, 3], [2, 3, 1], [3, 2, 1]]
data['y'] = [[6, 5, 4], [6, 5, 4], [6, 5, 4]]
data_sorted = [([1, 2, 3], [6, 5, 4]),
([1, 2, 3], [4, 6, 5]),
([1, 2, 3], [4, 5, 6])]
stream = ContainerDataset(data).get_default_stream()
wrapper = DataStreamMapping(stream,
mapping=SortMapping(operator.itemgetter(0)))
assert list(wrapper.get_epoch_iterator()) == data_sorted
def test_padding_data_stream():
# 1-D sequences
stream = BatchDataStream(
ContainerDataset([[1], [2, 3], [], [4, 5, 6], [7]])
.get_default_stream(),
ConstantScheme(2))
mask_stream = PaddingDataStream(stream)
assert mask_stream.sources == ("data", "data_mask")
it = mask_stream.get_epoch_iterator()
data, mask = next(it)
assert (data == numpy.array([[1, 0], [2, 3]])).all()
assert (mask == numpy.array([[1, 0], [1, 1]])).all()
data, mask = next(it)
assert (data == numpy.array([[0, 0, 0], [4, 5, 6]])).all()
assert (mask == numpy.array([[0, 0, 0], [1, 1, 1]])).all()
data, mask = next(it)
assert (data == numpy.array([[7]])).all()
assert (mask == numpy.array([[1]])).all()
# 2D sequences
stream2 = BatchDataStream(
ContainerDataset([numpy.ones((3, 4)), 2 * numpy.ones((2, 4))])
.get_default_stream(),
ConstantScheme(2))
it = PaddingDataStream(stream2).get_epoch_iterator()
data, mask = next(it)
assert data.shape == (2, 3, 4)
assert (data[0, :, :] == 1).all()
assert (data[1, :2, :] == 2).all()
assert (mask == numpy.array([[1, 1, 1], [1, 1, 0]])).all()
# 2 sources
stream3 = PaddingDataStream(BatchDataStream(
ContainerDataset(dict(features=[[1], [2, 3], []],
targets=[[4, 5, 6], [7]]))
.get_default_stream(),
ConstantScheme(2)))
assert len(next(stream3.get_epoch_iterator())) == 4
def test_dataset():
data = [1, 2, 3]
data_doubled = [2, 4, 6]
# The default stream requests an example at a time
stream = ContainerDataset([1, 2, 3]).get_default_stream()
epoch = stream.get_epoch_iterator()
assert list(epoch) == list(zip(data))
# Check if iterating over multiple epochs works
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
for i, epoch in enumerate(stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
if i == 1:
break
# Check whether the returning as a dictionary of sources works
assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
# Check whether basic stream wrappers work
wrapper = DataStreamMapping(
stream, lambda d: (2 * d[0],))
assert list(wrapper.get_epoch_iterator()) == list(zip(data_doubled))
def setup_mainloop(extension):
"""Create a MainLoop, register the given extension, supply it with a
DataStream and a minimal model/cost to optimize.
"""
features = [numpy.array(f, dtype=floatX)
for f in [[1, 2], [3, 4], [5, 6]]]
dataset = ContainerDataset(dict(features=features))
W = shared_floatx([0, 0], name='W')
x = tensor.vector('features')
cost = tensor.sum((x-W)**2)
cost.name = "cost"
algorithm = GradientDescent(cost=cost, params=[W],
step_rule=Scale(1e-3))
main_loop = MainLoop(
model=None, data_stream=dataset.get_default_stream(),
algorithm=algorithm,
extensions=[
FinishAfter(after_n_epochs=1),
extension])
return main_loop
def test_data_stream_mapping_sort():
data = [[1, 2, 3],
[2, 3, 1],
[3, 2, 1]]
data_sorted = [[1, 2, 3]] * 3
data_sorted_rev = [[3, 2, 1]] * 3
stream = ContainerDataset(data).get_default_stream()
wrapper1 = DataStreamMapping(stream,
mapping=SortMapping(operator.itemgetter(0)))
assert list(wrapper1.get_epoch_iterator()) == list(zip(data_sorted))
wrapper2 = DataStreamMapping(stream, SortMapping(lambda x: -x[0]))
assert list(wrapper2.get_epoch_iterator()) == list(zip(data_sorted_rev))
wrapper3 = DataStreamMapping(stream, SortMapping(operator.itemgetter(0),
reverse=True))
assert list(wrapper3.get_epoch_iterator()) == list(zip(data_sorted_rev))
def test_sources_selection():
features = [5, 6, 7, 1]
targets = [1, 0, 1, 1]
stream = ContainerDataset(OrderedDict(
[('features', features), ('targets', targets)])).get_default_stream()
assert list(stream.get_epoch_iterator()) == list(zip(features, targets))
stream = ContainerDataset({'features': features, 'targets': targets},
sources=('targets',)).get_default_stream()
assert list(stream.get_epoch_iterator()) == list(zip(targets))
def test_data_stream_mapping_sort_multisource_ndarrays():
data = OrderedDict()
data['x'] = [numpy.array([1, 2, 3]),
numpy.array([2, 3, 1]),
numpy.array([3, 2, 1])]
data['y'] = [numpy.array([6, 5, 4]),
numpy.array([6, 5, 4]),
numpy.array([6, 5, 4])]
data_sorted = [(numpy.array([1, 2, 3]), numpy.array([6, 5, 4])),
(numpy.array([1, 2, 3]), numpy.array([4, 6, 5])),
(numpy.array([1, 2, 3]), numpy.array([4, 5, 6]))]
stream = ContainerDataset(data).get_default_stream()
wrapper = DataStreamMapping(stream,
mapping=SortMapping(operator.itemgetter(0)))
for output, ground_truth in zip(wrapper.get_epoch_iterator(), data_sorted):
assert len(output) == len(ground_truth)
assert (output[0] == ground_truth[0]).all()
assert (output[1] == ground_truth[1]).all()
def test_dataset():
data = [1, 2, 3]
data_doubled = [2, 4, 6]
# The default stream requests an example at a time
stream = ContainerDataset([1, 2, 3]).get_default_stream()
epoch = stream.get_epoch_iterator()
assert list(epoch) == list(zip(data))
# Check if iterating over multiple epochs works
for i, epoch in zip(range(2), stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
for i, epoch in enumerate(stream.iterate_epochs()):
assert list(epoch) == list(zip(data))
if i == 1:
break
# Check whether the returning as a dictionary of sources works
assert next(stream.get_epoch_iterator(as_dict=True)) == {"data": 1}
# Check whether basic stream wrappers work
wrapper = DataStreamMapping(stream, lambda d: (2 * d[0], ))
assert list(wrapper.get_epoch_iterator()) == list(zip(data_doubled))
def test_data_stream_filter():
data = [1, 2, 3]
data_filtered = [1, 3]
stream = ContainerDataset(data).get_default_stream()
wrapper = DataStreamFilter(stream, lambda d: d[0] % 2 == 1)
assert list(wrapper.get_epoch_iterator()) == list(zip(data_filtered))