def test_parallel_queued_pipeline_with_step_name_n_worker_max_queue_size():
p = ParallelQueuedFeatureUnion([('1', 1, 5, MultiplyByN(2)),
('2', 1, 5, MultiplyByN(2)),
('3', 1, 5, MultiplyByN(2)),
('4', 1, 5, MultiplyByN(2))],
batch_size=10)
outputs = p.transform(list(range(100)))
assert np.array_equal(outputs, EXPECTED_OUTPUTS_PARALLEL)
def test_parallel_queued_pipeline_with_step_name_n_worker_additional_arguments():
n_workers = 4
worker_arguments = [('hyperparams', HyperparameterSamples({'multiply_by': 2})) for _ in range(n_workers)]
p = ParallelQueuedFeatureUnion([
('1', n_workers, worker_arguments, MultiplyByN()),
], batch_size=10, max_queue_size=5)
outputs = p.transform(list(range(100)))
expected = np.array(list(range(0, 200, 2)))
assert np.array_equal(outputs, expected)
def test_queued_pipeline_saving(tmpdir):
# Given
p = ParallelQueuedFeatureUnion([
('1', FitTransformCallbackStep()),
('2', FitTransformCallbackStep()),
('3', FitTransformCallbackStep()),
('4', FitTransformCallbackStep()),
], n_workers_per_step=1, max_queue_size=10, batch_size=10)
# When
p, outputs = p.fit_transform(list(range(100)), list(range(100)))
p.save(ExecutionContext(tmpdir))
p.apply('clear_callbacks')
# Then
assert len(p[0].wrapped.transform_callback_function.data) == 0
assert len(p[0].wrapped.fit_callback_function.data) == 0
assert len(p[1].wrapped.transform_callback_function.data) == 0
assert len(p[1].wrapped.fit_callback_function.data) == 0
assert len(p[2].wrapped.transform_callback_function.data) == 0
assert len(p[2].wrapped.fit_callback_function.data) == 0
assert len(p[3].wrapped.transform_callback_function.data) == 0
assert len(p[3].wrapped.fit_callback_function.data) == 0
p = p.load(ExecutionContext(tmpdir))
assert len(p[0].wrapped.transform_callback_function.data) == 10
assert len(p[0].wrapped.fit_callback_function.data) == 10
assert len(p[1].wrapped.transform_callback_function.data) == 10
assert len(p[1].wrapped.fit_callback_function.data) == 10
assert len(p[2].wrapped.transform_callback_function.data) == 10
assert len(p[2].wrapped.fit_callback_function.data) == 10
assert len(p[3].wrapped.transform_callback_function.data) == 10
assert len(p[3].wrapped.fit_callback_function.data) == 10
def test_queued_pipeline_with_savers(tmpdir):
# Given
p = ParallelQueuedFeatureUnion([
('1', MultiplyByN(2)),
('2', MultiplyByN(2)),
('3', MultiplyByN(2)),
('4', MultiplyByN(2)),
], n_workers_per_step=1, max_queue_size=10, batch_size=10, use_savers=True, cache_folder=tmpdir)
# When
outputs = p.transform(list(range(100)))
# Then
assert np.array_equal(outputs, EXPECTED_OUTPUTS_PARALLEL)
def test_queued_pipeline_saving(tmpdir, use_processes, use_savers):
# Given
p = ParallelQueuedFeatureUnion([
('1', 4, 10, FitTransformCallbackStep()),
('2', 4, 10, FitTransformCallbackStep()),
('3', 4, 10, FitTransformCallbackStep()),
('4', 4, 10, FitTransformCallbackStep()),
],
n_workers_per_step=4,
max_queue_size=10,
batch_size=10,
use_processes=use_processes,
use_savers=use_savers).with_context(
ExecutionContext(tmpdir))
# When
p, _ = p.fit_transform(list(range(200)), list(range(200)))
p = p.wrapped # clear execution context wrapper
p.save(ExecutionContext(tmpdir))
p.apply('clear_callbacks')
# Then
assert len(p[0].wrapped.transform_callback_function.data) == 0
assert len(p[0].wrapped.fit_callback_function.data) == 0
assert len(p[1].wrapped.transform_callback_function.data) == 0
assert len(p[1].wrapped.fit_callback_function.data) == 0
assert len(p[2].wrapped.transform_callback_function.data) == 0
assert len(p[2].wrapped.fit_callback_function.data) == 0
assert len(p[3].wrapped.transform_callback_function.data) == 0
assert len(p[3].wrapped.fit_callback_function.data) == 0
p = p.load(ExecutionContext(tmpdir))
assert len(p[0].wrapped.transform_callback_function.data) == 20
assert len(p[0].wrapped.fit_callback_function.data) == 20
assert len(p[1].wrapped.transform_callback_function.data) == 20
assert len(p[1].wrapped.fit_callback_function.data) == 20
assert len(p[2].wrapped.transform_callback_function.data) == 20
assert len(p[2].wrapped.fit_callback_function.data) == 20
assert len(p[3].wrapped.transform_callback_function.data) == 20
assert len(p[3].wrapped.fit_callback_function.data) == 20
