def test_parallel_queued_pipeline_with_step_name_n_worker_max_queue_size():
p = ParallelQueuedFeatureUnion([('1', 1, 5, MultiplyByN(2)),
('2', 1, 5, MultiplyByN(2)),
('3', 1, 5, MultiplyByN(2)),
('4', 1, 5, MultiplyByN(2))],
batch_size=10)
outputs = p.transform(list(range(100)))
assert np.array_equal(outputs, EXPECTED_OUTPUTS_PARALLEL)
def test_parallel_queued_pipeline_with_step_name_n_worker_additional_arguments():
n_workers = 4
worker_arguments = [('hyperparams', HyperparameterSamples({'multiply_by': 2})) for _ in range(n_workers)]
p = ParallelQueuedFeatureUnion([
('1', n_workers, worker_arguments, MultiplyByN()),
], batch_size=10, max_queue_size=5)
outputs = p.transform(list(range(100)))
expected = np.array(list(range(0, 200, 2)))
assert np.array_equal(outputs, expected)
def test_queued_pipeline_with_savers(tmpdir):
# Given
p = ParallelQueuedFeatureUnion([
('1', MultiplyByN(2)),
('2', MultiplyByN(2)),
('3', MultiplyByN(2)),
('4', MultiplyByN(2)),
], n_workers_per_step=1, max_queue_size=10, batch_size=10, use_savers=True, cache_folder=tmpdir)
# When
outputs = p.transform(list(range(100)))
# Then
assert np.array_equal(outputs, EXPECTED_OUTPUTS_PARALLEL)