def test_parallel_queued_pipeline_with_step_name_n_worker_max_queue_size(): p = ParallelQueuedFeatureUnion([('1', 1, 5, MultiplyByN(2)), ('2', 1, 5, MultiplyByN(2)), ('3', 1, 5, MultiplyByN(2)), ('4', 1, 5, MultiplyByN(2))], batch_size=10) outputs = p.transform(list(range(100))) assert np.array_equal(outputs, EXPECTED_OUTPUTS_PARALLEL)
def test_parallel_queued_pipeline_with_step_name_n_worker_additional_arguments(): n_workers = 4 worker_arguments = [('hyperparams', HyperparameterSamples({'multiply_by': 2})) for _ in range(n_workers)] p = ParallelQueuedFeatureUnion([ ('1', n_workers, worker_arguments, MultiplyByN()), ], batch_size=10, max_queue_size=5) outputs = p.transform(list(range(100))) expected = np.array(list(range(0, 200, 2))) assert np.array_equal(outputs, expected)
def test_queued_pipeline_saving(tmpdir): # Given p = ParallelQueuedFeatureUnion([ ('1', FitTransformCallbackStep()), ('2', FitTransformCallbackStep()), ('3', FitTransformCallbackStep()), ('4', FitTransformCallbackStep()), ], n_workers_per_step=1, max_queue_size=10, batch_size=10) # When p, outputs = p.fit_transform(list(range(100)), list(range(100))) p.save(ExecutionContext(tmpdir)) p.apply('clear_callbacks') # Then assert len(p[0].wrapped.transform_callback_function.data) == 0 assert len(p[0].wrapped.fit_callback_function.data) == 0 assert len(p[1].wrapped.transform_callback_function.data) == 0 assert len(p[1].wrapped.fit_callback_function.data) == 0 assert len(p[2].wrapped.transform_callback_function.data) == 0 assert len(p[2].wrapped.fit_callback_function.data) == 0 assert len(p[3].wrapped.transform_callback_function.data) == 0 assert len(p[3].wrapped.fit_callback_function.data) == 0 p = p.load(ExecutionContext(tmpdir)) assert len(p[0].wrapped.transform_callback_function.data) == 10 assert len(p[0].wrapped.fit_callback_function.data) == 10 assert len(p[1].wrapped.transform_callback_function.data) == 10 assert len(p[1].wrapped.fit_callback_function.data) == 10 assert len(p[2].wrapped.transform_callback_function.data) == 10 assert len(p[2].wrapped.fit_callback_function.data) == 10 assert len(p[3].wrapped.transform_callback_function.data) == 10 assert len(p[3].wrapped.fit_callback_function.data) == 10
def test_queued_pipeline_with_savers(tmpdir): # Given p = ParallelQueuedFeatureUnion([ ('1', MultiplyByN(2)), ('2', MultiplyByN(2)), ('3', MultiplyByN(2)), ('4', MultiplyByN(2)), ], n_workers_per_step=1, max_queue_size=10, batch_size=10, use_savers=True, cache_folder=tmpdir) # When outputs = p.transform(list(range(100))) # Then assert np.array_equal(outputs, EXPECTED_OUTPUTS_PARALLEL)
def test_queued_pipeline_saving(tmpdir, use_processes, use_savers): # Given p = ParallelQueuedFeatureUnion([ ('1', 4, 10, FitTransformCallbackStep()), ('2', 4, 10, FitTransformCallbackStep()), ('3', 4, 10, FitTransformCallbackStep()), ('4', 4, 10, FitTransformCallbackStep()), ], n_workers_per_step=4, max_queue_size=10, batch_size=10, use_processes=use_processes, use_savers=use_savers).with_context( ExecutionContext(tmpdir)) # When p, _ = p.fit_transform(list(range(200)), list(range(200))) p = p.wrapped # clear execution context wrapper p.save(ExecutionContext(tmpdir)) p.apply('clear_callbacks') # Then assert len(p[0].wrapped.transform_callback_function.data) == 0 assert len(p[0].wrapped.fit_callback_function.data) == 0 assert len(p[1].wrapped.transform_callback_function.data) == 0 assert len(p[1].wrapped.fit_callback_function.data) == 0 assert len(p[2].wrapped.transform_callback_function.data) == 0 assert len(p[2].wrapped.fit_callback_function.data) == 0 assert len(p[3].wrapped.transform_callback_function.data) == 0 assert len(p[3].wrapped.fit_callback_function.data) == 0 p = p.load(ExecutionContext(tmpdir)) assert len(p[0].wrapped.transform_callback_function.data) == 20 assert len(p[0].wrapped.fit_callback_function.data) == 20 assert len(p[1].wrapped.transform_callback_function.data) == 20 assert len(p[1].wrapped.fit_callback_function.data) == 20 assert len(p[2].wrapped.transform_callback_function.data) == 20 assert len(p[2].wrapped.fit_callback_function.data) == 20 assert len(p[3].wrapped.transform_callback_function.data) == 20 assert len(p[3].wrapped.fit_callback_function.data) == 20