def test_no_undesired_distributed_cache_hit(loop): # Dask has a pickle cache for callables that are called many times. Because # the dask backends used to wrapp both the functions and the arguments # under instances of the Batch callable class this caching mechanism could # lead to bugs as described in: https://github.com/joblib/joblib/pull/1055 # The joblib-dask backend has been refactored to avoid bundling the # arguments as an attribute of the Batch instance to avoid this problem. # This test serves as non-regression problem. # Use a large number of input arguments to give the AutoBatchingMixin # enough tasks to kick-in. lists = [[] for _ in range(100)] np = pytest.importorskip('numpy') X = np.arange(int(1e6)) def isolated_operation(list_, data=None): if data is not None: np.testing.assert_array_equal(data, X) list_.append(uuid4().hex) return list_ cluster = LocalCluster(n_workers=1, threads_per_worker=2) client = Client(cluster) try: with parallel_backend('dask') as (ba, _): # dispatches joblib.parallel.BatchedCalls res = Parallel()(delayed(isolated_operation)(list_) for list_ in lists) # The original arguments should not have been mutated as the mutation # happens in the dask worker process. assert lists == [[] for _ in range(100)] # Here we did not pass any large numpy array as argument to # isolated_operation so no scattering event should happen under the # hood. counts = count_events('receive-from-scatter', client) assert sum(counts.values()) == 0 assert all([len(r) == 1 for r in res]) with parallel_backend('dask') as (ba, _): # Append a large array which will be scattered by dask, and # dispatch joblib._dask.Batch res = Parallel()(delayed(isolated_operation)(list_, data=X) for list_ in lists) # This time, auto-scattering should have kicked it. counts = count_events('receive-from-scatter', client) assert sum(counts.values()) > 0 assert all([len(r) == 1 for r in res]) finally: client.close() cluster.close()
def test_wait_for_workers_timeout(): # Start a cluster with 0 worker: cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2) client = Client(cluster) try: with parallel_backend('dask', wait_for_workers_timeout=0.1): # Short timeout: DaskDistributedBackend msg = "DaskDistributedBackend has no worker after 0.1 seconds." with pytest.raises(TimeoutError, match=msg): Parallel()(delayed(inc)(i) for i in range(10)) with parallel_backend('dask', wait_for_workers_timeout=0): # No timeout: fallback to generic joblib failure: msg = "DaskDistributedBackend has no active worker" with pytest.raises(RuntimeError, match=msg): Parallel()(delayed(inc)(i) for i in range(10)) finally: client.close() cluster.close()
def test_wait_for_workers(cluster_strategy): cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2) client = Client(cluster) if cluster_strategy == "adaptive": cluster.adapt(minimum=0, maximum=2) elif cluster_strategy == "late_scaling": # Tell the cluster to start workers but this is a non-blocking call # and new workers might take time to connect. In this case the Parallel # call should wait for at least one worker to come up before starting # to schedule work. cluster.scale(2) try: with parallel_backend('dask'): # The following should wait a bit for at least one worker to # become available. Parallel()(delayed(inc)(i) for i in range(10)) finally: client.close() cluster.close()
def test_wait_for_workers_timeout(): # Start a cluster with 0 worker: cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2) client = Client(cluster) try: with parallel_backend('dask', wait_for_workers_timeout=0.1): # Short timeout: DaskDistributedBackend msg = "DaskDistributedBackend has no worker after 0.1 seconds." with pytest.raises(TimeoutError, match=msg): Parallel()(delayed(inc)(i) for i in range(10)) with parallel_backend('dask', wait_for_workers_timeout=0): # No timeout: fallback to generic joblib failure: msg = "DaskDistributedBackend has no active worker" with pytest.raises(RuntimeError, match=msg): Parallel()(delayed(inc)(i) for i in range(10)) finally: client.close() cluster.close()
def test_wait_for_workers(cluster_strategy): cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2) client = Client(cluster) if cluster_strategy == "adaptive": cluster.adapt(minimum=0, maximum=2) elif cluster_strategy == "late_scaling": # Tell the cluster to start workers but this is a non-blocking call # and new workers might take time to connect. In this case the Parallel # call should wait for at least one worker to come up before starting # to schedule work. cluster.scale(2) try: with parallel_backend('dask'): # The following should wait a bit for at least one worker to # become available. Parallel()(delayed(inc)(i) for i in range(10)) finally: client.close() cluster.close()