def ctx(request, dask_executor):
if request.param == 'inline':
yield Context.make_with('inline')
elif request.param == "dask_executor":
yield Context(executor=dask_executor)
elif request.param == "delayed_default":
yield Context(executor=DelayedJobExecutor())
elif request.param == "delayed_dist":
with distributed.Client(n_workers=2,
threads_per_worker=4,
processes=True) as _:
yield Context(executor=DelayedJobExecutor())
elif request.param == "dask_make_default":
try:
ctx = Context.make_with('dask-make-default')
yield ctx
finally:
# cleanup: Close cluster and client
# This is also tested below, here just to make
# sure things behave as expected.
assert isinstance(ctx.executor, DaskJobExecutor)
ctx.executor.is_local = True
ctx.close()
elif request.param == "dask_integration":
with distributed.Client(n_workers=2,
threads_per_worker=4,
processes=False) as _:
yield Context.make_with("dask-integration")
elif request.param == "concurrent":
yield Context.make_with("threads")
elif request.param == "delayed":
yield Context(executor=DelayedJobExecutor())
def test_make_default():
try:
ctx = Context.make_with("dask-make-default")
# This queries Dask which scheduler it is using
ctx2 = Context.make_with("dask-integration")
# make sure the second uses the Client of the first
assert ctx2.executor.client is ctx.executor.client
finally:
# dask-make-default starts a Client that will persist
# and not be closed automatically. We have to make sure
# to close everything ourselves
if ctx.executor.client.cluster is not None:
ctx.executor.client.cluster.close(timeout=30)
ctx.executor.client.close()
ctx.close()
def test_no_dangling_client():
# Within the whole test suite and LiberTEM we should not have
# a dangling dask.distributed Client set as default Dask scheduler.
# That means we confirm that we get a ConcurrentJobExecutor in the
# default case.
ctx = Context.make_with("dask-integration")
assert isinstance(ctx.executor, ConcurrentJobExecutor)
def test_use_threads():
with dask.config.set(scheduler="threads"):
ctx = Context.make_with("dask-integration")
assert isinstance(ctx.executor, ConcurrentJobExecutor)
assert isinstance(ctx.executor.client, (
concurrent.futures.ThreadPoolExecutor,
multiprocessing.pool.ThreadPool,
))
def test_use_distributed():
# This Client is pretty cheap to start
# since it only uses threads
with distributed.Client(n_workers=1, threads_per_worker=1,
processes=False) as c:
ctx = Context.make_with("dask-integration")
assert isinstance(ctx.executor, DaskJobExecutor)
assert ctx.executor.client is c
def test_connect_default(local_cluster_url):
try:
executor = DaskJobExecutor.connect(
local_cluster_url, client_kwargs={'set_as_default': True})
ctx = Context(executor=executor)
# This queries Dask which scheduler it is using
ctx2 = Context.make_with("dask-integration")
# make sure the second uses the Client of the first
assert ctx2.executor.client is ctx.executor.client
finally:
# Only close the Client, keep the cluster running
# since that is test infrastructure
executor.client.close()
ctx.close()
def test_make_with_unrecognized():
with pytest.raises(ValueError):
Context.make_with('not_an_executor')
def test_make_with_threads(concurrent_executor):
ctx = Context.make_with('threads')
assert isinstance(ctx.executor, concurrent_executor.__class__)
def test_make_with_inline(inline_executor):
ctx = Context.make_with('inline')
assert isinstance(ctx.executor, inline_executor.__class__)
def test_use_synchronous():
with dask.config.set(scheduler="synchronous"):
ctx = Context.make_with("dask-integration")
assert isinstance(ctx.executor, InlineJobExecutor)