def test_multiple_clients(local_cluster_url, default_raw):
ex1 = DaskJobExecutor.connect(local_cluster_url)
# this creates a second Client, and even though we are setting `set_as_default=False`,
# this Client is then used by functions like `dd.as_completed`. That is because
# `set_as_default` only sets the dask scheduler config to "dask.distributed", it does
# not affect setting the _client_ as the global default `Client`!
# so any time `as_completed` is called, the `loop` needs to be set correctly, otherwise
# this may result in strange hangs and crashes
DaskJobExecutor.connect(local_cluster_url)
udf = SumUDF()
cx1 = Context(executor=ex1)
cx1.run_udf(dataset=default_raw, udf=udf)
def dist_ctx(scheduler_addr):
"""
This Context needs to have an external dask cluster running, with the following
assumptions:
- two workers: hostnames worker-1 and worker-2
- one scheduler node
- data availability TBD
"""
executor = DaskJobExecutor.connect(scheduler_addr)
with lt.Context(executor=executor) as ctx:
yield ctx
def dist_ctx():
"""
This Context needs to have an external dask cluster running, with the following
assumptions:
- two workers: hostnames worker-1 and worker-2
- one scheduler node
- data availability TBD
- the address of the dask scheduler is passed in as DASK_SCHEDULER_ADDRESS
"""
scheduler_addr = os.environ['DASK_SCHEDULER_ADDRESS']
executor = DaskJobExecutor.connect(scheduler_addr)
with lt.Context(executor=executor) as ctx:
yield ctx
def test_connect_default(local_cluster_url):
try:
executor = DaskJobExecutor.connect(
local_cluster_url, client_kwargs={'set_as_default': True})
ctx = Context(executor=executor)
# This queries Dask which scheduler it is using
ctx2 = Context.make_with("dask-integration")
# make sure the second uses the Client of the first
assert ctx2.executor.client is ctx.executor.client
finally:
# Only close the Client, keep the cluster running
# since that is test infrastructure
executor.client.close()
ctx.close()
def main():
with DaskJobExecutor.connect('tcp://localhost:8786') as executor:
ctx = api.Context(executor=executor)
ds = ctx.load("RAW",
path=r"C:\Users\Dieter\testfile-32-32-32-32-float32.raw",
nav_shape=(32, 32),
sig_shape=(32, 32),
dtype=np.float32)
sum_analysis = ctx.create_sum_analysis(dataset=ds)
sum_result = ctx.run(sum_analysis)
sum_image = DM.CreateImage(sum_result.intensity.raw_data.copy())
sum_image.ShowImage()
haadf_analysis = ctx.create_ring_analysis(dataset=ds)
haadf_result = ctx.run(haadf_analysis)
haadf_image = DM.CreateImage(haadf_result.intensity.raw_data.copy())
haadf_image.ShowImage()
def dask_executor(local_cluster_url):
executor = DaskJobExecutor.connect(local_cluster_url)
yield executor
executor.close()