def test_multiple_clients(local_cluster_url, default_raw): ex1 = DaskJobExecutor.connect(local_cluster_url) # this creates a second Client, and even though we are setting `set_as_default=False`, # this Client is then used by functions like `dd.as_completed`. That is because # `set_as_default` only sets the dask scheduler config to "dask.distributed", it does # not affect setting the _client_ as the global default `Client`! # so any time `as_completed` is called, the `loop` needs to be set correctly, otherwise # this may result in strange hangs and crashes DaskJobExecutor.connect(local_cluster_url) udf = SumUDF() cx1 = Context(executor=ex1) cx1.run_udf(dataset=default_raw, udf=udf)
def dist_ctx(scheduler_addr): """ This Context needs to have an external dask cluster running, with the following assumptions: - two workers: hostnames worker-1 and worker-2 - one scheduler node - data availability TBD """ executor = DaskJobExecutor.connect(scheduler_addr) with lt.Context(executor=executor) as ctx: yield ctx
def dist_ctx(): """ This Context needs to have an external dask cluster running, with the following assumptions: - two workers: hostnames worker-1 and worker-2 - one scheduler node - data availability TBD - the address of the dask scheduler is passed in as DASK_SCHEDULER_ADDRESS """ scheduler_addr = os.environ['DASK_SCHEDULER_ADDRESS'] executor = DaskJobExecutor.connect(scheduler_addr) with lt.Context(executor=executor) as ctx: yield ctx
def test_connect_default(local_cluster_url): try: executor = DaskJobExecutor.connect( local_cluster_url, client_kwargs={'set_as_default': True}) ctx = Context(executor=executor) # This queries Dask which scheduler it is using ctx2 = Context.make_with("dask-integration") # make sure the second uses the Client of the first assert ctx2.executor.client is ctx.executor.client finally: # Only close the Client, keep the cluster running # since that is test infrastructure executor.client.close() ctx.close()
def main(): with DaskJobExecutor.connect('tcp://localhost:8786') as executor: ctx = api.Context(executor=executor) ds = ctx.load("RAW", path=r"C:\Users\Dieter\testfile-32-32-32-32-float32.raw", nav_shape=(32, 32), sig_shape=(32, 32), dtype=np.float32) sum_analysis = ctx.create_sum_analysis(dataset=ds) sum_result = ctx.run(sum_analysis) sum_image = DM.CreateImage(sum_result.intensity.raw_data.copy()) sum_image.ShowImage() haadf_analysis = ctx.create_ring_analysis(dataset=ds) haadf_result = ctx.run(haadf_analysis) haadf_image = DM.CreateImage(haadf_result.intensity.raw_data.copy()) haadf_image.ShowImage()
def dask_executor(local_cluster_url): executor = DaskJobExecutor.connect(local_cluster_url) yield executor executor.close()