def _determine_worker(self):
try:
get_worker()
self.worker = True
self.fs = filesystem(self.protocol, **self.storage_options)
except ValueError:
self.worker = False
self.client = _get_global_client()
self.rfs = dask.delayed(self)
def _determine_worker(self):
try:
get_worker()
self.worker = True
if self.fs is None:
self.fs = filesystem(self.target_protocol,
**(self.target_options or {}))
except ValueError:
self.worker = False
self.client = _get_client(self.client)
self.rfs = dask.delayed(self)
def __init__(self, name=None, client=None, maxsize=0):
try:
self.client = client or Client.current()
except ValueError:
# Initialise new client
self.client = get_worker().client
self.name = name or "variable-" + uuid.uuid4().hex
def get_scheduler(get=None, scheduler=None, collections=None, cls=None):
"""Get scheduler function
There are various ways to specify the scheduler to use:
1. Passing in scheduler= parameters
2. Passing these into global configuration
3. Using defaults of a dask collection
This function centralizes the logic to determine the right scheduler to use
from those many options
"""
if get:
raise TypeError(get_err_msg)
if scheduler is not None:
if callable(scheduler):
return scheduler
elif "Client" in type(scheduler).__name__ and hasattr(
scheduler, "get"):
return scheduler.get
elif scheduler.lower() in named_schedulers:
return named_schedulers[scheduler.lower()]
elif scheduler.lower() in ("dask.distributed", "distributed"):
from distributed.worker import get_client
return get_client().get
else:
raise ValueError("Expected one of [distributed, %s]" %
", ".join(sorted(named_schedulers)))
# else: # try to connect to remote scheduler with this name
# return get_client(scheduler).get
if config.get("scheduler", None):
return get_scheduler(scheduler=config.get("scheduler", None))
if config.get("get", None):
raise ValueError(get_err_msg)
if getattr(thread_state, "key", False):
from distributed.worker import get_worker
return get_worker().client.get
if cls is not None:
return cls.__dask_scheduler__
if collections:
collections = [c for c in collections if c is not None]
if collections:
get = collections[0].__dask_scheduler__
if not all(c.__dask_scheduler__ == get for c in collections):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler=` parameter explicitly in compute or "
"globally with `dask.config.set`.")
return get
return None
def get_scheduler(get=None, scheduler=None, collections=None, cls=None):
""" Get scheduler function
There are various ways to specify the scheduler to use:
1. Passing in scheduler= parameters
2. Passing these into global confiuration
3. Using defaults of a dask collection
This function centralizes the logic to determine the right scheduler to use
from those many options
"""
if get:
raise TypeError(get_err_msg)
if scheduler is not None:
if callable(scheduler):
return scheduler
elif "Client" in type(scheduler).__name__ and hasattr(scheduler, 'get'):
return scheduler.get
elif scheduler.lower() in named_schedulers:
return named_schedulers[scheduler.lower()]
elif scheduler.lower() in ('dask.distributed', 'distributed'):
from distributed.worker import get_client
return get_client().get
elif scheduler.lower() in ['processes', 'multiprocessing']:
raise ValueError("Please install cloudpickle to use the '%s' scheduler." % scheduler)
else:
raise ValueError("Expected one of [distributed, %s]" % ', '.join(sorted(named_schedulers)))
# else: # try to connect to remote scheduler with this name
# return get_client(scheduler).get
if config.get('scheduler', None):
return get_scheduler(scheduler=config.get('scheduler', None))
if config.get('get', None):
raise ValueError(get_err_msg)
if getattr(thread_state, 'key', False):
from distributed.worker import get_worker
return get_worker().client.get
if cls is not None:
return cls.__dask_scheduler__
if collections:
collections = [c for c in collections if c is not None]
if collections:
get = collections[0].__dask_scheduler__
if not all(c.__dask_scheduler__ == get for c in collections):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler=` parameter explicitly in compute or "
"globally with `dask.config.set`.")
return get
return None
def get_scheduler(get=None, scheduler=None, collections=None, cls=None):
""" Get scheduler function
There are various ways to specify the scheduler to use:
1. Passing in get= parameters (deprecated)
2. Passing in scheduler= parameters
3. Passing these into global confiuration
4. Using defaults of a dask collection
This function centralizes the logic to determine the right scheduler to use
from those many options
"""
if get is not None:
if scheduler is not None:
raise ValueError("Both get= and scheduler= provided. Choose one")
warn_on_get(get)
return get
if scheduler is not None:
if scheduler.lower() in named_schedulers:
return named_schedulers[scheduler.lower()]
elif scheduler.lower() in ('dask.distributed', 'distributed'):
from distributed.worker import get_client
return get_client().get
else:
raise ValueError("Expected one of [distributed, %s]" %
', '.join(sorted(named_schedulers)))
# else: # try to connect to remote scheduler with this name
# return get_client(scheduler).get
if config.get('scheduler', None):
return get_scheduler(scheduler=config.get('scheduler', None))
if config.get('get', None):
warn_on_get(config.get('get', None))
return config.get('get', None)
if getattr(thread_state, 'key', False):
from distributed.worker import get_worker
return get_worker().client.get
if cls is not None:
return cls.__dask_scheduler__
if collections:
collections = [c for c in collections if c is not None]
if collections:
get = collections[0].__dask_scheduler__
if not all(c.__dask_scheduler__ == get for c in collections):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler=` parameter explicitly in compute or "
"globally with `set_options`.")
return get
return None
def __init__(self, name=None, client=None):
try:
self.client = client or Client.current()
except ValueError:
# Initialise new client
self.client = get_worker().client
self.name = name or "lock-" + uuid.uuid4().hex
self.id = uuid.uuid4().hex
self._locked = False
def get_scheduler(get=None, scheduler=None, collections=None, cls=None):
""" Get scheduler function
There are various ways to specify the scheduler to use:
1. Passing in get= parameters (deprecated)
2. Passing in scheduler= parameters
3. Passing these into global confiuration
4. Using defaults of a dask collection
This function centralizes the logic to determine the right scheduler to use
from those many options
"""
if get is not None:
if scheduler is not None:
raise ValueError("Both get= and scheduler= provided. Choose one")
warn_on_get(get)
return get
if scheduler is not None:
if scheduler.lower() in named_schedulers:
return named_schedulers[scheduler.lower()]
elif scheduler.lower() in ('dask.distributed', 'distributed'):
from distributed.worker import get_client
return get_client().get
else:
raise ValueError("Expected one of [distributed, %s]" % ', '.join(sorted(named_schedulers)))
# else: # try to connect to remote scheduler with this name
# return get_client(scheduler).get
if config.get('scheduler', None):
return get_scheduler(scheduler=config.get('scheduler', None))
if config.get('get', None):
warn_on_get(config.get('get', None))
return config.get('get', None)
if getattr(thread_state, 'key', False):
from distributed.worker import get_worker
return get_worker().client.get
if cls is not None:
return cls.__dask_scheduler__
if collections:
collections = [c for c in collections if c is not None]
if collections:
get = collections[0].__dask_scheduler__
if not all(c.__dask_scheduler__ == get for c in collections):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler=` parameter explicitly in compute or "
"globally with `set_options`.")
return get
return None
def worker_client(timeout=None, separate_thread=True):
"""Get client for this thread
This context manager is intended to be called within functions that we run
on workers. When run as a context manager it delivers a client
``Client`` object that can submit other tasks directly from that worker.
Parameters
----------
timeout : Number or String
Timeout after which to error out. Defaults to the
``distributed.comm.timeouts.connect`` configuration value.
separate_thread : bool, optional
Whether to run this function outside of the normal thread pool
defaults to True
Examples
--------
>>> def func(x):
... with worker_client(timeout="10s") as c: # connect from worker back to scheduler
... a = c.submit(inc, x) # this task can submit more tasks
... b = c.submit(dec, x)
... result = c.gather([a, b]) # and gather results
... return result
>>> future = client.submit(func, 1) # submit func(1) on cluster
See Also
--------
get_worker
get_client
secede
"""
if timeout is None:
timeout = dask.config.get("distributed.comm.timeouts.connect")
timeout = dask.utils.parse_timedelta(timeout, "s")
worker = get_worker()
client = get_client(timeout=timeout)
if separate_thread:
duration = time() - thread_state.start_time
secede() # have this thread secede from the thread pool
worker.loop.add_callback(
worker.transition,
worker.tasks[thread_state.key],
"long-running",
stimulus_id=f"worker-client-secede-{time()}",
compute_duration=duration,
)
yield client
if separate_thread:
rejoin()
def f(x, sem, kill_address):
with sem:
from distributed.worker import get_worker
worker = get_worker()
if worker.address == kill_address:
import os
os.kill(os.getpid(), 15)
return x
async def f():
"""Trigger a memory_monitor() and reset memory_limit"""
w = get_worker()
# Set a host memory limit that triggers spilling to disk
w.memory_pause_fraction = False
memory = w.monitor.proc.memory_info().rss
w.memory_limit = memory - 10 ** 8
w.memory_target_fraction = 1
await w.memory_monitor()
# Check that host memory are freed
assert w.monitor.proc.memory_info().rss < memory - 10 ** 7
w.memory_limit = memory * 10 # Un-limit
def __init__(self, name=None, client=None, maxsize=0):
try:
self.client = client or Client.current()
except ValueError:
# Initialise new client
self.client = get_worker().client
self.name = name or "queue-" + uuid.uuid4().hex
self.maxsize = maxsize
if self.client.asynchronous:
self._started = asyncio.ensure_future(self._start())
else:
self.client.sync(self._start)
def __init__(
self,
max_leases=1,
name=None,
register=True,
scheduler_rpc=None,
loop=None,
):
try:
worker = get_worker()
self.scheduler = scheduler_rpc or worker.scheduler
self.loop = loop or worker.loop
except ValueError:
client = get_client()
self.scheduler = scheduler_rpc or client.scheduler
self.loop = loop or client.io_loop
self.name = name or "semaphore-" + uuid.uuid4().hex
self.max_leases = max_leases
self.id = uuid.uuid4().hex
self._leases = deque()
self.refresh_leases = True
self._registered = None
if register:
self._registered = self.register()
# this should give ample time to refresh without introducing another
# config parameter since this *must* be smaller than the timeout anyhow
refresh_leases_interval = (parse_timedelta(
dask.config.get("distributed.scheduler.locks.lease-timeout"),
default="s",
) / 5)
pc = PeriodicCallback(self._refresh_leases,
callback_time=refresh_leases_interval * 1000)
self.refresh_callback = pc
# Need to start the callback using IOLoop.add_callback to ensure that the
# PC uses the correct event loop.
self.loop.add_callback(pc.start)
def __init__(self, cls, address, key, worker=None):
super().__init__(key)
self._cls = cls
self._address = address
self._future = None
if worker:
self._worker = worker
self._client = None
else:
try:
# TODO: `get_worker` may return the wrong worker instance for async local clusters (most tests)
# when run outside of a task (when deserializing a key pointing to an Actor, etc.)
self._worker = get_worker()
except ValueError:
self._worker = None
try:
self._client = get_client()
self._future = Future(key, inform=self._worker is None)
# ^ When running on a worker, only hold a weak reference to the key, otherwise the key could become unreleasable.
except ValueError:
self._client = None
def persist(*args, **kwargs):
""" Persist multiple Dask collections into memory
This turns lazy Dask collections into Dask collections with the same
metadata, but now with their results fully computed or actively computing
in the background.
For example a lazy dask.array built up from many lazy calls will now be a
dask.array of the same shape, dtype, chunks, etc., but now with all of
those previously lazy tasks either computed in memory as many small NumPy
arrays (in the single-machine case) or asynchronously running in the
background on a cluster (in the distributed case).
This function operates differently if a ``dask.distributed.Client`` exists
and is connected to a distributed scheduler. In this case this function
will return as soon as the task graph has been submitted to the cluster,
but before the computations have completed. Computations will continue
asynchronously in the background. When using this function with the single
machine scheduler it blocks until the computations have finished.
When using Dask on a single machine you should ensure that the dataset fits
entirely within memory.
Examples
--------
>>> df = dd.read_csv('/path/to/*.csv') # doctest: +SKIP
>>> df = df[df.name == 'Alice'] # doctest: +SKIP
>>> df['in-debt'] = df.balance < 0 # doctest: +SKIP
>>> df = df.persist() # triggers computation # doctest: +SKIP
>>> df.value().min() # future computations are now fast # doctest: +SKIP
-10
>>> df.value().max() # doctest: +SKIP
100
>>> from dask import persist # use persist function on multiple collections
>>> a, b = persist(a, b) # doctest: +SKIP
Parameters
----------
*args: Dask collections
get : callable, optional
A scheduler ``get`` function to use. If not provided, the default
is to check the global settings first, and then fall back to
the collection defaults.
optimize_graph : bool, optional
If True [default], the graph is optimized before computation.
Otherwise the graph is run as is. This can be useful for debugging.
**kwargs
Extra keywords to forward to the scheduler ``get`` function.
Returns
-------
New dask collections backed by in-memory data
"""
collections = [a for a in args if is_dask_collection(a)]
if not collections:
return args
get = kwargs.pop('get', None) or _globals['get']
if get is None and getattr(thread_state, 'key', False):
from distributed.worker import get_worker
get = get_worker().client.get
if inspect.ismethod(get):
try:
from distributed.client import default_client
except ImportError:
pass
else:
try:
client = default_client()
except ValueError:
pass
else:
if client.get == _globals['get']:
collections = client.persist(collections, **kwargs)
if isinstance(collections,
list): # distributed is inconsistent here
collections = tuple(collections)
else:
collections = (collections, )
results_iter = iter(collections)
return tuple(
a if not is_dask_collection(a) else next(results_iter)
for a in args)
optimize_graph = kwargs.pop('optimize_graph', True)
if not get:
get = collections[0].__dask_scheduler__
if not all(a.__dask_scheduler__ == get for a in collections):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler `get` function using either "
"the `get` kwarg or globally with `set_options`.")
dsk = collections_to_dsk(collections, optimize_graph, **kwargs)
keys, postpersists = [], []
for a in args:
if is_dask_collection(a):
a_keys = list(flatten(a.__dask_keys__()))
rebuild, state = a.__dask_postpersist__()
keys.extend(a_keys)
postpersists.append((rebuild, a_keys, state))
else:
postpersists.append((None, None, a))
results = get(dsk, keys, **kwargs)
d = dict(zip(keys, results))
return tuple(s if r is None else r({k: d[k]
for k in ks}, *s)
for r, ks, s in postpersists)
def compute(*args, **kwargs):
"""Compute several dask collections at once.
Parameters
----------
args : object
Any number of objects. If it is a dask object, it's computed and the
result is returned. By default, python builtin collections are also
traversed to look for dask objects (for more information see the
``traverse`` keyword). Non-dask arguments are passed through unchanged.
traverse : bool, optional
By default dask traverses builtin python collections looking for dask
objects passed to ``compute``. For large collections this can be
expensive. If none of the arguments contain any dask objects, set
``traverse=False`` to avoid doing this traversal.
get : callable, optional
A scheduler ``get`` function to use. If not provided, the default is
to check the global settings first, and then fall back to defaults for
the collections.
optimize_graph : bool, optional
If True [default], the optimizations for each collection are applied
before computation. Otherwise the graph is run as is. This can be
useful for debugging.
kwargs
Extra keywords to forward to the scheduler ``get`` function.
Examples
--------
>>> import dask.array as da
>>> a = da.arange(10, chunks=2).sum()
>>> b = da.arange(10, chunks=2).mean()
>>> compute(a, b)
(45, 4.5)
By default, dask objects inside python collections will also be computed:
>>> compute({'a': a, 'b': b, 'c': 1}) # doctest: +SKIP
({'a': 45, 'b': 4.5, 'c': 1},)
"""
from dask.delayed import delayed
traverse = kwargs.pop('traverse', True)
if traverse:
args = tuple(
delayed(a) if isinstance(a, (list, set, tuple, dict,
Iterator)) else a for a in args)
optimize_graph = kwargs.pop('optimize_graph', True)
variables = [a for a in args if is_dask_collection(a)]
if not variables:
return args
get = kwargs.pop('get', None) or _globals['get']
if get is None and getattr(thread_state, 'key', False):
from distributed.worker import get_worker
get = get_worker().client.get
if not get:
get = variables[0].__dask_scheduler__
if not all(a.__dask_scheduler__ == get for a in variables):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler `get` function using either "
"the `get` kwarg or globally with `set_options`.")
dsk = collections_to_dsk(variables, optimize_graph, **kwargs)
keys = [var.__dask_keys__() for var in variables]
postcomputes = [
a.__dask_postcompute__() if is_dask_collection(a) else (None, a)
for a in args
]
results = get(dsk, keys, **kwargs)
results_iter = iter(results)
return tuple(a if f is None else f(next(results_iter), *a)
for f, a in postcomputes)
def persist(*args, **kwargs):
""" Persist multiple Dask collections into memory
This turns lazy Dask collections into Dask collections with the same
metadata, but now with their results fully computed or actively computing
in the background.
For example a lazy dask.array built up from many lazy calls will now be a
dask.array of the same shape, dtype, chunks, etc., but now with all of
those previously lazy tasks either computed in memory as many small :class:`numpy.array`
(in the single-machine case) or asynchronously running in the
background on a cluster (in the distributed case).
This function operates differently if a ``dask.distributed.Client`` exists
and is connected to a distributed scheduler. In this case this function
will return as soon as the task graph has been submitted to the cluster,
but before the computations have completed. Computations will continue
asynchronously in the background. When using this function with the single
machine scheduler it blocks until the computations have finished.
When using Dask on a single machine you should ensure that the dataset fits
entirely within memory.
Examples
--------
>>> df = dd.read_csv('/path/to/*.csv') # doctest: +SKIP
>>> df = df[df.name == 'Alice'] # doctest: +SKIP
>>> df['in-debt'] = df.balance < 0 # doctest: +SKIP
>>> df = df.persist() # triggers computation # doctest: +SKIP
>>> df.value().min() # future computations are now fast # doctest: +SKIP
-10
>>> df.value().max() # doctest: +SKIP
100
>>> from dask import persist # use persist function on multiple collections
>>> a, b = persist(a, b) # doctest: +SKIP
Parameters
----------
*args: Dask collections
get : callable, optional
A scheduler ``get`` function to use. If not provided, the default
is to check the global settings first, and then fall back to
the collection defaults.
optimize_graph : bool, optional
If True [default], the graph is optimized before computation.
Otherwise the graph is run as is. This can be useful for debugging.
**kwargs
Extra keywords to forward to the scheduler ``get`` function.
Returns
-------
New dask collections backed by in-memory data
"""
collections = [a for a in args if is_dask_collection(a)]
if not collections:
return args
get = kwargs.pop('get', None) or _globals['get']
if get is None and getattr(thread_state, 'key', False):
from distributed.worker import get_worker
get = get_worker().client.get
if inspect.ismethod(get):
try:
from distributed.client import default_client
except ImportError:
pass
else:
try:
client = default_client()
except ValueError:
pass
else:
if client.get == _globals['get']:
collections = client.persist(collections, **kwargs)
if isinstance(collections, list): # distributed is inconsistent here
collections = tuple(collections)
else:
collections = (collections,)
results_iter = iter(collections)
return tuple(a if not is_dask_collection(a)
else next(results_iter)
for a in args)
optimize_graph = kwargs.pop('optimize_graph', True)
if not get:
get = collections[0].__dask_scheduler__
if not all(a.__dask_scheduler__ == get for a in collections):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler `get` function using either "
"the `get` kwarg or globally with `set_options`.")
dsk = collections_to_dsk(collections, optimize_graph, **kwargs)
keys, postpersists = [], []
for a in args:
if is_dask_collection(a):
a_keys = list(flatten(a.__dask_keys__()))
rebuild, state = a.__dask_postpersist__()
keys.extend(a_keys)
postpersists.append((rebuild, a_keys, state))
else:
postpersists.append((None, None, a))
results = get(dsk, keys, **kwargs)
d = dict(zip(keys, results))
return tuple(s if r is None else r({k: d[k] for k in ks}, *s)
for r, ks, s in postpersists)
def compute(*args, **kwargs):
"""Compute several dask collections at once.
Parameters
----------
args : object
Any number of objects. If it is a dask object, it's computed and the
result is returned. By default, python builtin collections are also
traversed to look for dask objects (for more information see the
``traverse`` keyword). Non-dask arguments are passed through unchanged.
traverse : bool, optional
By default dask traverses builtin python collections looking for dask
objects passed to ``compute``. For large collections this can be
expensive. If none of the arguments contain any dask objects, set
``traverse=False`` to avoid doing this traversal.
get : callable, optional
A scheduler ``get`` function to use. If not provided, the default is
to check the global settings first, and then fall back to defaults for
the collections.
optimize_graph : bool, optional
If True [default], the optimizations for each collection are applied
before computation. Otherwise the graph is run as is. This can be
useful for debugging.
kwargs
Extra keywords to forward to the scheduler ``get`` function.
Examples
--------
>>> import dask.array as da
>>> a = da.arange(10, chunks=2).sum()
>>> b = da.arange(10, chunks=2).mean()
>>> compute(a, b)
(45, 4.5)
By default, dask objects inside python collections will also be computed:
>>> compute({'a': a, 'b': b, 'c': 1}) # doctest: +SKIP
({'a': 45, 'b': 4.5, 'c': 1},)
"""
from dask.delayed import delayed
traverse = kwargs.pop('traverse', True)
if traverse:
args = tuple(delayed(a)
if isinstance(a, (list, set, tuple, dict, Iterator))
else a for a in args)
optimize_graph = kwargs.pop('optimize_graph', True)
variables = [a for a in args if is_dask_collection(a)]
if not variables:
return args
get = kwargs.pop('get', None) or _globals['get']
if get is None and getattr(thread_state, 'key', False):
from distributed.worker import get_worker
get = get_worker().client.get
if not get:
get = variables[0].__dask_scheduler__
if not all(a.__dask_scheduler__ == get for a in variables):
raise ValueError("Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler `get` function using either "
"the `get` kwarg or globally with `set_options`.")
dsk = collections_to_dsk(variables, optimize_graph, **kwargs)
keys = [var.__dask_keys__() for var in variables]
postcomputes = [a.__dask_postcompute__() if is_dask_collection(a)
else (None, a) for a in args]
results = get(dsk, keys, **kwargs)
results_iter = iter(results)
return tuple(a if f is None else f(next(results_iter), *a)
for f, a in postcomputes)
def get_scheduler(get=None, scheduler=None, collections=None, cls=None):
"""Get scheduler function
There are various ways to specify the scheduler to use:
1. Passing in scheduler= parameters
2. Passing these into global configuration
3. Using defaults of a dask collection
This function centralizes the logic to determine the right scheduler to use
from those many options
"""
if get:
raise TypeError(get_err_msg)
if scheduler is not None:
if callable(scheduler):
return scheduler
elif "Client" in type(scheduler).__name__ and hasattr(scheduler, "get"):
return scheduler.get
elif isinstance(scheduler, str):
scheduler = scheduler.lower()
if scheduler in named_schedulers:
if config.get("scheduler", None) in ("dask.distributed", "distributed"):
warnings.warn(
"Running on a single-machine scheduler when a distributed client "
"is active might lead to unexpected results."
)
return named_schedulers[scheduler]
elif scheduler in ("dask.distributed", "distributed"):
from distributed.worker import get_client
return get_client().get
else:
raise ValueError(
"Expected one of [distributed, %s]"
% ", ".join(sorted(named_schedulers))
)
elif isinstance(scheduler, Executor):
# Get `num_workers` from `Executor`'s `_max_workers` attribute.
# If undefined, fallback to `config` or worst case CPU_COUNT.
num_workers = getattr(scheduler, "_max_workers", None)
if num_workers is None:
num_workers = config.get("num_workers", CPU_COUNT)
assert isinstance(num_workers, Integral) and num_workers > 0
return partial(local.get_async, scheduler.submit, num_workers)
else:
raise ValueError("Unexpected scheduler: %s" % repr(scheduler))
# else: # try to connect to remote scheduler with this name
# return get_client(scheduler).get
if config.get("scheduler", None):
return get_scheduler(scheduler=config.get("scheduler", None))
if config.get("get", None):
raise ValueError(get_err_msg)
if getattr(thread_state, "key", False):
from distributed.worker import get_worker
return get_worker().client.get
if cls is not None:
return cls.__dask_scheduler__
if collections:
collections = [c for c in collections if c is not None]
if collections:
get = collections[0].__dask_scheduler__
if not all(c.__dask_scheduler__ == get for c in collections):
raise ValueError(
"Compute called on multiple collections with "
"differing default schedulers. Please specify a "
"scheduler=` parameter explicitly in compute or "
"globally with `dask.config.set`."
)
return get
return None
def _get_current_task_state() -> Optional[TaskState]:
worker = get_worker()
logger.debug("current worker %s", f"{worker=}")
current_task = worker.get_current_task()
logger.debug("current task %s", f"{current_task=}")
return worker.tasks.get(current_task)
def from_dask_worker(cls, log: str) -> "TaskLogEvent":
return cls(job_id=get_worker().get_current_task(), log=log)
def from_dask_worker(cls, progress: float) -> "TaskProgressEvent":
return cls(job_id=get_worker().get_current_task(), progress=progress)
def from_dask_worker(
cls, state: RunningState, msg: Optional[str] = None
) -> "TaskStateEvent":
return cls(job_id=get_worker().get_current_task(), state=state, msg=msg)
