def dask_dumps(x, context=None):
"""Serialize object using the class-based registry"""
type_name = typename(type(x))
try:
dumps = dask_serialize.dispatch(type(x))
except TypeError:
raise NotImplementedError(type_name)
if has_keyword(dumps, "context"):
sub_header, frames = dumps(x, context=context)
else:
sub_header, frames = dumps(x)
header = {
"sub-header": sub_header,
"type": type_name,
"type-serialized": pickle.dumps(type(x), protocol=4),
"serializer": "dask",
}
return header, frames
def get_handlers(server, modules: list[str], prefix="/"):
prefix = prefix or ""
prefix = "/" + prefix.strip("/")
if not prefix.endswith("/"):
prefix = prefix + "/"
_routes = []
for module_name in modules:
module = importlib.import_module(module_name)
_routes.extend(module.routes) # type: ignore
routes = []
for url, cls, kwargs in _routes:
if has_keyword(cls.initialize, "dask_server"):
kwargs = toolz.assoc(kwargs, "dask_server", server)
routes.append((prefix + url.lstrip("/"), cls, kwargs))
return routes
def has_context_keyword(meth):
if isinstance(meth, MethodType):
return has_keyword(meth.__func__, 'context')
else:
return has_keyword(meth, 'context')
def call_with_context(meth, x, *args):
if has_keyword(meth, 'context'):
return meth(x, context=context)
else:
return meth(x)
async def handle_comm(self, comm):
"""Dispatch new communications to coroutine-handlers
Handlers is a dictionary mapping operation names to functions or
coroutines.
{'get_data': get_data,
'ping': pingpong}
Coroutines should expect a single Comm object.
"""
if self.__stopped:
comm.abort()
return
address = comm.peer_address
op = None
logger.debug("Connection from %r to %s", address, type(self).__name__)
self._comms[comm] = op
await self
try:
while True:
try:
msg = await comm.read()
logger.debug("Message from %r: %s", address, msg)
except OSError as e:
if not sys.is_finalizing():
logger.debug(
"Lost connection to %r while reading message: %s."
" Last operation: %s",
address,
e,
op,
)
break
except Exception as e:
logger.exception("Exception while reading from %s", address)
if comm.closed():
raise
else:
await comm.write(error_message(e, status="uncaught-error"))
continue
if not isinstance(msg, dict):
raise TypeError(
"Bad message type. Expected dict, got\n " + str(msg)
)
try:
op = msg.pop("op")
except KeyError as e:
raise ValueError(
"Received unexpected message without 'op' key: " + str(msg)
) from e
if self.counters is not None:
self.counters["op"].add(op)
self._comms[comm] = op
serializers = msg.pop("serializers", None)
close_desired = msg.pop("close", False)
reply = msg.pop("reply", True)
if op == "close":
if reply:
await comm.write("OK")
break
result = None
try:
if op in self.blocked_handlers:
_msg = (
"The '{op}' handler has been explicitly disallowed "
"in {obj}, possibly due to security concerns."
)
exc = ValueError(_msg.format(op=op, obj=type(self).__name__))
handler = raise_later(exc)
else:
handler = self.handlers[op]
except KeyError:
logger.warning(
"No handler %s found in %s",
op,
type(self).__name__,
exc_info=True,
)
else:
if serializers is not None and has_keyword(handler, "serializers"):
msg["serializers"] = serializers # add back in
logger.debug("Calling into handler %s", handler.__name__)
try:
if _expects_comm(handler):
result = handler(comm, **msg)
else:
result = handler(**msg)
if inspect.isawaitable(result):
result = asyncio.ensure_future(result)
self._ongoing_coroutines.add(result)
result = await result
except (CommClosedError, asyncio.CancelledError):
if self.status in Status.ANY_RUNNING:
logger.info("Lost connection to %r", address, exc_info=True)
break
except Exception as e:
logger.exception("Exception while handling op %s", op)
if comm.closed():
raise
else:
result = error_message(e, status="uncaught-error")
if reply and result != Status.dont_reply:
try:
await comm.write(result, serializers=serializers)
except (OSError, TypeError) as e:
logger.debug(
"Lost connection to %r while sending result for op %r: %s",
address,
op,
e,
)
break
self._comms[comm] = None
msg = result = None
if close_desired:
await comm.close()
if comm.closed():
break
finally:
del self._comms[comm]
if not sys.is_finalizing() and not comm.closed():
try:
comm.abort()
except Exception as e:
logger.error(
"Failed while closing connection to %r: %s", address, e
)
def register_serialization_family(name, dumps, loads):
families[name] = (dumps, loads, dumps and has_keyword(dumps, "context"))