def __init__(self, server_address: str, task_id: bytes):
#: Passed on from the constructor
self.task_id = task_id
self._zmq_ctx = util.create_zmq_ctx()
self._server_meta = util.get_server_meta(self._zmq_ctx, server_address)
self._dealer = self._create_dealer()
def __init__(self,
server_address: str,
push_address: str,
secret_key: str = None) -> None:
super().__init__(secret_key)
self.zmq_ctx = util.create_zmq_ctx()
self.router_sock = self.zmq_ctx.socket(zmq.ROUTER)
self.pub_sock = self.zmq_ctx.socket(zmq.PUB)
self.pull_sock = self.zmq_ctx.socket(zmq.PULL)
push_sock = self.zmq_ctx.socket(zmq.PUSH)
push_sock.connect(push_address)
try:
if server_address:
self.req_rep_address = server_address
self.router_sock.bind(self.req_rep_address)
if "ipc" in server_address:
self.pub_sub_address = util.bind_to_random_ipc(
self.pub_sock)
self.push_pull_address = util.bind_to_random_ipc(
self.pull_sock)
else:
self.pub_sub_address = util.bind_to_random_tcp(
self.pub_sock)
self.push_pull_address = util.bind_to_random_tcp(
self.pull_sock)
else:
self.req_rep_address = util.bind_to_random_address(
self.router_sock)
self.pub_sub_address = util.bind_to_random_address(
self.pub_sock)
self.push_pull_address = util.bind_to_random_address(
self.pull_sock)
except Exception:
push_sock.send(self._serializer.dumps(
exceptions.RemoteException()))
self.close()
else:
push_sock.send(self._serializer.dumps(self.req_rep_address))
finally:
push_sock.close()
# see State._get_subscribe_sock() for more
self.pub_sock.setsockopt(zmq.INVERT_MATCHING, 1)
self.state_store = defaultdict(
dict) # type:Dict[bytes, Dict[Any, Any]]
self.dispatch_dict = {
Commands.exec_atomic_fn: self.exec_atomic_fn,
Commands.exec_dict_method: self.exec_dict_method,
Commands.get_state: self.send_state,
Commands.set_state: self.set_state,
Commands.head: self.head,
Commands.ping: self.ping,
}
def ping(server_address: str,
*,
timeout: float = None,
payload: Union[bytes] = None) -> int:
"""
Ping the zproc server.
This can be used to easily detect if a server is alive and running, with the aid of a suitable ``timeout``.
:param server_address:
.. include:: /api/snippets/server_address.rst
:param timeout:
The timeout in seconds.
If this is set to ``None``, then it will block forever, until the zproc server replies.
For all other values, it will wait for a reply,
for that amount of time before returning with a :py:class:`TimeoutError`.
By default it is set to ``None``.
:param payload:
payload that will be sent to the server.
If it is set to None, then ``os.urandom(56)`` (56 random bytes) will be used.
(No real reason for the ``56`` magic number.)
:return:
The zproc server's **pid**.
"""
if payload is None:
payload = os.urandom(56)
with util.create_zmq_ctx() as zmq_ctx:
with zmq_ctx.socket(zmq.DEALER) as dealer_sock:
dealer_sock.connect(server_address)
if timeout is not None:
dealer_sock.setsockopt(zmq.RCVTIMEO, int(timeout * 1000))
dealer_sock.send(
serializer.dumps({
Msgs.cmd: Cmds.ping,
Msgs.info: payload
}))
try:
recv_payload, pid = serializer.loads(dealer_sock.recv())
except zmq.error.Again:
raise TimeoutError(
"Timed-out waiting while for the ZProc server to respond.")
assert (
recv_payload == payload
), "Payload doesn't match! The server connection may be compromised, or unstable."
return pid
def start_server(server_address: str = None,
*,
backend: Callable = multiprocessing.Process,
secret_key: str = None):
"""
Start a new zproc server.
:param server_address:
The zproc server's address.
If it is set to ``None``, then a random address will be generated.
Please read :ref:`server-address-spec` for a detailed explanation.
:param backend:
The backend to use for launching the server process.
For example, you may use ``threading.Thread`` as the backend.
.. warning::
Not guaranteed to work well with anything other than ``multiprocessing.Process``.
:return: ``tuple``, containing a ``multiprocessing.Process`` object for server and the server address.
"""
zmq_ctx = util.create_zmq_ctx()
sock = zmq_ctx.socket(zmq.PULL)
pull_address = util.bind_to_random_address(sock)
serializer = util.get_serializer(secret_key)
server_process = backend(
target=lambda *args, **kwargs: Server(*args, **kwargs).main(),
args=[server_address, pull_address, secret_key],
daemon=True,
)
server_process.start()
try:
server_address = util.recv(sock, serializer)
except zmq.ZMQError as e:
if e.errno == 98:
raise ConnectionError(
"Encountered - %s. Perhaps the server is already running?" %
repr(e))
if e.errno == 22:
raise ValueError(
"Encountered - %s. `server_address` must be a string containing a valid endpoint."
% repr(e))
raise
finally:
sock.close()
util.close_zmq_ctx(zmq_ctx)
return server_process, server_address
def __init__(self,
server_address: str,
*,
namespace: str = "default",
secret_key: Optional[str] = None) -> None:
"""
Allows accessing state stored on the zproc server, through a dict-like API.
Communicates to the zproc server using the ZMQ sockets.
Please don't share a State object between Processes/Threads.
A State object is not thread-safe.
Boasts the following ``dict``-like members, for accessing the state:
- Magic methods:
``__contains__()``, ``__delitem__()``, ``__eq__()``,
``__getitem__()``, ``__iter__()``,
``__len__()``, ``__ne__()``, ``__setitem__()``
- Methods:
``clear()``, ``copy()``, ``get()``,
``items()``, ``keys()``, ``pop()``, ``popitem()``,
``setdefault()``, ``update()``, ``values()``
:param server_address:
The address of zproc server.
If you are using a :py:class:`Context`, then this is automatically provided.
Please read :ref:`server-address-spec` for a detailed explanation.
:ivar server_address: Passed on from constructor.
"""
super().__init__(secret_key)
self.server_address = server_address
self.namespace = namespace
self._identity = os.urandom(ZMQ_IDENTITY_SIZE)
self._zmq_ctx = util.create_zmq_ctx()
self._dealer_sock = self._zmq_ctx.socket(zmq.DEALER)
self._dealer_sock.setsockopt(zmq.IDENTITY, self._identity)
self._dealer_sock.connect(self.server_address)
self._sub_address, self._push_address = self._head()
self._push_sock = self._zmq_ctx.socket(zmq.PUSH)
self._push_sock.connect(self._push_address)
self._active_sub_sock = self._create_sub_sock()
def __init__(self,
server_address: str,
*,
namespace: str = DEFAULT_NAMESPACE):
#: Passed on from the constructor.
self.server_address = server_address
#: Passed on from the constructor.
self.namespace = namespace
#: A ``list`` of :py:class:`multiprocessing.Process` objects for the wokers spawned.
self.worker_list = [] # type: List[multiprocessing.Process]
self._zmq_ctx = util.create_zmq_ctx()
self._server_meta = util.get_server_meta(self._zmq_ctx, server_address)
self._task_push = self._zmq_ctx.socket(zmq.PUSH)
self._task_push.connect(self._server_meta.task_proxy_in)
def main(self):
@wraps(self.target)
def target_wrapper(*args, **kwargs):
while True:
self.retries += 1
try:
return self.target(*args, **kwargs)
except exceptions.ProcessExit as e:
self.exitcode = e.exitcode
return None
except self.to_catch as e:
self._handle_exc(e, handle_retry=True)
if self.retry_args is not None:
self.target_args = self.retry_args
if self.retry_kwargs is not None:
self.target_kwargs = self.retry_kwargs
try:
if self.pass_context:
from .context import Context # this helps avoid a circular import
return_value = target_wrapper(
Context(
self.kwargs["server_address"],
namespace=self.kwargs["namespace"],
start_server=False,
), *self.target_args, **self.target_kwargs)
else:
return_value = target_wrapper(*self.target_args,
**self.target_kwargs)
# print(return_value)
with util.create_zmq_ctx(linger=True) as zmq_ctx:
with zmq_ctx.socket(zmq.PAIR) as result_sock:
result_sock.connect(self.kwargs["result_address"])
result_sock.send(serializer.dumps(return_value))
except Exception as e:
self._handle_exc(e)
finally:
util.clean_process_tree(self.exitcode)
def __init__(self,
server_address: str,
*,
namespace: str = DEFAULT_NAMESPACE) -> None:
"""
Allows accessing the state stored on the zproc server, through a dict-like API.
Also allows changing the namespace.
Serves the following ``dict``-like members, for accessing the state:
- Magic methods:
``__contains__()``, ``__delitem__()``, ``__eq__()``,
``__getitem__()``, ``__iter__()``,
``__len__()``, ``__ne__()``, ``__setitem__()``
- Methods:
``clear()``, ``copy()``, ``get()``,
``items()``, ``keys()``, ``pop()``, ``popitem()``,
``setdefault()``, ``update()``, ``values()``
Please don't share a State object between Processes/Threads.
A State object is not thread-safe.
:param server_address:
.. include:: /api/snippets/server_address.rst
If you are using a :py:class:`Context`, then this is automatically provided.
"""
#: Passed on from constructor. This is read-only
self.server_address = server_address
self.namespace = namespace
self._zmq_ctx = util.create_zmq_ctx()
self._s_dealer = self._create_s_dealer()
self._w_dealer = self._create_w_dealer()
def ping(server_address: str,
*,
timeout: Optional[Union[float, int]] = None,
sent_payload: Optional[Union[bytes]] = None,
secret_key: str = None) -> Optional[int]:
"""
Ping the zproc server.
This can be used to easily detect if a server is alive and running, with the aid of a suitable ``timeout``.
:param server_address:
The zproc server's address.
Please read :ref:`server-address-spec` for a detailed explanation.
:param timeout:
The timeout in seconds.
If this is set to ``None``, then it will block forever, until the zproc server replies.
For all other values, it will wait for a reply,
for that amount of time before returning with a ``TimeoutError``.
By default it is set to ``None``.
:param sent_payload:
payload that will be sent to the server.
If it is set to None, then ``os.urandom(56)`` (56 random bytes) will be used.
(No real reason for the ``56`` magic number.)
:return:
The zproc server's **pid** if the ping was successful, else ``None``
If this returns ``None``,
then it probably means there is some fault in communication with the server.
"""
if sent_payload is None:
sent_payload = os.urandom(56)
serializer = util.get_serializer(secret_key)
zmq_ctx = util.create_zmq_ctx()
sock = zmq_ctx.socket(zmq.DEALER)
sock.connect(server_address)
if timeout is not None:
sock.setsockopt(zmq.RCVTIMEO, int(timeout * 1000))
sock.send(
serializer.dumps({
Msgs.cmd: Commands.ping,
Msgs.info: sent_payload
}))
try:
response = serializer.loads(sock.recv())
except zmq.error.Again:
raise TimeoutError(
"Timed-out waiting while for the ZProc server to respond.")
else:
recv_payload, pid = response[Msgs.info]
if recv_payload == sent_payload:
return pid
else:
return None
finally:
sock.close()
def __init__(
self,
server_address: str,
target: Callable,
*,
args: Sequence = None,
kwargs: Mapping = None,
start: bool = True,
pass_context: bool = True,
retry_for: Iterable[Union[signal.Signals, Type[BaseException]]] = (),
retry_delay: Union[int, float] = 5,
max_retries: Optional[int] = 0,
retry_args: tuple = None,
retry_kwargs: dict = None,
backend: Callable = multiprocessing.Process,
namespace: str = DEFAULT_NAMESPACE,
) -> None:
"""
Provides a higher level interface to :py:class:`multiprocessing.Process`.
Please don't share a Process object between Processes / Threads.
A Process object is not thread-safe.
:param server_address:
.. include:: /api/snippets/server_address.rst
If you are using a :py:class:`Context`, then this is automatically provided.
:param target:
The Callable to be invoked inside a new process.
*The* ``target`` *is invoked with the following signature:*
.. code-block:: python
target(state, *args, **kwargs)
*Where:*
- ``state`` is a :py:class:`State` instance.
- ``args`` and ``kwargs`` are passed from the constructor.
:param args:
The argument tuple for ``target``.
By default, it is an empty ``tuple``.
:param kwargs:
A dictionary of keyword arguments for ``target``.
By default, it is an empty ``dict``.
:param pass_state:
Weather this process needs to access the state.
If this is set to ``False``,
then the ``state`` argument won't be provided to the ``target``.
*If this is enabled, the* ``target`` *is invoked with the following signature:*
.. code-block:: python
target(*args, **kwargs)
*Where:*
- ``args`` and ``kwargs`` are passed from the constructor.
Has no effect if ``pass_context`` is set to ``True``.
:param pass_context:
Weather to pass a :py:class:`Context` to this process.
If this is set to ``True``,
then the first argument to ``target`` will be a new :py:class:`Context` object
This will take the place of the default - :py:class:`State`.
*If this is enabled, the* ``target`` *is invoked with the following signature*:
.. code-block:: python
target(ctx, *args, **kwargs)
*Where:*
- ``ctx`` is a :py:class:`Context` object.
- ``args`` and ``kwargs`` are passed from the constructor.
.. note::
The :py:class:`Context` object provided here,
will be a new object, NOT the one used to create this process.
Such that,
this new :py:class:`Context` can be used to spwan new processes,
that share the same state.
**This is the recommended way to create nested processes
that share the same state.**
:param start:
Automatically call :py:meth:`.start()` on the process.
:param retry_for:
Retry only when one of these ``Exception``/``signal.Signals`` is raised.
.. code-block:: python
:caption: Example
import signal
# retry if a ConnectionError, ValueError or signal.SIGTERM is received.
ctx.spawn(
my_process,
retry_for=(ConnectionError, ValueError, signal.SIGTERM)
)
To retry for *any* Exception - ``retry_for=(Exception, )``
The items of this sequence MUST be a subclass of ``BaseException`` or of type ``signal.Signals``.
:param retry_delay:
The delay in seconds, before retrying.
:param max_retries:
Maximum number of retries before giving up.
If set to ``None``, the Process will never stop retrying.
After "max_tries", any Exception / Signal will exhibit default behavior.
:param retry_args:
Used in place of ``args`` when retrying.
If set to ``None``, then it has no effect.
:param retry_kwargs:
Used in place of ``kwargs`` when retrying.
If set to ``None``, then it has no effect.
:param backend:
.. include:: /api/snippets/backend.rst
"""
#: Passed on from the constructor.
self.server_address = server_address
#: Passed on from the constructor.
self.namespace = namespace
#: Passed on from the constructor.
self.target = target
if args is None:
args = ()
if kwargs is None:
kwargs = {}
self._zmq_ctx = util.create_zmq_ctx()
self._result_sock = self._zmq_ctx.socket(zmq.PAIR)
# The result socket is meant to be used only after the process completes (after `join()`).
# That implies -- we shouldn't need to wait for the result message.
self._result_sock.setsockopt(zmq.RCVTIMEO, 0)
result_address = util.bind_to_random_address(self._result_sock)
#: The :py:class:`multiprocessing.Process` instance for the child process.
self.child = backend(
target=ChildProcess,
kwargs=dict(
target=self.target,
server_address=self.server_address,
namespace=self.namespace,
pass_context=pass_context,
target_args=args,
target_kwargs=kwargs,
retry_for=retry_for,
retry_delay=retry_delay,
max_retries=max_retries,
retry_args=retry_args,
retry_kwargs=retry_kwargs,
result_address=result_address,
),
)
if start:
self.child.start()