def execute_process(
self, process_class, init_args=None, init_kwargs=None, loader=None, nowait=False, no_reply=False
):
"""
Execute a process. This call will first send a create task and then a continue task over
the communicator. This means that if communicator messages are durable then the process
will run until the end even if this interpreter instance ceases to exist.
:param process_class: the process class to execute
:param init_args: the positional arguments to the class constructor
:param init_kwargs: the keyword arguments to the class constructor
:param loader: the class loader to use
:param nowait: if True, don't wait for the process to send a response
:param no_reply: if True, this call will be fire-and-forget, i.e. no return value
:return: the result of executing the process
"""
# pylint: disable=too-many-arguments
message = create_create_body(process_class, init_args, init_kwargs, persist=True, loader=loader)
execute_future = kiwipy.Future()
create_future = futures.unwrap_kiwi_future(self._communicator.task_send(message))
def on_created(_):
with kiwipy.capture_exceptions(execute_future):
pid = create_future.result()
continue_future = self.continue_process(pid, nowait=nowait, no_reply=no_reply)
kiwipy.chain(continue_future, execute_future)
create_future.add_done_callback(on_created)
return execute_future
def execute_process(self,
process_class,
init_args=None,
init_kwargs=None,
nowait=False,
loader=None):
message = create_create_body(process_class,
init_args,
init_kwargs,
persist=True,
loader=loader)
execute_future = kiwipy.Future()
create_future = futures.unwrap_kiwi_future(
self._communicator.task_send(message))
def on_created(_):
try:
pid = create_future.result()
continue_future = self.continue_process(pid, nowait=nowait)
kiwipy.chain(continue_future, execute_future)
except Exception as exception:
execute_future.set_exception(exception)
create_future.add_done_callback(on_created)
return execute_future
def test_broadcast_filter_sender_and_subject(self):
senders_and_subects = set()
EXPECTED = {
('bob.jones', 'purchase.car'),
('bob.jones', 'purchase.piano'),
('alice.jones', 'purchase.car'),
('alice.jones', 'purchase.piano'),
}
done = kiwipy.Future()
def on_broadcast_1(body, sender=None, subject=None, correlation_id=None):
senders_and_subects.add((sender, subject))
if len(senders_and_subects) == len(EXPECTED):
done.set_result(True)
filtered = kiwipy.BroadcastFilter(on_broadcast_1)
filtered.add_sender_filter("*.jones")
filtered.add_subject_filter("purchase.*")
self.communicator.add_broadcast_subscriber(filtered)
for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']:
for subject in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']:
self.communicator.broadcast_send(None, sender=sender, subject=subject)
self.communicator.await(done, timeout=self.WAIT_TIMEOUT)
self.assertEqual(4, len(senders_and_subects))
self.assertSetEqual(EXPECTED, senders_and_subects)
def test_custom_task_queue(self):
"""Test creating a custom task queue"""
TASK = 'The meaning?'
RESULT = 42
result_future = kiwipy.Future()
tasks = []
def on_task(_comm, task):
tasks.append(task)
return result_future
task_queue = self.communicator.task_queue(
f'test-queue-{utils.rand_string(5)}')
task_queue.add_task_subscriber(on_task)
task_future = task_queue.task_send(TASK).result(
timeout=self.WAIT_TIMEOUT)
result_future.set_result(42)
result = task_future.result(timeout=self.WAIT_TIMEOUT)
self.assertEqual(TASK, tasks[0])
self.assertEqual(RESULT, result)
def test_broadcast_filter_sender_and_subject_regex(self):
"""As the standard broadcast test but using regular expressions instead of wildcards"""
# pylint: disable=invalid-name
senders_and_subjects = set()
EXPECTED = {
('bob.jones', 'purchase.car'),
('bob.jones', 'purchase.piano'),
('alice.jones', 'purchase.car'),
('alice.jones', 'purchase.piano'),
}
done = kiwipy.Future()
def on_broadcast_1(_communicator, _body, sender=None, subject=None, _correlation_id=None):
senders_and_subjects.add((sender, subject))
if len(senders_and_subjects) == len(EXPECTED):
done.set_result(True)
filtered = kiwipy.BroadcastFilter(on_broadcast_1)
filtered.add_sender_filter(re.compile('.*.jones'))
filtered.add_subject_filter(re.compile('purchase.*'))
self.communicator.add_broadcast_subscriber(filtered)
for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']:
for subj in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']:
self.communicator.broadcast_send(None, sender=sender, subject=subj)
done.result(timeout=self.WAIT_TIMEOUT)
self.assertSetEqual(EXPECTED, senders_and_subjects)
def _on_connection_closed(self, connection, reply_code, reply_text):
"""This method is invoked by pika when the connection to RabbitMQ is
closed unexpectedly. Since it is unexpected, we will reconnect to
RabbitMQ if it disconnects.
:param pika.connection.Connection connection: The closed connection obj
:param int reply_code: The server provided reply_code if given
:param str reply_text: The server provided reply_text if given
"""
self._connection = None
LOGGER.info("The connection was closed: ({}) {}".format(
reply_code, reply_text))
if self._state is not ConnectorState.DISCONNECTED and self._reconnect_timeout is not None:
self._connecting_future = kiwipy.Future()
self._state = ConnectorState.WAITING_TO_RECONNECT
LOGGER.warning(
"Connection closed, reopening in {} seconds: ({}) {}".format(
self._reconnect_timeout, reply_code, reply_text))
self._connection.add_timeout(self._reconnect_timeout,
self._reconnect)
else:
self._state = ConnectorState.DISCONNECTED
self._connecting_future = None
if self._disconnecting_future:
self._disconnecting_future.set_result(True)
self._event_helper.fire_event(
ConnectionListener.on_connection_closed, self,
self._state == ConnectorState.WAITING_TO_RECONNECT)
def _schedule_rpc(self, callback, *args, **kwargs):
"""
Schedule a call to a callback as a result of an RPC communication call, this will return
a future that resolves to the final result (even after one or more layer of futures being
returned) of the callback.
:param callback: the callback function or coroutine
:param args: the positional arguments to the callback
:param kwargs: the keyword arguments to the callback
:return: a kiwi future that resolves to the outcome of the callback
:rtype: :class:`kiwipy.Future`
"""
kiwi_future = kiwipy.Future()
@gen.coroutine
def run_callback():
with kiwipy.capture_exceptions(kiwi_future):
result = callback(*args, **kwargs)
while concurrent.is_future(result):
result = yield result
kiwi_future.set_result(result)
# Schedule the task and give back a kiwi future
self.loop().add_callback(run_callback)
return kiwi_future
def do_publish(self, correlation_id, *args, **kwargs):
if self._confirm_deliveries and correlation_id is None:
# Give a temporary ID to be able to keep track of returned messages
correlation_id = str(uuid.uuid4())
try:
properties = kwargs['properties']
except KeyError:
properties = pika.BasicProperties()
properties.correlation_id = correlation_id
with self._connector.blocking_channel() as publish_channel:
publish_channel.publish(*args, **kwargs)
delivery_future = None
if self._confirm_deliveries:
delivery_future = kiwipy.Future()
self._num_published += 1
self._delivery_info.append(
DeliveryInfo(self._num_published, correlation_id,
delivery_future))
return delivery_future
def unwrap_kiwi_future(future):
"""
Create a kiwi future that represents the final results of a nested series of futures,
meaning that if the futures provided itself resolves to a future the returned
future will not resolve to a value until the final chain of futures is not a future
but a concrete value. If at any point in the chain a future resolves to an exception
then the returned future will also resolve to that exception.
:param future: the future to unwrap
:type future: :class:`kiwipy.Future`
:return: the unwrapping future
:rtype: :class:`kiwipy.Future`
"""
unwrapping = kiwipy.Future()
def unwrap(fut):
if fut.cancelled():
unwrapping.cancel()
else:
with kiwipy.capture_exceptions(unwrapping):
result = fut.result()
if isinstance(result, kiwipy.Future):
result.add_done_callback(unwrap)
else:
unwrapping.set_result(result)
future.add_done_callback(unwrap)
return unwrapping
def __init__(self, body, sender=None, subject=None, correlation_id=None):
self.message = {
BroadcastMessage.BODY: body,
BroadcastMessage.SENDER: sender,
BroadcastMessage.SUBJECT: subject,
BroadcastMessage.CORRELATION_ID: correlation_id,
}
self._future = kiwipy.Future()
def test_capture_exceptions(self):
future = kiwipy.Future()
exception = RuntimeError()
with kiwipy.capture_exceptions(future):
raise exception
self.assertIs(exception, future.exception())
def await (self, future=None, timeout=None):
# Ensure we're connected
if future is None:
future = kiwipy.Future()
self.connect()
try:
return self._loop.run_sync(lambda: future, timeout=timeout)
except tornado.ioloop.TimeoutError as e:
raise kiwipy.TimeoutError(str(e))
def test_add_remove_broadcast_subscriber(self):
broadcast_received = kiwipy.Future()
def broadcast_subscriber(body, sender=None, subject=None, correlation_id=None):
broadcast_received.set_result(True)
# Check we're getting messages
self.communicator.add_broadcast_subscriber(broadcast_subscriber)
self.communicator.broadcast_send(None)
self.assertTrue(self.communicator.await(broadcast_received, timeout=self.WAIT_TIMEOUT))
def disconnect(self):
"""This method closes the connection to RabbitMQ."""
if self._state is not ConnectorState.DISCONNECTED:
LOGGER.info('Closing connection')
self._disconnecting_future = kiwipy.Future()
self._connection.close()
self._connection = None
self._connecting_future = None
self._state = ConnectorState.DISCONNECTED
return self._disconnecting_future
def __init__(self, amqp_url, auto_reconnect_timeout=None, loop=None):
self._connection_params = pika.URLParameters(amqp_url)
self._reconnect_timeout = auto_reconnect_timeout
self._loop = loop if loop is not None else loops.new_event_loop()
self._event_helper = kiwipy.EventHelper(ConnectionListener)
self._running_future = None
self._stopping = False
self._connecting_future = kiwipy.Future()
self._disconnecting_future = None
self._state = ConnectorState.DISCONNECTED
def exchange_declare(self, channel, nowait=False, **kwargs):
params = dict(kwargs)
params['nowait'] = nowait
if nowait:
callback = None
future = None
else:
future = kiwipy.Future()
callback = future.set_result
params['callback'] = callback
LOGGER.info('Declaring exchange {}'.format(params))
channel.exchange_declare(**params)
return future
def queue_bind(self, channel, nowait=False, **kwargs):
params = dict(kwargs)
params['nowait'] = nowait
if nowait:
callback = None
future = None
else:
future = kiwipy.Future()
callback = future.set_result
params['callback'] = callback
LOGGER.info('Binding queue {}'.format(params))
channel.queue_bind(**params)
return future
def test_add_remove_broadcast_subscriber(self):
"""Test adding, removing and readding a broadcast subscriber"""
broadcast_received = kiwipy.Future()
def broadcast_subscriber(_comm, body, sender=None, subject=None, correlation_id=None):
# pylint: disable=unused-argument
broadcast_received.set_result(True)
# Check we're getting messages
self.communicator.add_broadcast_subscriber(broadcast_subscriber, broadcast_subscriber.__name__)
self.communicator.broadcast_send(None)
self.assertTrue(broadcast_received.result(timeout=self.WAIT_TIMEOUT))
self.communicator.remove_broadcast_subscriber(broadcast_subscriber.__name__)
# Check that we're unsubscribed
broadcast_received = kiwipy.Future()
with self.assertRaises(kiwipy.TimeoutError):
broadcast_received.result(timeout=self.WAIT_TIMEOUT)
broadcast_received = kiwipy.Future()
self.communicator.add_broadcast_subscriber(broadcast_subscriber, broadcast_subscriber.__name__)
self.communicator.broadcast_send(None)
self.assertTrue(broadcast_received.result(timeout=self.WAIT_TIMEOUT))
def test_broadcast_send(self):
SUBJECT = 'yo momma' # pylint: disable=invalid-name
BODY = 'so fat' # pylint: disable=invalid-name
SENDER_ID = 'me' # pylint: disable=invalid-name
FULL_MSG = {'body': BODY, 'subject': SUBJECT, 'sender': SENDER_ID, 'correlation_id': None} # pylint: disable=invalid-name
message1 = kiwipy.Future()
message2 = kiwipy.Future()
def on_broadcast_1(_comm, body, sender, subject, correlation_id):
message1.set_result({'body': body, 'subject': subject, 'sender': sender, 'correlation_id': correlation_id})
def on_broadcast_2(_comm, body, sender, subject, correlation_id):
message2.set_result({'body': body, 'subject': subject, 'sender': sender, 'correlation_id': correlation_id})
self.communicator.add_broadcast_subscriber(on_broadcast_1)
self.communicator.add_broadcast_subscriber(on_broadcast_2)
self.communicator.broadcast_send(**FULL_MSG)
# Wait fot the send and receive
kiwipy.wait((message1, message2), timeout=self.WAIT_TIMEOUT)
self.assertDictEqual(message1.result(), FULL_MSG)
self.assertDictEqual(message2.result(), FULL_MSG)
def test_context_manager(self):
MESSAGE = 'get this yo'
rpc_future = kiwipy.Future()
def rpc_get(_comm, msg):
rpc_future.set_result(msg)
self.communicator.add_rpc_subscriber(rpc_get, 'test_context_manager')
# Check the context manager of the communicator works
with self.communicator as comm:
comm.rpc_send('test_context_manager', MESSAGE)
message = rpc_future.result(self.WAIT_TIMEOUT)
self.assertEqual(MESSAGE, message)
def test_broadcast_send(self):
SUBJECT = 'yo momma'
BODY = 'so fat'
SENDER_ID = 'me'
FULL_MSG = {'body': BODY, 'subject': SUBJECT, 'sender': SENDER_ID, 'correlation_id': None}
message1 = kiwipy.Future()
message2 = kiwipy.Future()
def on_broadcast_1(*args, **msg):
message1.set_result(msg)
def on_broadcast_2(*args, **msg):
message2.set_result(msg)
self.communicator.add_broadcast_subscriber(on_broadcast_1)
self.communicator.add_broadcast_subscriber(on_broadcast_2)
self.communicator.broadcast_send(**FULL_MSG)
# Wait fot the send and receive
self.communicator.await(kiwipy.gather(message1, message2), timeout=self.WAIT_TIMEOUT)
self.assertDictEqual(message1.result(), FULL_MSG)
self.assertDictEqual(message2.result(), FULL_MSG)
def response_to_future(response, future=None):
if not isinstance(response, collections.Mapping):
raise TypeError("Response must be a mapping")
if future is None:
future = kiwipy.Future()
if CANCELLED_KEY in response:
future.cancel()
elif EXCEPTION_KEY in response:
future.set_exception(kiwipy.RemoteException(response[EXCEPTION_KEY]))
elif RESULT_KEY in response:
future.set_result(response[RESULT_KEY])
elif not PENDING_KEY in response:
raise ValueError("Unknown response type '{}'".format(response))
return future
def _on_response(self, ch, method, props, body):
""" Called when we get a message on our response queue """
correlation_id = props.correlation_id
try:
callback = self._awaiting_response[correlation_id]
except KeyError:
# TODO: Log
pass
else:
response = self._response_decode(body)
response_future = kiwipy.Future()
utils.response_to_future(response, response_future)
if response_future.done():
self._awaiting_response.pop(correlation_id)
callback(response_future)
else:
pass # Keep waiting
def converted(_comm, msg):
kiwi_future = kiwipy.Future()
def task_done(task):
try:
result = task.result()
if concurrent.is_future(result):
result = plum_to_kiwi_future(communicator, result)
kiwi_future.set_result(result)
except Exception as exception:
kiwi_future.set_exception(exception)
msg_fn = functools.partial(to_convert, communicator, msg)
task_future = futures.create_task(msg_fn, loop)
task_future.add_done_callback(task_done)
return kiwi_future
def test_broadcast_filter_subject(self):
subjects = []
EXPECTED_SUBJECTS = ['purchase.car', 'purchase.piano'] # pylint: disable=invalid-name
done = kiwipy.Future()
def on_broadcast_1(_comm, _body, _sender=None, subject=None, _correlation_id=None):
subjects.append(subject)
if len(subjects) == len(EXPECTED_SUBJECTS):
done.set_result(True)
self.communicator.add_broadcast_subscriber(kiwipy.BroadcastFilter(on_broadcast_1, subject='purchase.*'))
for subj in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']:
self.communicator.broadcast_send(None, subject=subj)
done.result(timeout=self.WAIT_TIMEOUT)
self.assertEqual(len(subjects), 2)
self.assertListEqual(EXPECTED_SUBJECTS, subjects)
def test_broadcast_filter_match(self):
"""Test exact match broadcast filter"""
EXPECTED_SENDERS = ['alice.jones'] # pylint: disable=invalid-name
senders = []
done = kiwipy.Future()
def on_broadcast_1(_comm, _body, sender=None, _subject=None, _correlation_id=None):
senders.append(sender)
if len(senders) == len(EXPECTED_SENDERS):
done.set_result(True)
self.communicator.add_broadcast_subscriber(kiwipy.BroadcastFilter(on_broadcast_1, sender='alice.jones'))
for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']:
self.communicator.broadcast_send(None, sender=sender)
done.result(timeout=self.WAIT_TIMEOUT)
self.assertListEqual(EXPECTED_SENDERS, senders)
def test_future_task(self):
TASK = 'The meaning?'
RESULT = 42
result_future = kiwipy.Future()
tasks = []
def on_task(task):
tasks.append(task)
return result_future
self.communicator.add_task_subscriber(on_task)
task_future = self.communicator.task_send(TASK)
result_future.set_result(42)
result = self.communicator.await(task_future)
self.assertEqual(tasks[0], TASK)
self.assertEqual(result, RESULT)
def test_broadcast_filter_subject(self):
subjects = []
EXPECTED_SUBJECTS = ['purchase.car', 'purchase.piano']
done = kiwipy.Future()
def on_broadcast_1(body, sender=None, subject=None, correlation_id=None):
subjects.append(subject)
if len(subjects) == len(EXPECTED_SUBJECTS):
done.set_result(True)
self.communicator.add_broadcast_subscriber(
kiwipy.BroadcastFilter(on_broadcast_1, subject="purchase.*"))
for subject in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']:
self.communicator.broadcast_send(None, subject=subject)
self.communicator.await(done, timeout=self.WAIT_TIMEOUT)
self.assertEqual(len(subjects), 2)
self.assertListEqual(EXPECTED_SUBJECTS, subjects)
def test_broadcast_filter_sender(self):
EXPECTED_SENDERS = ['bob.jones', 'alice.jones']
senders = []
done = kiwipy.Future()
def on_broadcast_1(body, sender=None, subject=None, correlation_id=None):
senders.append(sender)
if len(senders) == len(EXPECTED_SENDERS):
done.set_result(True)
self.communicator.add_broadcast_subscriber(
kiwipy.BroadcastFilter(on_broadcast_1, sender="*.jones"))
for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']:
self.communicator.broadcast_send(None, sender=sender)
self.communicator.await(done, timeout=self.WAIT_TIMEOUT)
self.assertEqual(2, len(senders))
self.assertListEqual(EXPECTED_SENDERS, senders)
def test_future_task(self):
"""
Test a task that returns a future meaning that will be resolve to a value later
"""
TASK = 'The meaning?'
RESULT = 42
result_future = kiwipy.Future()
tasks = []
def on_task(_comm, task):
tasks.append(task)
return result_future
self.communicator.add_task_subscriber(on_task)
task_future = self.communicator.task_send(TASK).result(timeout=self.WAIT_TIMEOUT)
result_future.set_result(42)
result = task_future.result(timeout=self.WAIT_TIMEOUT)
self.assertEqual(TASK, tasks[0])
self.assertEqual(RESULT, result)
