def execute_process( self, process_class, init_args=None, init_kwargs=None, loader=None, nowait=False, no_reply=False ): """ Execute a process. This call will first send a create task and then a continue task over the communicator. This means that if communicator messages are durable then the process will run until the end even if this interpreter instance ceases to exist. :param process_class: the process class to execute :param init_args: the positional arguments to the class constructor :param init_kwargs: the keyword arguments to the class constructor :param loader: the class loader to use :param nowait: if True, don't wait for the process to send a response :param no_reply: if True, this call will be fire-and-forget, i.e. no return value :return: the result of executing the process """ # pylint: disable=too-many-arguments message = create_create_body(process_class, init_args, init_kwargs, persist=True, loader=loader) execute_future = kiwipy.Future() create_future = futures.unwrap_kiwi_future(self._communicator.task_send(message)) def on_created(_): with kiwipy.capture_exceptions(execute_future): pid = create_future.result() continue_future = self.continue_process(pid, nowait=nowait, no_reply=no_reply) kiwipy.chain(continue_future, execute_future) create_future.add_done_callback(on_created) return execute_future
def execute_process(self, process_class, init_args=None, init_kwargs=None, nowait=False, loader=None): message = create_create_body(process_class, init_args, init_kwargs, persist=True, loader=loader) execute_future = kiwipy.Future() create_future = futures.unwrap_kiwi_future( self._communicator.task_send(message)) def on_created(_): try: pid = create_future.result() continue_future = self.continue_process(pid, nowait=nowait) kiwipy.chain(continue_future, execute_future) except Exception as exception: execute_future.set_exception(exception) create_future.add_done_callback(on_created) return execute_future
def test_broadcast_filter_sender_and_subject(self): senders_and_subects = set() EXPECTED = { ('bob.jones', 'purchase.car'), ('bob.jones', 'purchase.piano'), ('alice.jones', 'purchase.car'), ('alice.jones', 'purchase.piano'), } done = kiwipy.Future() def on_broadcast_1(body, sender=None, subject=None, correlation_id=None): senders_and_subects.add((sender, subject)) if len(senders_and_subects) == len(EXPECTED): done.set_result(True) filtered = kiwipy.BroadcastFilter(on_broadcast_1) filtered.add_sender_filter("*.jones") filtered.add_subject_filter("purchase.*") self.communicator.add_broadcast_subscriber(filtered) for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']: for subject in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']: self.communicator.broadcast_send(None, sender=sender, subject=subject) self.communicator.await(done, timeout=self.WAIT_TIMEOUT) self.assertEqual(4, len(senders_and_subects)) self.assertSetEqual(EXPECTED, senders_and_subects)
def test_custom_task_queue(self): """Test creating a custom task queue""" TASK = 'The meaning?' RESULT = 42 result_future = kiwipy.Future() tasks = [] def on_task(_comm, task): tasks.append(task) return result_future task_queue = self.communicator.task_queue( f'test-queue-{utils.rand_string(5)}') task_queue.add_task_subscriber(on_task) task_future = task_queue.task_send(TASK).result( timeout=self.WAIT_TIMEOUT) result_future.set_result(42) result = task_future.result(timeout=self.WAIT_TIMEOUT) self.assertEqual(TASK, tasks[0]) self.assertEqual(RESULT, result)
def test_broadcast_filter_sender_and_subject_regex(self): """As the standard broadcast test but using regular expressions instead of wildcards""" # pylint: disable=invalid-name senders_and_subjects = set() EXPECTED = { ('bob.jones', 'purchase.car'), ('bob.jones', 'purchase.piano'), ('alice.jones', 'purchase.car'), ('alice.jones', 'purchase.piano'), } done = kiwipy.Future() def on_broadcast_1(_communicator, _body, sender=None, subject=None, _correlation_id=None): senders_and_subjects.add((sender, subject)) if len(senders_and_subjects) == len(EXPECTED): done.set_result(True) filtered = kiwipy.BroadcastFilter(on_broadcast_1) filtered.add_sender_filter(re.compile('.*.jones')) filtered.add_subject_filter(re.compile('purchase.*')) self.communicator.add_broadcast_subscriber(filtered) for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']: for subj in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']: self.communicator.broadcast_send(None, sender=sender, subject=subj) done.result(timeout=self.WAIT_TIMEOUT) self.assertSetEqual(EXPECTED, senders_and_subjects)
def _on_connection_closed(self, connection, reply_code, reply_text): """This method is invoked by pika when the connection to RabbitMQ is closed unexpectedly. Since it is unexpected, we will reconnect to RabbitMQ if it disconnects. :param pika.connection.Connection connection: The closed connection obj :param int reply_code: The server provided reply_code if given :param str reply_text: The server provided reply_text if given """ self._connection = None LOGGER.info("The connection was closed: ({}) {}".format( reply_code, reply_text)) if self._state is not ConnectorState.DISCONNECTED and self._reconnect_timeout is not None: self._connecting_future = kiwipy.Future() self._state = ConnectorState.WAITING_TO_RECONNECT LOGGER.warning( "Connection closed, reopening in {} seconds: ({}) {}".format( self._reconnect_timeout, reply_code, reply_text)) self._connection.add_timeout(self._reconnect_timeout, self._reconnect) else: self._state = ConnectorState.DISCONNECTED self._connecting_future = None if self._disconnecting_future: self._disconnecting_future.set_result(True) self._event_helper.fire_event( ConnectionListener.on_connection_closed, self, self._state == ConnectorState.WAITING_TO_RECONNECT)
def _schedule_rpc(self, callback, *args, **kwargs): """ Schedule a call to a callback as a result of an RPC communication call, this will return a future that resolves to the final result (even after one or more layer of futures being returned) of the callback. :param callback: the callback function or coroutine :param args: the positional arguments to the callback :param kwargs: the keyword arguments to the callback :return: a kiwi future that resolves to the outcome of the callback :rtype: :class:`kiwipy.Future` """ kiwi_future = kiwipy.Future() @gen.coroutine def run_callback(): with kiwipy.capture_exceptions(kiwi_future): result = callback(*args, **kwargs) while concurrent.is_future(result): result = yield result kiwi_future.set_result(result) # Schedule the task and give back a kiwi future self.loop().add_callback(run_callback) return kiwi_future
def do_publish(self, correlation_id, *args, **kwargs): if self._confirm_deliveries and correlation_id is None: # Give a temporary ID to be able to keep track of returned messages correlation_id = str(uuid.uuid4()) try: properties = kwargs['properties'] except KeyError: properties = pika.BasicProperties() properties.correlation_id = correlation_id with self._connector.blocking_channel() as publish_channel: publish_channel.publish(*args, **kwargs) delivery_future = None if self._confirm_deliveries: delivery_future = kiwipy.Future() self._num_published += 1 self._delivery_info.append( DeliveryInfo(self._num_published, correlation_id, delivery_future)) return delivery_future
def unwrap_kiwi_future(future): """ Create a kiwi future that represents the final results of a nested series of futures, meaning that if the futures provided itself resolves to a future the returned future will not resolve to a value until the final chain of futures is not a future but a concrete value. If at any point in the chain a future resolves to an exception then the returned future will also resolve to that exception. :param future: the future to unwrap :type future: :class:`kiwipy.Future` :return: the unwrapping future :rtype: :class:`kiwipy.Future` """ unwrapping = kiwipy.Future() def unwrap(fut): if fut.cancelled(): unwrapping.cancel() else: with kiwipy.capture_exceptions(unwrapping): result = fut.result() if isinstance(result, kiwipy.Future): result.add_done_callback(unwrap) else: unwrapping.set_result(result) future.add_done_callback(unwrap) return unwrapping
def __init__(self, body, sender=None, subject=None, correlation_id=None): self.message = { BroadcastMessage.BODY: body, BroadcastMessage.SENDER: sender, BroadcastMessage.SUBJECT: subject, BroadcastMessage.CORRELATION_ID: correlation_id, } self._future = kiwipy.Future()
def test_capture_exceptions(self): future = kiwipy.Future() exception = RuntimeError() with kiwipy.capture_exceptions(future): raise exception self.assertIs(exception, future.exception())
def await (self, future=None, timeout=None): # Ensure we're connected if future is None: future = kiwipy.Future() self.connect() try: return self._loop.run_sync(lambda: future, timeout=timeout) except tornado.ioloop.TimeoutError as e: raise kiwipy.TimeoutError(str(e))
def test_add_remove_broadcast_subscriber(self): broadcast_received = kiwipy.Future() def broadcast_subscriber(body, sender=None, subject=None, correlation_id=None): broadcast_received.set_result(True) # Check we're getting messages self.communicator.add_broadcast_subscriber(broadcast_subscriber) self.communicator.broadcast_send(None) self.assertTrue(self.communicator.await(broadcast_received, timeout=self.WAIT_TIMEOUT))
def disconnect(self): """This method closes the connection to RabbitMQ.""" if self._state is not ConnectorState.DISCONNECTED: LOGGER.info('Closing connection') self._disconnecting_future = kiwipy.Future() self._connection.close() self._connection = None self._connecting_future = None self._state = ConnectorState.DISCONNECTED return self._disconnecting_future
def __init__(self, amqp_url, auto_reconnect_timeout=None, loop=None): self._connection_params = pika.URLParameters(amqp_url) self._reconnect_timeout = auto_reconnect_timeout self._loop = loop if loop is not None else loops.new_event_loop() self._event_helper = kiwipy.EventHelper(ConnectionListener) self._running_future = None self._stopping = False self._connecting_future = kiwipy.Future() self._disconnecting_future = None self._state = ConnectorState.DISCONNECTED
def exchange_declare(self, channel, nowait=False, **kwargs): params = dict(kwargs) params['nowait'] = nowait if nowait: callback = None future = None else: future = kiwipy.Future() callback = future.set_result params['callback'] = callback LOGGER.info('Declaring exchange {}'.format(params)) channel.exchange_declare(**params) return future
def queue_bind(self, channel, nowait=False, **kwargs): params = dict(kwargs) params['nowait'] = nowait if nowait: callback = None future = None else: future = kiwipy.Future() callback = future.set_result params['callback'] = callback LOGGER.info('Binding queue {}'.format(params)) channel.queue_bind(**params) return future
def test_add_remove_broadcast_subscriber(self): """Test adding, removing and readding a broadcast subscriber""" broadcast_received = kiwipy.Future() def broadcast_subscriber(_comm, body, sender=None, subject=None, correlation_id=None): # pylint: disable=unused-argument broadcast_received.set_result(True) # Check we're getting messages self.communicator.add_broadcast_subscriber(broadcast_subscriber, broadcast_subscriber.__name__) self.communicator.broadcast_send(None) self.assertTrue(broadcast_received.result(timeout=self.WAIT_TIMEOUT)) self.communicator.remove_broadcast_subscriber(broadcast_subscriber.__name__) # Check that we're unsubscribed broadcast_received = kiwipy.Future() with self.assertRaises(kiwipy.TimeoutError): broadcast_received.result(timeout=self.WAIT_TIMEOUT) broadcast_received = kiwipy.Future() self.communicator.add_broadcast_subscriber(broadcast_subscriber, broadcast_subscriber.__name__) self.communicator.broadcast_send(None) self.assertTrue(broadcast_received.result(timeout=self.WAIT_TIMEOUT))
def test_broadcast_send(self): SUBJECT = 'yo momma' # pylint: disable=invalid-name BODY = 'so fat' # pylint: disable=invalid-name SENDER_ID = 'me' # pylint: disable=invalid-name FULL_MSG = {'body': BODY, 'subject': SUBJECT, 'sender': SENDER_ID, 'correlation_id': None} # pylint: disable=invalid-name message1 = kiwipy.Future() message2 = kiwipy.Future() def on_broadcast_1(_comm, body, sender, subject, correlation_id): message1.set_result({'body': body, 'subject': subject, 'sender': sender, 'correlation_id': correlation_id}) def on_broadcast_2(_comm, body, sender, subject, correlation_id): message2.set_result({'body': body, 'subject': subject, 'sender': sender, 'correlation_id': correlation_id}) self.communicator.add_broadcast_subscriber(on_broadcast_1) self.communicator.add_broadcast_subscriber(on_broadcast_2) self.communicator.broadcast_send(**FULL_MSG) # Wait fot the send and receive kiwipy.wait((message1, message2), timeout=self.WAIT_TIMEOUT) self.assertDictEqual(message1.result(), FULL_MSG) self.assertDictEqual(message2.result(), FULL_MSG)
def test_context_manager(self): MESSAGE = 'get this yo' rpc_future = kiwipy.Future() def rpc_get(_comm, msg): rpc_future.set_result(msg) self.communicator.add_rpc_subscriber(rpc_get, 'test_context_manager') # Check the context manager of the communicator works with self.communicator as comm: comm.rpc_send('test_context_manager', MESSAGE) message = rpc_future.result(self.WAIT_TIMEOUT) self.assertEqual(MESSAGE, message)
def test_broadcast_send(self): SUBJECT = 'yo momma' BODY = 'so fat' SENDER_ID = 'me' FULL_MSG = {'body': BODY, 'subject': SUBJECT, 'sender': SENDER_ID, 'correlation_id': None} message1 = kiwipy.Future() message2 = kiwipy.Future() def on_broadcast_1(*args, **msg): message1.set_result(msg) def on_broadcast_2(*args, **msg): message2.set_result(msg) self.communicator.add_broadcast_subscriber(on_broadcast_1) self.communicator.add_broadcast_subscriber(on_broadcast_2) self.communicator.broadcast_send(**FULL_MSG) # Wait fot the send and receive self.communicator.await(kiwipy.gather(message1, message2), timeout=self.WAIT_TIMEOUT) self.assertDictEqual(message1.result(), FULL_MSG) self.assertDictEqual(message2.result(), FULL_MSG)
def response_to_future(response, future=None): if not isinstance(response, collections.Mapping): raise TypeError("Response must be a mapping") if future is None: future = kiwipy.Future() if CANCELLED_KEY in response: future.cancel() elif EXCEPTION_KEY in response: future.set_exception(kiwipy.RemoteException(response[EXCEPTION_KEY])) elif RESULT_KEY in response: future.set_result(response[RESULT_KEY]) elif not PENDING_KEY in response: raise ValueError("Unknown response type '{}'".format(response)) return future
def _on_response(self, ch, method, props, body): """ Called when we get a message on our response queue """ correlation_id = props.correlation_id try: callback = self._awaiting_response[correlation_id] except KeyError: # TODO: Log pass else: response = self._response_decode(body) response_future = kiwipy.Future() utils.response_to_future(response, response_future) if response_future.done(): self._awaiting_response.pop(correlation_id) callback(response_future) else: pass # Keep waiting
def converted(_comm, msg): kiwi_future = kiwipy.Future() def task_done(task): try: result = task.result() if concurrent.is_future(result): result = plum_to_kiwi_future(communicator, result) kiwi_future.set_result(result) except Exception as exception: kiwi_future.set_exception(exception) msg_fn = functools.partial(to_convert, communicator, msg) task_future = futures.create_task(msg_fn, loop) task_future.add_done_callback(task_done) return kiwi_future
def test_broadcast_filter_subject(self): subjects = [] EXPECTED_SUBJECTS = ['purchase.car', 'purchase.piano'] # pylint: disable=invalid-name done = kiwipy.Future() def on_broadcast_1(_comm, _body, _sender=None, subject=None, _correlation_id=None): subjects.append(subject) if len(subjects) == len(EXPECTED_SUBJECTS): done.set_result(True) self.communicator.add_broadcast_subscriber(kiwipy.BroadcastFilter(on_broadcast_1, subject='purchase.*')) for subj in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']: self.communicator.broadcast_send(None, subject=subj) done.result(timeout=self.WAIT_TIMEOUT) self.assertEqual(len(subjects), 2) self.assertListEqual(EXPECTED_SUBJECTS, subjects)
def test_broadcast_filter_match(self): """Test exact match broadcast filter""" EXPECTED_SENDERS = ['alice.jones'] # pylint: disable=invalid-name senders = [] done = kiwipy.Future() def on_broadcast_1(_comm, _body, sender=None, _subject=None, _correlation_id=None): senders.append(sender) if len(senders) == len(EXPECTED_SENDERS): done.set_result(True) self.communicator.add_broadcast_subscriber(kiwipy.BroadcastFilter(on_broadcast_1, sender='alice.jones')) for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']: self.communicator.broadcast_send(None, sender=sender) done.result(timeout=self.WAIT_TIMEOUT) self.assertListEqual(EXPECTED_SENDERS, senders)
def test_future_task(self): TASK = 'The meaning?' RESULT = 42 result_future = kiwipy.Future() tasks = [] def on_task(task): tasks.append(task) return result_future self.communicator.add_task_subscriber(on_task) task_future = self.communicator.task_send(TASK) result_future.set_result(42) result = self.communicator.await(task_future) self.assertEqual(tasks[0], TASK) self.assertEqual(result, RESULT)
def test_broadcast_filter_subject(self): subjects = [] EXPECTED_SUBJECTS = ['purchase.car', 'purchase.piano'] done = kiwipy.Future() def on_broadcast_1(body, sender=None, subject=None, correlation_id=None): subjects.append(subject) if len(subjects) == len(EXPECTED_SUBJECTS): done.set_result(True) self.communicator.add_broadcast_subscriber( kiwipy.BroadcastFilter(on_broadcast_1, subject="purchase.*")) for subject in ['purchase.car', 'purchase.piano', 'sell.guitar', 'sell.house']: self.communicator.broadcast_send(None, subject=subject) self.communicator.await(done, timeout=self.WAIT_TIMEOUT) self.assertEqual(len(subjects), 2) self.assertListEqual(EXPECTED_SUBJECTS, subjects)
def test_broadcast_filter_sender(self): EXPECTED_SENDERS = ['bob.jones', 'alice.jones'] senders = [] done = kiwipy.Future() def on_broadcast_1(body, sender=None, subject=None, correlation_id=None): senders.append(sender) if len(senders) == len(EXPECTED_SENDERS): done.set_result(True) self.communicator.add_broadcast_subscriber( kiwipy.BroadcastFilter(on_broadcast_1, sender="*.jones")) for sender in ['bob.jones', 'bob.smith', 'martin.uhrin', 'alice.jones']: self.communicator.broadcast_send(None, sender=sender) self.communicator.await(done, timeout=self.WAIT_TIMEOUT) self.assertEqual(2, len(senders)) self.assertListEqual(EXPECTED_SENDERS, senders)
def test_future_task(self): """ Test a task that returns a future meaning that will be resolve to a value later """ TASK = 'The meaning?' RESULT = 42 result_future = kiwipy.Future() tasks = [] def on_task(_comm, task): tasks.append(task) return result_future self.communicator.add_task_subscriber(on_task) task_future = self.communicator.task_send(TASK).result(timeout=self.WAIT_TIMEOUT) result_future.set_result(42) result = task_future.result(timeout=self.WAIT_TIMEOUT) self.assertEqual(TASK, tasks[0]) self.assertEqual(RESULT, result)