예제 #1
0
    def ingest(self):
        """
        Monitor AMQP for messages

        """

        if self.stoq.worker.name:
            # Define our RabbitMQ route
            routing_key = self.stoq.worker.name

            # If this is an error message, let's make sure our queue
            # has "-errors" affixed to it
            if self.stoq.worker.error_queue is True:
                routing_key = routing_key + "-errors".strip()

            exchange = Exchange(self.exchange_name, type=self.exchange_type)
            queue = Queue(routing_key, exchange, routing_key=routing_key)

            # Setup our broker connection with RabbitMQ
            with Connection(hostname=self.host,
                            port=self.port,
                            userid=self.user,
                            password=self.password,
                            virtual_host=self.virtual_host) as conn:

                consumer = Consumer(conn, queue,
                                    callbacks=[self.queue_callback])
                consumer.qos(prefetch_count=int(self.prefetch))
                consumer.consume()

                while True:
                    conn.drain_events()
        else:
            self.stoq.log.error("No worker name defined!")
예제 #2
0
    def add_queue_rule(self, handler, name, autoack=True, prefetch_size=0,
                       prefetch_count=0, **kwargs):
        """Add queue rule to Microservice

        :param prefetch_count: count of messages for getting from mq
        :param prefetch_size: size in bytes for getting data from mq
        :param handler: function for handling messages
        :param autoack: if True message.ack() after callback
        :type handler: callable object
        :param name: name of queue
        :type name: str
        """

        if self.with_pool:
            if self.workers_override_prefetch:
                prefetch_count = self.workers
            rule = Rule(name, handler, self.logger, autoack=autoack,
                        deferred_callbacks=self.deferred_callbacks,
                        pool=self.pool, **kwargs)
        else:
            rule = Rule(name, handler, self.logger, autoack=autoack, **kwargs)
        self.connect()
        consumer = Consumer(self.connection, queues=[Queue(rule.name)],
                            callbacks=[rule.callback], auto_declare=True)
        consumer.qos(prefetch_count=prefetch_count, prefetch_size=prefetch_size)
        self.consumers.append(consumer)
        self.logger.debug('Rule "%s" added!', rule.name)
예제 #3
0
    def test_consumer_tag_prefix(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, tag_prefix='consumer_')
        consumer.consume()

        assert consumer._active_tags[queue.name].startswith('consumer_')
예제 #4
0
    def test_on_callback_error_with_continue(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        callback_exception = Exception('callback_exception')
        received, called = [], []

        def first_callback(message_data, message):
            called.append('first')

        def second_callback(message_data, message):
            called.append('second')
            raise callback_exception

        def third_callback(message_data, message):
            called.append('third')

        def on_callback_error(message_data, message, exc, callback):
            received.append((message_data, message, exc, callback))
            return True  # continue callbacks

        callbacks = [first_callback, second_callback, third_callback]
        consumer = Consumer(channel, [b1], callbacks=callbacks,
                            on_callback_error=on_callback_error)
        consumer._receive_callback({'foo': 'bar'})

        self.assertEqual(called, ['first', 'second', 'third'])

        self.assertEqual(len(received), 1)
        message_data, message, exc, callback = received[0]
        self.assertEqual(message_data, {'foo': 'bar'})
        self.assertEqual(exc, callback_exception)
        self.assertEqual(callback, second_callback)
예제 #5
0
 def test_enter_exit_cancel_raises(self):
     c = Consumer(self.connection)
     c.cancel = Mock(name='Consumer.cancel')
     c.cancel.side_effect = KeyError('foo')
     with c:
         pass
     c.cancel.assert_called_with()
예제 #6
0
파일: amqbench.py 프로젝트: ask/amqbench
def _kconsume(n, name, conn, durable=DURABLE):
    channel = conn.channel()
    exchange = Exchange(name, type="direct", durable=durable,
                        auto_delete=True)
    queue = Queue(name, exchange, name, durable=durable,
                  auto_delete=True)
    consumer = Consumer(channel, queue)
    ucon = conn.connect()

    i = [0]

    def callback(message_data, message=None):
        i[0] += 1
        if not i[0] % 10000:
            print(i[0])

    consumer.register_callback(callback)
    consumer.consume(no_ack=True)

    start = time()
    while i[0] < n:
        try:
            conn.drain_events()
        except socket.timeout:
            pass
    print(time() - start)
예제 #7
0
    def test_decode_error(self):
        channel = self.connection.channel()
        b1 = Queue("qname1", self.exchange, "rkey")
        consumer = Consumer(channel, [b1])
        consumer.channel.throw_decode_error = True

        with self.assertRaises(ValueError):
            consumer._receive_callback({"foo": "bar"})
예제 #8
0
 def test_enter_exit_cancel_not_called_on_connection_error(self):
     c = Consumer(self.connection)
     c.cancel = Mock(name='Consumer.cancel')
     assert self.connection.connection_errors
     with pytest.raises(self.connection.connection_errors[0]):
         with c:
             raise self.connection.connection_errors[0]()
     c.cancel.assert_not_called()
예제 #9
0
 def test_consume__cancel(self):
     channel = self.connection.channel()
     queue = Queue('qname', self.exchange, 'rkey')
     consumer = Consumer(channel, queue, auto_declare=True)
     consumer.consume()
     consumer.cancel()
     assert 'basic_cancel' in channel
     assert not consumer._active_tags
예제 #10
0
 def test_consume__cancel(self):
     channel = self.connection.channel()
     queue = Queue('qname', self.exchange, 'rkey')
     consumer = Consumer(channel, queue, auto_declare=True)
     consumer.consume()
     consumer.cancel()
     self.assertIn('basic_cancel', channel)
     self.assertFalse(consumer._active_tags)
예제 #11
0
    def test_decode_error(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])
        consumer.channel.throw_decode_error = True

        with self.assertRaises(ValueError):
            consumer._receive_callback({'foo': 'bar'})
예제 #12
0
 def run(self):
     consumer = Consumer(self.channel, self.queue, callbacks=[self.handle_message])
     consumer.consume()
     while True:
         try:
             self.connection.drain_events()
         except KeyboardInterrupt:
             return
예제 #13
0
 def test_timeout(self):
     if not self.verify_alive():
         return
     chan = self.connection.channel()
     self.purge([self.queue.name])
     consumer = Consumer(chan, self.queue)
     self.assertRaises(socket.timeout, self.connection.drain_events,
             timeout=0.3)
     consumer.cancel()
예제 #14
0
 def test_timeout(self):
     if not self.connected:
         raise SkipTest("Broker not running.")
     chan = self.connection.channel()
     self.purge([self.queue.name])
     consumer = Consumer(chan, self.queue)
     self.assertRaises(socket.timeout, self.connection.drain_events,
             timeout=0.3)
     consumer.cancel()
예제 #15
0
파일: __init__.py 프로젝트: DXist/celery
 def consumer(self, wakeup=True):
     """Create event consumer."""
     consumer = Consumer(self.connection,
                         queues=[self.queue], no_ack=True)
     consumer.register_callback(self._receive)
     with consumer:
         if wakeup:
             self.wakeup_workers(channel=consumer.channel)
         yield consumer
예제 #16
0
 def test_purge(self):
     channel = self.connection.channel()
     b1 = Queue("qname1", self.exchange, "rkey")
     b2 = Queue("qname2", self.exchange, "rkey")
     b3 = Queue("qname3", self.exchange, "rkey")
     b4 = Queue("qname4", self.exchange, "rkey")
     consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
     consumer.purge()
     self.assertEqual(channel.called.count("queue_purge"), 4)
예제 #17
0
 def test_revive(self):
     channel = self.connection.channel()
     b1 = Queue('qname1', self.exchange, 'rkey')
     consumer = Consumer(channel, [b1])
     channel2 = self.connection.channel()
     consumer.revive(channel2)
     self.assertIs(consumer.channel, channel2)
     self.assertIs(consumer.queues[0].channel, channel2)
     self.assertIs(consumer.queues[0].exchange.channel, channel2)
예제 #18
0
 def test_purge(self):
     channel = self.connection.channel()
     b1 = Queue('qname1', self.exchange, 'rkey')
     b2 = Queue('qname2', self.exchange, 'rkey')
     b3 = Queue('qname3', self.exchange, 'rkey')
     b4 = Queue('qname4', self.exchange, 'rkey')
     consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
     consumer.purge()
     self.assertEqual(channel.called.count('queue_purge'), 4)
예제 #19
0
    def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
        self.conn = BrokerConnection(hostname, userid, password, virtual_host)
        # define Web2Server exchange
        exchange = Exchange(self.kwargs["X7_E"], type="direct")
        self.queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
        channel = self.conn.channel()

        consumer = Consumer(channel, self.queue, callbacks=[self.callback])
        consumer.consume()
예제 #20
0
    def test_produce_consume(self):
        channel = self.c.channel()
        producer = Producer(channel, self.e)
        consumer1 = Consumer(channel, self.q)
        consumer2 = Consumer(channel, self.q2)
        self.q2(channel).declare()

        for i in range(10):
            producer.publish({'foo': i}, routing_key='test_transport_memory')
        for i in range(10):
            producer.publish({'foo': i}, routing_key='test_transport_memory2')

        _received1 = []
        _received2 = []

        def callback1(message_data, message):
            _received1.append(message)
            message.ack()

        def callback2(message_data, message):
            _received2.append(message)
            message.ack()

        consumer1.register_callback(callback1)
        consumer2.register_callback(callback2)

        consumer1.consume()
        consumer2.consume()

        while 1:
            if len(_received1) + len(_received2) == 20:
                break
            self.c.drain_events()

        self.assertEqual(len(_received1) + len(_received2), 20)

        # compression
        producer.publish({'compressed': True},
                         routing_key='test_transport_memory',
                         compression='zlib')
        m = self.q(channel).get()
        self.assertDictEqual(m.payload, {'compressed': True})

        # queue.delete
        for i in range(10):
            producer.publish({'foo': i}, routing_key='test_transport_memory')
        self.assertTrue(self.q(channel).get())
        self.q(channel).delete()
        self.q(channel).declare()
        self.assertIsNone(self.q(channel).get())

        # queue.purge
        for i in range(10):
            producer.publish({'foo': i}, routing_key='test_transport_memory2')
        self.assertTrue(self.q2(channel).get())
        self.q2(channel).purge()
        self.assertIsNone(self.q2(channel).get())
예제 #21
0
    def test_receive_callback_accept(self):
        message = Mock(name='Message')
        callback = Mock(name='on_message')
        c = Consumer(self.connection, accept=['json'], on_message=callback)
        c.channel = Mock(name='channel')
        c.channel.message_to_python = None

        c._receive_callback(message)
        callback.assert_called_with(message)
        self.assertSetEqual(message.accept, c.accept)
예제 #22
0
    def _get_receiver(self, receiver, callback):

        queue = Queue(name=receiver)

        result = Consumer(
            self.conn, queues=queue, no_ack=(not self.safe),
            callbacks=(lambda x, y: callback(x),))

        result.consume()

        return result
예제 #23
0
 def test___enter____exit__(self):
     channel = self.connection.channel()
     queue = Queue('qname', self.exchange, 'rkey')
     consumer = Consumer(channel, queue, auto_declare=True)
     context = consumer.__enter__()
     assert context is consumer
     assert consumer._active_tags
     res = consumer.__exit__(None, None, None)
     assert not res
     assert 'basic_cancel' in channel
     assert not consumer._active_tags
예제 #24
0
 def test___enter____exit__(self):
     channel = self.connection.channel()
     queue = Queue('qname', self.exchange, 'rkey')
     consumer = Consumer(channel, queue, auto_declare=True)
     context = consumer.__enter__()
     self.assertIs(context, consumer)
     self.assertTrue(consumer._active_tags)
     res = consumer.__exit__(None, None, None)
     self.assertFalse(res)
     self.assertIn('basic_cancel', channel)
     self.assertFalse(consumer._active_tags)
예제 #25
0
    def test_basic_reject__requeue(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.requeue()

        consumer.register_callback(callback)
        consumer._receive_callback({'foo': 'bar'})
        self.assertIn('basic_reject:requeue', channel)
예제 #26
0
    def test_basic_reject__requeue(self):
        channel = self.connection.channel()
        b1 = Queue("qname1", self.exchange, "rkey")
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.requeue()

        consumer.register_callback(callback)
        consumer._receive_callback({"foo": "bar"})
        self.assertIn("basic_reject:requeue", channel)
예제 #27
0
    def test_receive_callback_accept(self):
        message = Mock(name='Message')
        message.errors = []
        callback = Mock(name='on_message')
        c = Consumer(self.connection, accept=['json'], on_message=callback)
        c.on_decode_error = None
        c.channel = Mock(name='channel')
        c.channel.message_to_python = None

        c._receive_callback(message)
        callback.assert_called_with(message)
        assert message.accept == c.accept
예제 #28
0
    def test_basic_ack_twice(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.ack()
            message.ack()

        consumer.register_callback(callback)
        with self.assertRaises(MessageStateError):
            consumer._receive_callback({'foo': 'bar'})
예제 #29
0
    def test_basic_reject_twice(self):
        channel = self.connection.channel()
        b1 = Queue("qname1", self.exchange, "rkey")
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.reject()
            message.reject()

        consumer.register_callback(callback)
        with self.assertRaises(MessageStateError):
            consumer._receive_callback({"foo": "bar"})
        self.assertIn("basic_reject", channel)
예제 #30
0
    def test_basic_reject(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.reject()

        consumer.register_callback(callback)
        consumer._receive_callback({'foo': 'bar'})
        assert 'basic_reject' in channel
예제 #31
0
    def purge(self, timeout: int = 5) -> None:

        def _purge_errors(exc, interval):
            self._logger.error('Purging error: %s, will retry triggering in %s seconds', exc, interval, exc_info=True)

        def _purge_messages(cnsmr: BrightsideConsumer):
            cnsmr.purge()
            self._message = None

        connection = BrokerConnection(hostname=self._amqp_uri)
        with connections[connection].acquire(block=True) as conn:
            self._logger.debug('Got connection: %s', conn.as_uri())
            with Consumer([self._queue], callbacks=[_purge_messages]) as consumer:
                ensure_kwargs = self.RETRY_OPTIONS.copy()
                ensure_kwargs['errback'] = _purge_errors
                safe_purge = conn.ensure(consumer, _purge_messages, **ensure_kwargs)
                safe_purge(consumer)
예제 #32
0
 def run(self):
     queue_write_db = Queue(name='dbwriter.request.api_write_db',
                            exchange=self.exchange,
                            routing_key='dbwriter.request.api_write_db')
     while 1:
         try:
             self.consumer_connection.ensure_connection(max_retries=1)
             with Consumer(self.consumer_connection,
                           queues=queue_write_db,
                           callbacks=[self.api_write_db],
                           no_ack=True):
                 while True:
                     self.consumer_connection.drain_events()
         except (ConnectionRefusedError, exceptions.OperationalError):
             print('Connection lost')
         except self.consumer_connection.connection_errors:
             print('Connection error')
예제 #33
0
 def call(self, key):
     self.response = None
     self.correlation_id = uuid()
     with Producer(self.connection) as producer:
         producer.publish({'key': key},
                          exchange='',
                          routing_key='db.read',
                          declare=[self.callback_queue],
                          reply_to=self.callback_queue.name,
                          correlation_id=self.correlation_id)
     with Consumer(self.connection,
                   on_message=self.on_response,
                   queues=[self.callback_queue],
                   no_ack=True):
         while self.response is None:
             self.connection.drain_events()
     return self.response['result']
예제 #34
0
    def test_basic_ack_twice(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.ack()
            message.ack()

        consumer.register_callback(callback)
        with self.assertRaises(MessageStateError):
            consumer._receive_callback({'foo': 'bar'})
예제 #35
0
    def test_receive_callback_accept(self):
        message = Mock(name='Message')
        message.errors = []
        callback = Mock(name='on_message')
        c = Consumer(self.connection, accept=['json'], on_message=callback)
        c.on_decode_error = None
        c.channel = Mock(name='channel')
        c.channel.message_to_python = None

        c._receive_callback(message)
        callback.assert_called_with(message)
        self.assertSetEqual(message.accept, c.accept)
예제 #36
0
def consumer_ack_wait():
    connection, channel = getChannelAndConnection()
    exchange = Exchange(CONF.region_id + "_ack", 'direct', channel)
    queue = Queue(CONF.region_id + "_ack",
                  exchange=exchange,
                  routing_key=CONF.region_id + "_ack",
                  channel=channel)
    consumer = Consumer(channel, queues=[queue], callbacks=[process_ack_msg])
    consumer.consume()
    while True:
        connection.drain_events()
    consumer.cancel()
예제 #37
0
파일: API.py 프로젝트: tovin07/K59_training
def get_things_info_by_platform_id(platform_id, thing_status, item_status):

    print(
        "API get things info in platform_id with thing_status and item_status")

    if (thing_status in ["active", "inactive", "all"]) \
            and (item_status in ["active", "inactive", "all"]):

        message_request = {
            'reply_to': 'registry.response.api.api_get_things_by_platform_id',
            'thing_status': thing_status,
            'item_status': item_status,
            'platform_id': platform_id
        }

        # request to api_get_things of Registry
        queue_response = Queue(
            name='registry.response.api.api_get_things_by_platform_id',
            exchange=exchange,
            routing_key='registry.response.api.api_get_things_by_platform_id')
        request_routing_key = 'registry.request.api_get_things_by_platform_id'
        rabbitmq_connection.ensure_connection()
        with Producer(rabbitmq_connection) as producer:
            producer.publish(json.dumps(message_request),
                             exchange=exchange.name,
                             routing_key=request_routing_key,
                             declare=[queue_response],
                             retry=True)

        message_response = None

        def on_response(body, message):
            nonlocal message_response
            message_response = json.loads(body)

        with Consumer(rabbitmq_connection,
                      queues=queue_response,
                      callbacks=[on_response],
                      no_ack=True):
            while message_response is None:
                rabbitmq_connection.drain_events()

        return message_response
    else:
        return None
예제 #38
0
 def call(self, n):
     self.response = None
     self.correlation_id = uuid()
     with Producer(self.connection) as producer:
         producer.publish(
             {'n': n},
             exchange='',
             routing_key='rpc_queue',
             declare=[self.callback_queue],
             reply_to=self.callback_queue,
             correlation_id=self.correlation_id,
         )
     with Consumer(self.connection,
                   on_message=self.on_response,
                   queues=[self.callback_queue], no_ack=True):
         while self.response is None:
             self.connection.drain_events()
     return self.response
 def start_consuming(
     self,
     callback: Callable,
     queue_name: str,
     prefetch_count: int = 1,
     no_ack: bool = False,
     expires: int = None,
     callback_ready: Callable = None,
 ):
     if self._logger is not None:
         self._logger.debug("Start consuming queue: %s" % queue_name)
     self._consuming = True
     while self._consuming:
         revived_connection = self._connection.clone()
         revived_connection.ensure_connection()
         channel = revived_connection.channel()
         channel.basic_qos(0, prefetch_count, True)
         queues = []
         queue_obj = Queue(
             channel=channel,
             name=queue_name,
             no_ack=no_ack,
             durable=False,
             expires=expires,
             queue_arguments={"x-max-priority": 255},
         )
         queue_obj.declare()
         queues.append(queue_obj)
         consumer = Consumer(
             revived_connection,
             queues,
             callbacks=[callback],
             accept=["json"],
             auto_declare=False,
             prefetch_count=prefetch_count,
         )
         consumer.revive(channel)
         consumer.consume()
         while self._consuming:
             callback_ready is not None and callback_ready()
             try:
                 revived_connection.drain_events(timeout=2)
             except socket.timeout:
                 revived_connection.heartbeat_check()
             except self._connection.connection_errors + (
                 AMQPError,
                 ConnectionForced,
                 ConnectionError,
             ):  # pragma: no cover
                 if self._logger is not None:
                     self._logger.exception("Connection error", stack_info=True)
                 break
예제 #40
0
 def send_task(self,
               payload,
               routing_key,
               local=False,
               get_response=False,
               timeout=10):
     if local:
         declare_queues = self.control_queues
     else:
         declare_queues = self.declare_queues
     reply_to = None
     callback_queue = []
     if get_response:
         reply_to = self.callback_queue.name
         callback_queue = [self.callback_queue]
         self.correlation_id = uuid()
     try:
         with producers[self.connection].acquire(block=True,
                                                 timeout=10) as producer:
             producer.publish(
                 payload,
                 exchange=None if local else self.exchange,
                 declare=declare_queues,
                 routing_key=routing_key,
                 reply_to=reply_to,
                 correlation_id=self.correlation_id,
                 retry=True,
                 headers={'epoch': time.time()},
             )
         if get_response:
             with Consumer(self.connection,
                           on_message=self.on_response,
                           queues=callback_queue,
                           no_ack=True):
                 while self.response is self._response:
                     self.connection.drain_events(timeout=timeout)
             return self.response
     except socket.timeout:
         log.exception(
             "Error waiting for task: '%s' sent with routing key '%s'",
             payload, routing_key)
     except Exception:
         log.exception("Error queueing async task: '%s'. for %s", payload,
                       routing_key)
예제 #41
0
파일: actions.py 프로젝트: leopoul/artemis
 def send(self, hijack_key, prefix, type_, hijack_as, action):
     log.debug(
         "Send 'learn_new_rule - {0}' hijack message with key: {1}".format(
             action, hijack_key))
     self.response = None
     self.correlation_id = uuid()
     callback_queue = Queue(
         uuid(),
         durable=False,
         exclusive=True,
         auto_delete=True,
         max_priority=4,
         consumer_arguments={"x-priority": 4},
     )
     with Connection(RABBITMQ_URI) as connection:
         with Producer(connection) as producer:
             producer.publish(
                 {
                     "key": hijack_key,
                     "prefix": prefix,
                     "type": type_,
                     "hijack_as": hijack_as,
                     "action": action,
                 },
                 exchange="",
                 routing_key="configuration.rpc.hijack-learn-rule",
                 retry=True,
                 declare=[callback_queue],
                 reply_to=callback_queue.name,
                 correlation_id=self.correlation_id,
                 priority=4,
                 serializer="ujson",
             )
         with Consumer(
                 connection,
                 on_message=self.on_response,
                 queues=[callback_queue],
                 accept=["ujson"],
         ):
             while self.response is None:
                 connection.drain_events()
     if self.response["success"]:
         return self.response["new_yaml_conf"], True
     return self.response["new_yaml_conf"], False
예제 #42
0
파일: mq.py 프로젝트: ndrlslz/ternya
    def create_consumer(self, exchange_name, queue_name, process):
        try:
            channel = self.connection.channel()
            exchange = Exchange(exchange_name, type="topic")
            queue = Queue(queue_name, exchange, routing_key="notifications.#")
            consumer = Consumer(channel, queue)
            consumer.register_callback(process)
            consumer.consume()
            log.info("create consumer: " + repr(consumer))
        except OSError:
            raise MQConnectionError("please check your mq user, password and host configuration.")

        return self.connection
예제 #43
0
파일: worker.py 프로젝트: ritasker/Paramore
    def _read_monitoring_messages(self):

        def _drain(cnx, timeout):
            try:
                cnx.drain_events(timeout=timeout)
            except kombu_exceptions.TimeoutError:
                pass

        def _drain_errors(exc, interval):
            self._logger.error('Draining error: %s, will retry triggering in %s seconds', exc, interval, exc_info=True)

        def _read_message(body, message):
            self._logger.debug("Monitoring event received at: %s headers: %s payload: %s", datetime.utcnow().isoformat(), message.headers, message.payload)
            now = datetime.utcnow().isoformat()
            activity = body
            print("{time}: {event}".format(time=now, event=activity))
            message.ack()

        # read the next batch number of monitoring messages from the control bus
        # evaluate for color coding (error is red)
        # print to stdout

        connection = BrokerConnection(hostname=self._amqp_uri)
        with connections[connection].acquire(block=True) as conn:
            self._logger.debug('Got connection: %s', conn.as_uri())
            with Consumer(conn, [self._monitoring_queue], callbacks=[_read_message], accept=['json', 'text/plain']) as consumer:
                self._running.set()
                ensure_kwargs = self.RETRY_OPTIONS.copy()
                ensure_kwargs['errback'] = _drain_errors
                lines = 0
                updates = 0
                while self._running.is_set():
                    # page size number before we sleep
                    safe_drain = conn.ensure(consumer, _drain, **ensure_kwargs)
                    safe_drain(conn, DRAIN_EVENTS_TIMEOUT)
                    lines += 1
                    if lines == self.page_size:
                        if self.limit != -1 and updates > self.limit:
                            self._running.clear()
                        else:
                            sleep(self.delay_between_refreshes)
                            lines = 0
                            updates += 1
예제 #44
0
 def get_consumers(self, channel):
     ep = settings.RABBIT_SETTINGS['RabbitEndpoints']['Events']
     ep_tasks = settings.RABBIT_SETTINGS['RabbitEndpoints']['Tasks']
     event_queue = Queue(
         ep['Queue'],
         Exchange(name=ep['Exchange'],
                  type=ep['ExchangeType'],
                  durable=False), ep['RoutingKey'])
     task_queue = Queue(
         ep_tasks['Queue'],
         Exchange(name=ep_tasks['Exchange'],
                  type=ep_tasks['ExchangeType'],
                  durable=False), ep_tasks['RoutingKey'])
     return [
         Consumer(channel,
                  queues=[event_queue],
                  callbacks=[self.handle_message],
                  accept=['json'])
     ]
예제 #45
0
    def receive_event(self):
        def handle_notification(body, message):
            self.logger.info("Receive Event!")
            # event_name = json.loads(body)["event_name"]
            event_id = json.loads(body)["event_id"]
            event_source = json.loads(body)["event_source"]
            trigger_id = json.loads(body)["trigger_id"]
            # event_generator_id = json.loads(body)["event_generator_id"]
            time = json.loads(body)["time"]

            # mapping trigger_id to condition_id and action_id
            condition_id, condition_type, condition_content, action_id, action_content = self.mapping(
                trigger_id)

            # Check if condition has condition_id is true
            is_condition_satisfice = self.check_condition(
                condition_id, condition_type, condition_content)

            self.logger.info("check condition result: " +
                             str(is_condition_satisfice))

            if (is_condition_satisfice == True):
                # Execute an action
                self.call_to_action(action_id, action_content)

            self.logger.info("Finish receive one event!\n\n\n")
            # End handle_notification

        try:
            self.consumer_connection.ensure_connection(max_retries=1)
            with nested(
                    Consumer(self.consumer_connection,
                             queues=self.queue_get_states,
                             callbacks=[handle_notification],
                             no_ack=True)):
                while True:
                    self.consumer_connection.drain_events()
        except (ConnectionRefusedError, exceptions.OperationalError):
            self.logger.error("Connection lost", exc_info=True)
        except self.consumer_connection.connection_errors:
            self.logger.error("Connection error", exc_info=True)
        except Exception as e:
            self.logger.error("Error receive event", exc_info=True)
예제 #46
0
파일: observer.py 프로젝트: leopoul/artemis
        def check_changes(self):
            with open(self.path, "r") as f:
                content = f.readlines()
            # Taken any action here when a file is modified.
            changes = "".join(difflib.unified_diff(self.content, content))
            if changes:
                self.response = None
                self.correlation_id = uuid()
                callback_queue = Queue(
                    uuid(),
                    durable=False,
                    auto_delete=True,
                    max_priority=4,
                    consumer_arguments={"x-priority": 4},
                )
                with Producer(self.connection) as producer:
                    producer.publish(
                        content,
                        exchange="",
                        routing_key="configuration.rpc.modify",
                        serializer="yaml",
                        retry=True,
                        declare=[callback_queue],
                        reply_to=callback_queue.name,
                        correlation_id=self.correlation_id,
                        priority=4,
                    )
                with Consumer(
                        self.connection,
                        on_message=self.on_response,
                        queues=[callback_queue],
                        accept=["ujson"],
                ):
                    while self.response is None:
                        self.connection.drain_events()

                if self.response["status"] == "accepted":
                    text = "new configuration accepted:\n{}".format(changes)
                    log.info(text)
                    self.content = content
                else:
                    log.error("invalid configuration:\n{}".format(content))
                self.response = None
예제 #47
0
def main():
    parser = ArgumentParser(
        description="Spam a particular individual in `spam_domain`")
    parser.add_argument("--name",
                        help='The local name of the email to spam',
                        default='randomuser')
    args = parser.parse_args()

    spammer = Spammer(args.name)
    with Connection(config.get_rabbit_url()) as connection:
        main_queue = Queue(args.name,
                           exchange=spammer_exchange,
                           routing_key=args.name)
        with Consumer(connection,
                      queues=[main_queue],
                      callbacks=[spammer.spam],
                      prefetch_count=1):
            while True:
                connection.drain_events()  # start consuming
예제 #48
0
def run():
    queue_notification = Queue(name='monitor.request.alert',
                               exchange=exchange,
                               routing_key='monitor.request.alert')

    while 1:
        try:
            consumer_connection.ensure_connection(max_retries=1)
            with nested(
                    Consumer(consumer_connection,
                             queues=queue_notification,
                             callbacks=[handle_notification],
                             no_ack=True)):
                while True:
                    consumer_connection.drain_events()
        except (ConnectionRefusedError, exceptions.OperationalError):
            print('Connection lost')
        except consumer_connection.connection_errors:
            print('Connection error')
예제 #49
0
    def ingest(self):
        """
        Monitor AMQP for messages

        """

        # Define our RabbitMQ route
        if self.stoq.worker.source_queue:
            routing_key = self.stoq.worker.source_queue
        else:
            routing_key = self.stoq.worker.name

        # If this is an error message, let's make sure our queue
        # has "-errors" affixed to it
        if self.stoq.worker.error_queue is True:
            routing_key = routing_key + "-errors".strip()

        exchange = Exchange(self.exchange_name, type=self.exchange_type)
        queue_arguments = {'x-max-priority': 10}
        queue = Queue(routing_key,
                      exchange,
                      routing_key=routing_key,
                      queue_arguments=queue_arguments)

        self.log.info(
            "Monitoring {} queue for messages...".format(routing_key))

        # Setup our broker connection with RabbitMQ
        with Connection(hostname=self.host,
                        port=self.port,
                        userid=self.user,
                        password=self.password,
                        virtual_host=self.virtual_host,
                        ssl=self.ssl_config) as conn:

            conn.connect()

            consumer = Consumer(conn, queue, callbacks=[self.queue_callback])
            consumer.qos(prefetch_count=int(self.prefetch))
            consumer.consume()

            while True:
                try:
                    conn.drain_events()
                except Exception as err:
                    self.log.critical(
                        "Unable to process queue: {}".format(err),
                        exc_info=True)
                    conn.release()
                    break
예제 #50
0
    def __init__(self):
        client_id = settings.DISCORD_APP_ID

        intents = discord.Intents.default()
        intents.members = True

        super().__init__(
            command_prefix=DISCORD_BOT_PREFIX,
            description=description,
            intents=intents,
        )

        self.redis = None
        self.redis = self.loop.run_until_complete(
            aioredis.create_pool(getattr(settings, "BROKER_URL",
                                         "redis://localhost:6379/0"),
                                 minsize=5,
                                 maxsize=10))
        print('redis pool started', self.redis)
        self.client_id = client_id
        self.session = aiohttp.ClientSession(loop=self.loop)
        self.tasks = []

        self.message_connection = Connection(
            getattr(settings, "BROKER_URL", 'redis://localhost:6379/0'))
        queues = []
        for que in queue_keys:
            queues.append(Queue(que))
        self.message_consumer = Consumer(self.message_connection,
                                         queues,
                                         callbacks=[self.on_queue_message],
                                         accept=['json'])

        django.setup()

        for hook in hooks.get_hooks("discord_cogs_hook"):
            for cog in hook():
                try:
                    self.load_extension(cog)
                except Exception as e:
                    print(f"Failed to load cog {cog}", file=sys.stderr)
                    traceback.print_exc()
예제 #51
0
        def on_modified(self, event):
            if event.is_directory:
                return None
            elif event.src_path == self.path:
                with open(self.path, 'r') as f:
                    content = f.readlines()
                # Taken any action here when a file is modified.
                changes = ''.join(difflib.unified_diff(self.content, content))
                if len(changes) > 0:
                    self.response = None
                    self.correlation_id = uuid()
                    callback_queue = Queue(
                        uuid(),
                        durable=False,
                        auto_delete=True,
                        max_priority=4,
                        consumer_arguments={'x-priority': 4})
                    with Producer(self.connection) as producer:
                        producer.publish(content,
                                         exchange='',
                                         routing_key='config-modify-queue',
                                         serializer='yaml',
                                         retry=True,
                                         declare=[callback_queue],
                                         reply_to=callback_queue.name,
                                         correlation_id=self.correlation_id,
                                         priority=4)
                    with Consumer(self.connection,
                                  on_message=self.on_response,
                                  queues=[callback_queue],
                                  no_ack=True):
                        while self.response is None:
                            self.connection.drain_events()

                    if self.response['status'] == 'accepted':
                        text = 'new configuration accepted:\n{}'.format(
                            changes)
                        log.info(text)
                        self.content = content
                    else:
                        log.error('invalid configuration:\n{}'.format(content))
                    self.response = None
예제 #52
0
        def config_request_rpc(self) -> NoReturn:
            """
            Initial RPC of this service to request the configuration.
            The RPC is blocked until the configuration service replies back.
            """
            self.correlation_id = uuid()
            callback_queue = Queue(
                uuid(),
                durable=False,
                auto_delete=True,
                max_priority=4,
                consumer_arguments={"x-priority": 4},
            )

            self.producer.publish(
                "",
                exchange="",
                routing_key="config-request-queue",
                reply_to=callback_queue.name,
                correlation_id=self.correlation_id,
                retry=True,
                declare=[
                    Queue(
                        "config-request-queue",
                        durable=False,
                        max_priority=4,
                        consumer_arguments={"x-priority": 4},
                    ),
                    callback_queue,
                ],
                priority=4,
                serializer="ujson",
            )
            with Consumer(
                self.connection,
                on_message=self.handle_config_request_reply,
                queues=[callback_queue],
                accept=["ujson"],
            ):
                while self.rules is None:
                    self.connection.drain_events()
            log.debug("{}".format(self.rules))
예제 #53
0
    def test_produce__consume_large_messages(self,
                                             bytes=1048576,
                                             n=10,
                                             charset=string.punctuation +
                                             string.letters + string.digits):
        if not self.verify_alive():
            return
        bytes = min(filter(None, [bytes, self.message_size_limit]))
        messages = [
            "".join(random.choice(charset) for j in xrange(bytes)) + "--%s" % n
            for i in xrange(n)
        ]
        digests = []
        chan1 = self.connection.channel()
        consumer = Consumer(chan1, self.queue)
        for queue in consumer.queues:
            queue.purge()
        producer = Producer(chan1, self.exchange)
        for i, message in enumerate(messages):
            producer.publish({
                "text": message,
                "i": i
            },
                             routing_key=self.prefix)
            digests.append(self._digest(message))

        received = [(msg["i"], msg["text"])
                    for msg in consumeN(self.connection, consumer, n)]
        self.assertEqual(len(received), n)
        ordering = [i for i, _ in received]
        if ordering != range(n):
            warnings.warn("%s did not deliver messages in FIFO order: %r" %
                          (self.transport, ordering))

        for i, text in received:
            if text != messages[i]:
                raise AssertionError("%i: %r is not %r" %
                                     (i, text[-100:], messages[i][-100:]))
            self.assertEqual(self._digest(text), digests[i])

        chan1.close()
        self.purge([self.queue.name])
예제 #54
0
 def control_with_reply(self, command, timeout=5):
     logger.warn('checking {} {} for {}'.format(self.service, command,
                                                self.queuename))
     reply_queue = Queue(name="amq.rabbitmq.reply-to")
     self.result = None
     with Connection(settings.BROKER_URL) as conn:
         with Consumer(conn,
                       reply_queue,
                       callbacks=[self.process_message],
                       no_ack=True):
             self.publish({'control': command},
                          conn,
                          reply_to='amq.rabbitmq.reply-to')
             try:
                 conn.drain_events(timeout=timeout)
             except socket.timeout:
                 logger.error('{} did not reply within {}s'.format(
                     self.service, timeout))
                 raise
     return self.result
예제 #55
0
    def listen_for_response(self, connection):
        """
        Set up a consumer and listen for a response. If successful, this will
        return the contents of :py:attr:`self.result` which will have been set
        up by the callback.

        :param connection:
           A Kombu Connection instance.

        :return:
            A dictionary containing the result, or None if the request failed.

        """
        self.result = None
        queue = self.get_response_queue(connection)
        with Consumer(connection, queue, callbacks=[self.callback]):
            while self.result is None:
                connection.drain_events(timeout=self.amqp_timeout)

        return self.process_response(self.result)
    def call(self, message: dict, event_type: str, entity_type: str,
             site: Site) -> any:
        self.response = None
        self.correlation_id = uuid()
        with Producer(self.connection) as producer:
            _logger.info(
                f'CELERY RPC call {site.domain} with {event_type} - {entity_type} {message} reply to {self.correlation_id}'
            )
            producer.publish(
                {
                    "event_type": event_type,
                    "entity_type": entity_type,
                    "entity_site": site.domain,
                    "data": message
                },
                exchange='',
                routing_key=self.routing_key,
                declare=[self.callback_queue],
                reply_to=self.callback_queue.name,
                correlation_id=self.correlation_id,
                serializer='msgpack',
            )

        with Consumer(self.connection,
                      on_message=self.on_response,
                      queues=[self.callback_queue],
                      no_ack=True):
            _logger.info(
                f'CELERY RPC call consume {site.domain} with {event_type} - {entity_type} {message} reply to {self.correlation_id}'
            )
            t_current = time.time()
            while self.response is None:
                self.connection.drain_events(timeout=1)
                # time.sleep(0.25)  # sleep for 250 milliseconds
                # if time.time() >= t_current + 60000:
                #     break
        _logger.info(
            f'CELERY RPC call consume {site.domain} with response {self.response}'
        )

        return self.response
예제 #57
0
 def consume(self, conn, queue):
     with conn.channel() as channel:
         consumer = Consumer(
             channel=channel,
             queues=[queue],
             accept=['json'],
         )
         consumer.register_callback(self._process_message)
         consumer.consume()
         while True:
             try:
                 conn.drain_events(timeout=5)
             except socket.timeout:
                 self.app.logger.debug('Timeout waiting for events')
                 conn.heartbeat_check()
 def call(self, method):
     if method not in ALLOWED_METHODS:
         raise Exception("Method '{}' not allowed".format(method))
     self.response = None
     self.correlation_id = uuid()
     with Producer(self.connection) as producer:
         producer.publish(
             {'method': method},
             exchange=self.exchange,
             routing_key=self.routing_key,
             declare=[self.callback_queue],
             reply_to=self.callback_queue.name,
             correlation_id=self.correlation_id,
         )
     with Consumer(self.connection,
                   on_message=self.on_response,
                   queues=[self.callback_queue],
                   no_ack=True):
         while self.response is None:
             self.connection.drain_events()
     return self.response
예제 #59
0
    def send_request(self, fun, args, kwargs):
        payload = {'fun': fun, 'args': args, 'kwargs': kwargs}

        with Producer(self.connection) as producer:
            producer.publish(
                payload,
                exchange='',
                routing_key='rpc_queue',
                declare=[self.callback_queue],
                reply_to=self.callback_queue.name,
                correlation_id=self.correlation_id,
            )

        # 接受服务器发布的消息, 接受到后进入 on_response 回调
        with Consumer(self.connection,
                      on_message=self.on_response,
                      queues=[self.callback_queue],
                      no_ack=True):
            while self.response is None:
                self.connection.drain_events()
        return self.response