def test_drain_nowait(self): c = Connection(transport=Mock) c.drain_events = Mock() c.drain_events.side_effect = socket.timeout() c.more_to_read = True self.assertFalse(c.drain_nowait()) self.assertFalse(c.more_to_read) c.drain_events.side_effect = socket.error() c.drain_events.side_effect.errno = errno.EAGAIN c.more_to_read = True self.assertFalse(c.drain_nowait()) self.assertFalse(c.more_to_read) c.drain_events.side_effect = socket.error() c.drain_events.side_effect.errno = errno.EPERM with self.assertRaises(socket.error): c.drain_nowait() c.more_to_read = False c.drain_events = Mock() self.assertTrue(c.drain_nowait()) c.drain_events.assert_called_with(timeout=0) self.assertTrue(c.more_to_read)
def test_drain_nowait(self): c = Connection(transport=Mock) c.drain_events = Mock() c.drain_events.side_effect = socket.timeout() c.more_to_read = True self.assertFalse(c.drain_nowait()) self.assertFalse(c.more_to_read) c.drain_events.side_effect = socket.error() c.drain_events.side_effect.errno = errno.EAGAIN c.more_to_read = True self.assertFalse(c.drain_nowait()) self.assertFalse(c.more_to_read) c.drain_events.side_effect = socket.error() c.drain_events.side_effect.errno = errno.EPERM with self.assertRaises(socket.error): c.drain_nowait() c.more_to_read = False c.drain_events = Mock() self.assertTrue(c.drain_nowait()) c.drain_events.assert_called_with(timeout=0) self.assertTrue(c.more_to_read)
class KombuConsumer( AbstractConsumer, ): """ 使用kombu作为中间件,这个能直接一次性支持很多种小众中间件,但性能很差,除非是分布式函数调度框架没实现的中间件种类用户才可以用这种,用户也可以自己对比性能。 """ BROKER_KIND = 15 def custom_init(self): self._middware_name = frame_config.KOMBU_URL.split(":")[0] logger_name = f'{self._logger_prefix}{self.__class__.__name__}--{self._middware_name}--{self._queue_name}' self.logger = LogManager(logger_name).get_logger_and_add_handlers( self._log_level, log_filename=f'{logger_name}.log' if self._create_logger_file else None, formatter_template=frame_config. NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER, ) # patch_kombu_redis() # noinspection DuplicatedCode def _shedual_task(self): # 这个倍while 1 启动的,会自动重连。 def callback(body: dict, message: kombu.transport.virtual.base.Message): # print(type(body),body,type(message),message) self._print_message_get_from_broker('kombu', body) # self.logger.debug(f""" 从 kombu {self._middware_name} 中取出的消息是 {body}""") kw = { 'body': body, 'message': message, } self._submit_task(kw) self.exchange = Exchange('distributed_framework_exchange', 'direct', durable=True) self.queue = Queue(self._queue_name, exchange=self.exchange, routing_key=self._queue_name, auto_delete=False) self.conn = Connection(frame_config.KOMBU_URL, transport_options={"visibility_timeout": 600}) # 默认3600秒unacked重回队列 self.queue(self.conn).declare() with self.conn.Consumer(self.queue, callbacks=[callback], no_ack=False, prefetch_count=100) as consumer: # Process messages and handle events on all channels channel = consumer.channel # type:Channel channel.body_encoding = 'no_encode' # 这里改了编码,存到中间件的参数默认把消息base64了,我觉得没必要不方便查看消息明文。 while True: self.conn.drain_events() def _confirm_consume(self, kw): pass # redis没有确认消费的功能。 kw['message'].ack() def _requeue(self, kw): kw['message'].requeue()
def run_pulse_listener(username, password, timeout, no_send): """Run a Pulse message queue listener.""" connection = Connection( hostname='pulse.mozilla.org', port=5671, ssl=True, userid=username, password=password, ) # Connect and pass in our own low value for retries so the connection # fails fast if there is a problem. connection.ensure_connection( max_retries=1 ) # Retries must be >=1 or it will retry forever. with closing(connection): hgpush_exchange = Exchange(config.PULSE_EXCHANGE, 'topic', channel=connection) # Pulse queue names need to be prefixed with the username queue_name = f'queue/{username}/{config.PULSE_QUEUE_NAME}' queue = Queue( queue_name, exchange=hgpush_exchange, routing_key=config.PULSE_QUEUE_ROUTING_KEY, durable=True, exclusive=False, auto_delete=False, channel=connection, ) # Passing passive=True will assert that the exchange exists but won't # try to declare it. The Pulse server forbids declaring exchanges. hgpush_exchange.declare(passive=True) # Queue.declare() also declares the exchange, which isn't allowed by # the Pulse server. Use the low-level Queue API to only declare the # queue itself. queue.queue_declare() queue.queue_bind() callback = partial(process_push_message, no_send=no_send) # Pass auto_declare=False so that Consumer does not try to declare the # exchange. Declaring exchanges is not allowed by the Pulse server. with connection.Consumer( queue, callbacks=[callback], auto_declare=False ) as consumer: if no_send: log.info('transmission of ping data has been disabled') log.info('message acks has been disabled') log.info('reading messages') try: connection.drain_events(timeout=timeout) except socket.timeout: log.info('message queue is empty, nothing to do') log.info('done')
def run(self): connection = Connection(**connection_params) channel = connection.channel() channel.basic_qos(prefetch_size=0, prefetch_count=1, a_global=True) while True: with Consumer(channel, queues, callbacks=[self.on_message]): connection.drain_events()
def consumer(): rabbit_url = "amqp://localhost:5672/" conn = Connection(rabbit_url) exchange = Exchange("scrapy", type="direct") queue = Queue(name="quotation", exchange=exchange, routing_key="quotes") with Consumer(conn, queues=queue, callbacks=[process_message], accept=["json"]): conn.drain_events(timeout=2) return "consumed successfully"
class _AMQPServerWrapper(object): def __init__(self, amqp_url, logs): self.__logs = logs self.__amqp_url = amqp_url self.__monitors = {} self.__connection = Connection(self.__amqp_url) self.__connection.connect() self.__running = True self.__consumer_gl = gevent.spawn(self.__consumer_greenlet_main) self.__consumer_gl.greenlet_name = 'amqp-consumer-gl' # allowing flogging to print a nice name gevent.sleep(0.0) def __consumer_greenlet_main(self): gevent.sleep(0) while self.__running: try: self.__connection.drain_events(timeout=0.5) except Exception as ex: # NOQA: assigned but not used (left in for super-duper-low-level-debug) # print("was woken because {}".format(ex)) pass gevent.sleep(0.1) # make -sure- to yield cpu... # print("---loop") def stop_greenlet(self): self.__running = False @property def connected(self): return self.__connection.connected def create_add_tracker(self, exchange, routing_key, event_cb, queue_name=None): self.__logs.irl.debug("AMQPServerWrapper: create_add_tracker ex=%s, rk=%s, event_cb=%s", exchange, routing_key, event_cb) mon = _KeyedConsumerHandler.get_keyed_consumer( self.__logs, self.__connection, exchange, routing_key, queue_name, event_cb) return mon.exchange def inject(self, exchange, routing_key, payload): self.__logs.irl.debug("Injecting a test AMQP message: ex=%s, rk=%s, payload=%s", exchange, routing_key, payload) if not isinstance(exchange, Exchange): exchange = Exchange(exchange, 'topic') prod = Producer(self.__connection, exchange=exchange, routing_key=routing_key) prod.publish(payload) def test_helper_sync_send_msg(self, exchange, ex_rk, send_rk, payload): ex = Exchange(exchange, 'topic') queue = Queue(exchange=ex, routing_key=ex_rk + '.*', exclusive=True, channel=self.__connection) queue.declare() prod = Producer(self.__connection, exchange=ex, routing_key=send_rk) prod.publish(payload) return queue def test_helper_sync_recv_msg(self, queue): for tick in range(10): msg = queue.get() if msg is not None: break return msg
def worker(mq_url): connection = Connection(mq_url) channel = connection.channel() consumer_json = Consumer(channel, task_json_queue, callbacks=[process_json], accept=["json"]) consumer_json.consume() consumer_pickle = Consumer(channel, task_pickle_queue, callbacks=[process_pickle], accept=["pickle"]) consumer_pickle.consume() while True: connection.drain_events()
class ClientMQ(object): connection = None nodeName = None callback = None def __init__(self, host, vhost, username, password, nodeName, callback): self.init_connect(host, vhost, username, password) self.nodeName = nodeName self.callback = callback self.queue = ThreadQueue() self.produder = ClientProducer(self.connection, 'HOST') self.initRecv() self.startThread() def init_connect(self, host, vhost, username, password): self.connection = Connection(hostname=host, virtual_host=vhost, userid=username, password=password) def initRecv(self): self.consumer = ClientConsumer(self.connection, self.nodeName, self.callback) while not self.consumer.init(): time.sleep(1) def start_consume(): while True: self.connection.drain_events() t = Thread(target=start_consume) t.start() def startThread(self): def start_produce(): while True: try: msg = self.queue.get() while not self.produder.publish(message=msg): time.sleep(1) except Exception as e: logger.exception(e) t = Thread(target=start_produce) t.start() def is_open(self): if self.connection: return self.connection.connected else: return False def update_bean(self, cls_name, content): msg = '1' + to_unicode(cls_name) + ':' + to_unicode(content) self.queue.put(msg) def delete_bean(self, cls_name, ids): msg = '0' + to_unicode(cls_name) + ':' + to_unicode(ids) self.queue.put(msg)
class DBwriter: def __init__(self, broker_cloud, host_influxdb): self.clientDB = InfluxDBClient(host_influxdb, 8086, 'root', 'root', 'Collector_DB') self.clientDB.create_database('Collector_DB') self.consumer_connection = Connection(broker_cloud) self.exchange = Exchange("IoT", type="direct") def write_db(self, data_points): print(len(data_points)) for point in data_points: record = [{ 'measurement': point['MetricId'], 'tags': { 'DataType': point['DataType'], }, 'fields': { 'Value': point['Value'], }, 'time': point['TimeCollect'] }] try: self.clientDB.write_points(record) print(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ': Updated Database') except: print("Can't write to database : {}".format(point['MetricId'])) print("Delete mesurement....") self.clientDB.drop_measurement(measurement=point['MetricId']) def api_write_db(self, body, message): data_points = json.loads(body)['body']['data_points'] self.write_db(data_points) def run(self): queue_write_db = Queue(name='dbwriter.request.api_write_db', exchange=self.exchange, routing_key='dbwriter.request.api_write_db', message_ttl=20) while 1: try: self.consumer_connection.ensure_connection(max_retries=1) with Consumer(self.consumer_connection, queues=queue_write_db, callbacks=[self.api_write_db], no_ack=True): while True: self.consumer_connection.drain_events() except (ConnectionRefusedError, exceptions.OperationalError): print('Connection lost') except self.consumer_connection.connection_errors: print('Connection error')
def listen(self): logger.debug('AutophonePulseMonitor: start shared_lock.acquire') connection = None restart = True while restart: restart = False self.shared_lock.acquire() try: # connection does not connect to the server until # either the connection.connect() method is called # explicitly or until kombu calls it implicitly as # needed. connection = Connection(hostname=self.hostname, userid=self.userid, password=self.password, virtual_host=self.virtual_host, port=DEFAULT_SSL_PORT, ssl=True) consumer = connection.Consumer(self.queues, callbacks=[self.handle_message], accept=['json'], auto_declare=False) for queue in self.queues: queue(connection).queue_declare(passive=False) queue(connection).queue_bind() with consumer: while not self._stopping.is_set(): try: logger.debug('AutophonePulseMonitor shared_lock.release') self.shared_lock.release() connection.drain_events(timeout=self.timeout) except socket.timeout: pass except KeyboardInterrupt: raise finally: logger.debug('AutophonePulseMonitor shared_lock.acquire') self.shared_lock.acquire() logger.debug('AutophonePulseMonitor.listen: stopping') except: logger.exception('AutophonePulseMonitor Exception') if connection: connection.release() restart = True time.sleep(1) finally: logger.debug('AutophonePulseMonitor exit shared_lock.release') if connection and not restart: connection.release() self.shared_lock.release()
def test_accept__content_disallowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], callbacks=[callback]) as consumer: with self.assertRaises(consumer.ContentDisallowed): conn.drain_events(timeout=1) self.assertFalse(callback.called)
def test_accept__content_disallowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], callbacks=[callback]) as consumer: with self.assertRaises(consumer.ContentDisallowed): conn.drain_events(timeout=1) callback.assert_not_called()
def worker(mq_url): connection = Connection(mq_url) channel = connection.channel() consumer_json = Consumer(channel, task_json_queue, callbacks=[process_json], accept=['json']) consumer_json.consume() consumer_pickle = Consumer(channel, task_pickle_queue, callbacks=[process_pickle], accept=['pickle']) consumer_pickle.consume() while True: connection.drain_events()
def run(self): try: connection = Connection(hostname=self.host,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.virtual_host) channel = connection.channel() self.producer=Producer(channel) task_queue = Queue(self.queue_name,durable=True) consumer = Consumer(channel,task_queue,no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(self.RequestCallBack) consumer.consume() while True: connection.drain_events() connection.close() except BaseException,e: print e
def test_accept__content_allowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], accept=['pickle'], callbacks=[callback]): conn.drain_events(timeout=1) callback.assert_called() body, message = callback.call_args[0] self.assertTrue(body['complex'])
def test_accept__content_allowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], accept=['pickle'], callbacks=[callback]): conn.drain_events(timeout=1) self.assertTrue(callback.called) body, message = callback.call_args[0] self.assertTrue(body['complex'])
class Consumer: def __init__(self, queue): self.conn = Connection('amqp://*****:*****@rabbit:5672//') self.task_queue = Queue(queue, routing_key=queue) def start_consumer(self): consumer = Consumer(self.conn, [self.task_queue], callbacks=[self.process_message]) consumer.consume() self.conn.drain_events() @staticmethod def process_message(body, message): print(body['message']['Value'], flush=True) message.ack()
def consumer(): rabbit_host = "amqp://{host}:5672/".format(host=sys.argv[1]) print("Connection to {host}".format(host=rabbit_host)) # Create the connection conn = Connection(rabbit_host) # Create the exchange test_exchange = Exchange("test_exchange", type="direct") # Create the queue queue = Queue(name="queue", exchange=test_exchange, routing_key="test") # Create the consumer with Consumer(conn, queues=queue, callbacks=[process_message], accept=["text/plain"]): conn.drain_events()
def start(self): log.info("Listening for Pulse messages") self.running = True connection = Connection( hostname=self.pulse_host, userid=self.pulse_user, password=self.pulse_password, ssl=True, # Kombu doesn't support the port correctly for amqp with ssl... port=5671, ) consumers = [] for event in self.events: log.debug("Setting up queue on exchange: %s with routing_key: %s", event.exchange, event.routing_key) # Passive exchanges must be used, otherwise kombu will try to # create the exchange (which we don't want, we're consuming # an existing one!) e = Exchange(name=event.exchange, type="topic", passive=True) q = Queue( name=event.queue_name, exchange=e, routing_key=event.routing_key, durable=True, exclusive=False, auto_delete=False ) c = connection.Consumer( queues=[q], callbacks=[event.callback] ) c.consume() consumers.append(c) try: # XXX: drain_events only returns after receiving a message. Is # there a way we can have it return regularly to be non-blocking? # Its timeout parameter seems to break receiving of messages. # Maybe it doesn't matter if we can't shut down gracefully since # messages will be reprocessed next time. while self.running: connection.drain_events() finally: for c in consumers: c.close() connection.close()
def start(self): log.info("Listening for Pulse messages") self.running = True connection = Connection( hostname=self.pulse_host, userid=self.pulse_user, password=self.pulse_password, ssl=True, # Kombu doesn't support the port correctly for amqp with ssl... port=5671, ) consumers = [] for event in self.events: log.debug("Setting up queue on exchange: %s with routing_key: %s", event.exchange, event.routing_key) # Passive exchanges must be used, otherwise kombu will try to # create the exchange (which we don't want, we're consuming # an existing one!) e = Exchange(name=event.exchange, type="topic", passive=True) q = Queue( name=event.queue_name, exchange=e, routing_key=event.routing_key, durable=True, exclusive=False, auto_delete=False ) c = connection.Consumer( queues=[q], callbacks=[event.callback] ) c.consume() consumers.append(c) try: # XXX: drain_events only returns after receiving a message. Is # there a way we can have it return regularly to be non-blocking? # Its timeout parameter seems to break receiving of messages. # Maybe it doesn't matter if we can't shut down gracefully since # messages will be reprocessed next time. while self.running: connection.drain_events() finally: for c in consumers: c.close() connection.close()
class Amqp(object): def __init__(self, url, exchange, queue, routing_key): self.conn = Connection(url) self.exchange = Exchange(exchange, 'direct') self.routing_key = routing_key self.queue = Queue(queue, self.exchange, self.routing_key) self.producer = None self.consumer = None def send(self, obj): if not self.producer: self.producer = self.conn.Producer() self.producer.publish(obj, exchange=self.exchange, routing_key=self.routing_key, declare=[self.queue], serializer='json', compression='zlib') def poll(self, cb_func): if not self.consumer: self.consumer = self.conn.Consumer(self.queue, callbacks=[cb_func]) self.consumer.qos(prefetch_count=1) self.consumer.consume() while True: self.conn.drain_events() def _release(self): if self.consumer: self.consumer.close() self.consumer = None if self.producer: self.producer.close() self.producer = None if self.conn: self.conn.release() self.conn = None def __enter__(self): return self def __exit__(self, exec_type, exc_value, traceback): self._release()
class Actor_1(Actor_Base): def __init__(self, actor_name, actor_id): Actor_Base.__init__(self, actor_name, actor_id) BROKER_CLOUD = "localhost" self.producer_connection = Connection(BROKER_CLOUD) self.consumer_connection = Connection(BROKER_CLOUD) self.exchange = Exchange("IoT", type="direct") self.queue_get_states = Queue(name='event_generator.to.' + str(self.actor_id), exchange=self.exchange, routing_key='event_generator.to.' + str(self.actor_id)) #, message_ttl=20) self.db = MySQLdb.connect(host="0.0.0.0", user="******", passwd="root") self.cursor = self.db.cursor() def execute(self, action_type, action_id): pass def receiveCallToAction(self): def handle_notification(body, message): print("Receive Event!") action_id = json.loads(body)["action_id"] action_name = json.loads(body)["action_name"] action_type = json.loads(body)["action_type"] self.execute(action_type, action_id) print("Executed action: ", action_id) # End handle_notification try: self.consumer_connection.ensure_connection(max_retries=1) with nested( Consumer(self.consumer_connection, queues=self.queue_get_states, callbacks=[handle_notification], no_ack=True)): while True: self.consumer_connection.drain_events() except (ConnectionRefusedError, exceptions.OperationalError): print('Connection lost') except self.consumer_connection.connection_errors: print('Connection error')
class Amqp(object): def __init__(self, url, exchange, queue, routing_key): self.conn = Connection(url) self.exchange = Exchange(exchange, 'direct') self.routing_key = routing_key self.queue = Queue(queue, self.exchange, self.routing_key) self.producer = None self.consumer = None def send(self, obj): if not self.producer: self.producer = self.conn.Producer() self.producer.publish(obj, exchange=self.exchange, routing_key=self.routing_key, declare=[self.queue], serializer='json', compression='zlib') def poll(self, cb_func): if not self.consumer: self.consumer = self.conn.Consumer(self.queue, callbacks=[cb_func]) self.consumer.qos(prefetch_count=1) self.consumer.consume() while True: self.conn.drain_events() def _release(self): if self.consumer: self.consumer.close() self.consumer = None if self.producer: self.producer.close() self.producer = None if self.conn: self.conn.release() self.conn = None def __enter__(self): return self def __exit__(self, exec_type, exc_value, traceback): self._release()
def run(self): connection = Connection(hostname=self.host,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.virtual_host) channel = connection.channel() self.producer=Producer(channel) queueargs={} if self.msg_timeout: queueargs['x-message-ttl']=self.msg_timeout task_queue = Queue(self.queue_name,durable=True,queue_arguments=queueargs if queueargs else None) consumer = Consumer(channel,task_queue,no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(self.RequestCallBack) consumer.consume() while self.task_count: connection.drain_events() self.task_count-=1 connection.close()
def run(self): try: connection = Connection(hostname=self.host, port=self.port, userid=self.usr, password=self.psw, virtual_host=self.virtual_host) channel = connection.channel() self.producer = Producer(channel) task_queue = Queue(self.queue_name, durable=True) consumer = Consumer(channel, task_queue, no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(self.RequestCallBack) consumer.consume() while True: connection.drain_events() connection.close() except BaseException, e: print e
def heartbeat_check(): print("Heartbeat check") rabbit_host = "amqp://{host}:5672/".format(host=sys.argv[1]) print("Connection to {host}".format(host=rabbit_host)) # Create the connection conn = Connection(rabbit_host) # Create the exchange test_exchange = Exchange("test_exchange", type="direct") # Create the queue queue = Queue(name="queue", exchange=test_exchange, routing_key="test") while True: print("hb ping") conn.heartbeat_check(rate=5) conn.drain_events(timeout=0.01) print("hb pong") time.sleep(10)
def listen(self): logger = utils.getLogger() connect_timeout = 5 wait = 30 connection = None restart = True while restart: restart = False try: # connection does not connect to the server until # either the connection.connect() method is called # explicitly or until kombu calls it implicitly as # needed. logger.debug('AutophonePulseMonitor: Connection()') connection = Connection(hostname=self.hostname, userid=self.userid, password=self.password, virtual_host=self.virtual_host, port=DEFAULT_SSL_PORT, ssl=True, connect_timeout=connect_timeout) logger.debug('AutophonePulseMonitor: connection.Consumer()') consumer = connection.Consumer(self.queues, callbacks=[self.handle_message], accept=['json'], auto_declare=False) logger.debug('AutophonePulseMonitor: bind queues') for queue in self.queues: queue(connection).queue_declare(passive=False) queue(connection).queue_bind() with consumer: while not self._stopping.is_set(): try: connection.drain_events(timeout=self.timeout) except socket.timeout: pass except socket.error, e: if "timed out" not in str(e): raise logger.debug('AutophonePulseMonitor.listen: stopping') except:
def listen(self): logger = utils.getLogger() connect_timeout = 5 wait = 30 connection = None restart = True while restart: restart = False try: # connection does not connect to the server until # either the connection.connect() method is called # explicitly or until kombu calls it implicitly as # needed. logger.debug('AutophonePulseMonitor: Connection()') connection = Connection(hostname=self.hostname, userid=self.userid, password=self.password, virtual_host=self.virtual_host, port=DEFAULT_SSL_PORT, ssl=True, connect_timeout=connect_timeout) logger.debug('AutophonePulseMonitor: connection.Consumer()') consumer = connection.Consumer(self.queues, callbacks=[self.handle_message], accept=['json'], auto_declare=False) logger.debug('AutophonePulseMonitor: bind queues') for queue in self.queues: queue(connection).queue_declare(passive=False) queue(connection).queue_bind() with consumer: while not self._stopping.is_set(): try: connection.drain_events(timeout=self.timeout) except socket.timeout: pass except socket.error, e: if "timed out" not in str(e): raise logger.debug('AutophonePulseMonitor.listen: stopping') except:
def run(rabbit_url): print rabbit_url conn = Connection(rabbit_url) conn.ensure_connection() conn.connect() exchange = Exchange(config.EXCHANGE_NAME, type='direct') queue = Queue(name=config.QUEUE_NAME, exchange=exchange, routing_key=config.ROUTING_KEY) consumer = Consumer(conn, queues=queue, callbacks=[process_message], accept=['text/plain']) consumer.consume() while True: try: print 'drain_events' conn.drain_events(timeout=2) # timeout except socket.timeout: pass
class RpcProxy(object): def __init__(self, amqp_url): self.connection = Connection(amqp_url) self.callback_queue = Queue(uuid(), exclusive=True, auto_delete=True) def on_response(self, message): if message.properties['correlation_id'] == self.correlation_id: result = message.payload ret = pickle.loads(result) if isinstance(ret, Exception): raise ret else: self.response = ret def __getattr__(self, name): def do_rpc(*args, **kwargs): self.response = None self.correlation_id = uuid() with Producer(self.connection) as producer: producer.publish( pickle.dumps((name, args, kwargs)), exchange='', routing_key=settings.RPC_QUEUE, declare=[self.callback_queue], reply_to=self.callback_queue.name, correlation_id=self.correlation_id, ) with Consumer(self.connection, on_message=self.on_response, queues=[self.callback_queue], no_ack=True, accept=['json', 'pickle', 'msgpack']): while self.response is None: self.connection.drain_events() return self.response return do_rpc
class NlpPipe: def __init__(self, conn_str='amqp://*****:*****@localhost/'): self.conn_str = conn_str self.conn = Connection(conn_str) self.callback_queue = Queue(str(uuid()), exclusive=True, auto_delete=True) def on_response(self, message): if message.properties['correlation_id'] == self.correlation_id: res = message.payload if isinstance(res, str): res = json.loads(res) self.response = res def process(self, data, steps): self.response = None self.correlation_id = uuid() logger.info('publish queue={} next={} reply_to={}'.format(steps[0], steps[1:] or None, self.callback_queue.name)) with Producer(self.conn) as producer: producer.publish( data, exchange='', routing_key=steps[0], # headers={'next': json.loads(json.dumps(steps[1:]))}, # headers={'next': ('aa', 'bbb')}, # headers={'next': json.dumps(['aa', 'bbb'])}, headers={'next': '|'.join(steps[1:])}, declare=[self.callback_queue], reply_to=self.callback_queue.name, correlation_id=self.correlation_id, ) with Consumer(self.conn, on_message=self.on_response, queues=[self.callback_queue], no_ack=True): while self.response is None: self.conn.drain_events() logger.info("Response {}".format(self.response)) return self.response
class FibonacciRpcClient(object): def __init__(self): self.connection = Connection('amqp://localhost//') self.channel = self.connection.channel() self.response_queue = self.channel.queue_declare(exclusive=True) self.channel.basic_consume( queue=self.response_queue.queue, callback=self.on_response, no_ack=True) def on_response(self, response): '''consume callback''' correlation_id = response.properties['correlation_id'] if correlation_id == self.corr_id: self.response = response.body def call(self, n): #ipdb.set_trace() self.response = None self.corr_id = str(uuid.uuid4()) producer = Producer(self.connection) producer.publish( str(n), exchange='', routing_key='rpc_queue', reply_to=self.response_queue.queue, correlation_id=self.corr_id) # FIXME(Ray): if we use pika this should be # self.connection.process_data_events() # But in kombu I can not find one for this if self.response is None: self.connection.drain_events() return int(self.response)
def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, queues=[self.queue]) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) with self.assertRaises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close()
def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, self.queue) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) with self.assertRaises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close()
class RabbitmqDaemon(threading.Thread): def __init__(self, cmd_q=None, reply_q=None): threading.Thread.__init__(self) self.settings = ConfigParser.ConfigParser() self.settings.read('../config/site.ini') self.rabbitmqUsername = self.settings.get('rabbitmq', 'username') self.rabbitmqPassword = self.settings.get('rabbitmq', 'password') self.rabbitmqHost = self.settings.get('rabbitmq', 'host') self.conn = Connection('amqp://'+self.rabbitmqUsername+':'+self.rabbitmqPassword+'@'+self.rabbitmqHost+':5672//') self.producer = Producer(self.conn.channel(), exchange = Exchange('eyezon.status', type='fanout'), serializer="json") self.rpcProducer= Producer(self.conn.channel(), serializer="json") self.cmd_q = cmd_q or Queue.Queue() self.reply_q = reply_q or Queue.Queue() queue = kombu.Queue( name="eyezon.cmd", exchange=Exchange('eyezon.cmd'), channel=self.conn.channel(), durable=False, exclusive=False, auto_delete=True) self.consumer = Consumer(self.conn.channel(), queues = queue, auto_declare=True, callbacks=[self.send_cmd]) self.consumer.consume(no_ack=True) self.alarmCache = { "zoneTimerDump": None, "keypadUpdate": None, "zoneStateChange": None, "partitionStateChange": None, "realtimeCIDEvent": None, "zoneTimerDump": None } def send_cmd(self, message, req): msg = message.encode('utf-8') if msg == "^02,$" and self.alarmCache['zoneTimerDump'] != None: self.rpcReply(self.alarmCache['zoneTimerDump'], req) elif msg == "getKeypadStatus" and self.alarmCache['keypadUpdate'] != None: self.rpcReply(self.alarmCache['keypadUpdate'], req) if msg != "getKeypadStatus": self.cmd_q.put(msg) def rpcReply(self, message, req): self.rpcProducer.publish(body=message, **dict({'routing_key': req.properties['reply_to'], 'correlation_id': req.properties.get('correlation_id'), 'content_encoding': req.content_encoding})) def publishEvent(self, event): if event['name'] == "Zone Timer Dump": self.alarmCache["zoneTimerDump"] = event elif event['name'] == "Virtual Keypad Update": self.alarmCache["keypadUpdate"] = event elif event['name'] == "Zone State Change": self.alarmCache["zoneStateChange"] = event elif event['name'] == "Partition State Change": self.alarmCache["partitionStateChange"] = event elif event['name'] == "Realtime CID Event": self.alarmCache["realtimeCIDEvent"] = event elif event['name'] == "Zone Timer Dump": self.alarmCache["zoneTimerDump"] = event self.producer.publish(exchange = 'eyezon.status', routing_key = "", body = event) def run(self): while 1: try: cmd = self.reply_q.get(True, 0.1) self.publishEvent(cmd) except Queue.Empty as e: try: self.conn.drain_events(timeout=0.1) except socket.timeout: None
class test_FilesystemTransport: def setup(self): self.channels = set() try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.channels.add(self.c.default_channel) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.channels.add(self.p.default_channel) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def teardown(self): # make sure we don't attempt to restore messages at shutdown. for channel in self.channels: try: channel._qos._dirty.clear() except AttributeError: pass try: channel._qos._delivered.clear() except AttributeError: pass def _add_channel(self, channel): self.channels.add(channel) return channel def test_produce_consume_noack(self): producer = Producer(self._add_channel(self.p.channel()), self.e) consumer = Consumer(self._add_channel(self.c.channel()), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() assert len(_received) == 10 def test_produce_consume(self): producer_channel = self._add_channel(self.p.channel()) consumer_channel = self._add_channel(self.c.channel()) producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() assert len(_received1) + len(_received2) == 20 # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() assert m.payload == {'compressed': True} # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') assert self.q(consumer_channel).get() self.q(consumer_channel).delete() self.q(consumer_channel).declare() assert self.q(consumer_channel).get() is None # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') assert self.q2(consumer_channel).get() self.q2(consumer_channel).purge() assert self.q2(consumer_channel).get() is None
class test_FilesystemTransport(TestCase): def setUp(self): try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def test_produce_consume_noack(self): producer = Producer(self.p.channel(), self.e) consumer = Consumer(self.c.channel(), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): producer_channel = self.p.channel() consumer_channel = self.c.channel() producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') self.assertTrue(self.q(consumer_channel).get()) self.q(consumer_channel).delete() self.q(consumer_channel).declare() self.assertIsNone(self.q(consumer_channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') self.assertTrue(self.q2(consumer_channel).get()) self.q2(consumer_channel).purge() self.assertIsNone(self.q2(consumer_channel).get())
class AutophonePulseMonitor(object): """AutophonePulseMonitor provides the means to be notified when Android builds are available for testing and when users have initiated retriggers and cancels via the Treeherder UI. Builds can be selected using repository names, Android platform names or build types. AutophonePulseMonitor detects new builds by listening to un-normalized buildbot initiated pulse messages rather than the normalized messages in order to obtain the check-in comment for a build. The comment is used to determine if a try build has requested Autophone testing. :param hostname: Hostname of Pulse. Defaults to the production pulse server pulse.mozilla.org. :param userid: Pulse User id :param password: Pulse Password :param virtual_host: AMQP virtual host, defaults to '/'. :param durable_queues: If True, will create durable queues in Pulse for the build and job action messages. Defaults to False. In production, durable_queues should be set to True to avoid losing messages if the connection is broken or the application crashes. :param build_exchange_name: Name of build exchange. Defaults to 'exchange/build/'. :param build_queue_name: Build queue name suffix. Defaults to 'builds'. The pulse build queue will be created with a name of the form 'queue/<userid>/<build_queue_name>'. :param jobaction_exchange_name: Name of job action exchange. Defaults to 'exchange/treeherder/v1/job-actions'. Use 'exchange/treeherder-stage/v1/job-actions' to listen to job action messages for Treeherder staging. :param jobaction_queue_name: Job action queue name suffix. Defaults to 'jobactions'. The pulse jobaction queue will be created with a name of the form 'queue/<userid>/<jobaction_queue_name>'. :param build_callback: Required callback function which takes a single `build_data` object as argument containing information on matched builds. `build_callback` is always called on a new thread. `build_data` is an object which is guaranteed to contain the following keys: 'appName': Will always be 'Fennec' 'branch': The repository name of the build, e.g. 'mozilla-central'. 'comments': Check-in comment. 'packageUrl': The url to the apk package for the build. 'platform': The platform name of the build, e.g. 'android-api-11' `build_data` may also contain the following keys: 'buildid': Build id in CCYYMMDDHHMMSS format. 'robocopApkUrl': Url to robocop apk for the build. 'symbolsUrl': Url to the symbols zip file for the build. 'testsUrl': Url to the tests zip file for the build. 'who': Check-in Commiter. :param jobaction_callback: Required callback function which takes a single `jobaction_data` object as argument containing information on matched actions. `jobaction_callback` is always called on a new thread. `jobaction_data` is an object which is contains the following keys: 'action': 'cancel' or 'retrigger', 'project': repository name, 'job_id': treeherder job_id, 'job_guid': treeherder job_guid, 'build_type': 'opt' or 'debug', 'platform': the detected platform, 'build_url': build url, 'machine_name': name of machine , 'job_group_name': treeherder job group name, 'job_group_symbol': treeherder job group symbol, 'job_type_name': treeherder job type name, 'job_type_symbol': treeherder job type symbol, 'result': test result result', :param treeherder_url: Optional Treeherder server url if Treeherder job action pulse messages are to be processed. Defaults to None. :param trees: Required list of repository names to be matched. :param platforms: Required list of platforms to be matched. Currently, the possible values are 'android', 'android-api-9', 'android-api-10', 'android-api-11', and 'android-x86'. :param buildtypes: Required list of build types to process. Possible values are 'opt', 'debug' :param timeout: Timeout in seconds for the kombu connection drain_events. Defaults to 5 seconds. :param shared_lock: Required lock used to control concurrent access. Used to prevent socket based deadlocks. :param verbose: If True, will log build and job action messages. Defaults to False. Usage: :: import threading import time from optparse import OptionParser parser = OptionParser() def build_callback(build_data): logger = logging.getLogger() logger.debug('PULSE BUILD FOUND %s' % build_data) def jobaction_callback(job_action): logger = logging.getLogger() if job_action['job_group_name'] != 'Autophone': return logger.debug('JOB ACTION FOUND %s' % json.dumps( job_action, sort_keys=True, indent=4)) logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.DEBUG) parser.add_option('--pulse-user', action='store', type='string', dest='pulse_user', default='', help='user id for connecting to PulseGuardian') parser.add_option('--pulse-password', action='store', type='string', dest='pulse_password', default='', help='password for connecting to PulseGuardian') (options, args) = parser.parse_args() shared_lock = threading.Lock() monitor = AutophonePulseMonitor( userid=options.pulse_user, password=options.pulse_password, jobaction_exchange_name='exchange/treeherder-stage/v1/job-actions', build_callback=build_callback, jobaction_callback=jobaction_callback, trees=['try', 'mozilla-inbound'], platforms=['android-api-9', 'android-api-11'], buildtypes=['opt'], shared_lock=shared_lock) monitor.start() time.sleep(3600) """ def __init__(self, hostname='pulse.mozilla.org', userid=None, password=None, virtual_host='/', durable_queues=False, build_exchange_name='exchange/build/', build_queue_name='builds', jobaction_exchange_name='exchange/treeherder/v1/job-actions', jobaction_queue_name='jobactions', build_callback=None, jobaction_callback=None, treeherder_url=None, trees=[], platforms=[], buildtypes=[], timeout=5, shared_lock=None, verbose=False): assert userid, "userid is required." assert password, "password is required." assert build_callback, "build_callback is required." assert trees, "trees is required." assert platforms, "platforms is required." assert buildtypes, "buildtypes is required." assert shared_lock, "shared_lock is required." self.treeherder_url = treeherder_url self.build_callback = build_callback self.jobaction_callback = jobaction_callback self.trees = list(trees) self.platforms = list(platforms) # Sort the platforms in descending order of length, so we do # not make a match on a substring of the platform prematurely. self.platforms.sort(cmp=lambda x,y: (len(y) - len(x))) self.buildtypes = list(buildtypes) self.timeout = timeout self.shared_lock = shared_lock self.verbose = verbose self._stopping = threading.Event() self.listen_thread = None # connection does not connect to the server until either the # connection.connect() method is called explicitly or until # kombu calls it implicitly as needed. self.connection = Connection(hostname=hostname, userid=userid, password=password, virtual_host=virtual_host, port=DEFAULT_SSL_PORT, ssl=True) build_exchange = Exchange(name=build_exchange_name, type='topic') self.queues = [Queue(name='queue/%s/build' % userid, exchange=build_exchange, routing_key='build.#.finished', durable=durable_queues, auto_delete=not durable_queues)] if treeherder_url: jobaction_exchange = Exchange(name=jobaction_exchange_name, type='topic') self.queues.append(Queue(name='queue/%s/jobactions' % userid, exchange=jobaction_exchange, routing_key='#', durable=durable_queues, auto_delete=not durable_queues)) def start(self): """Runs the `listen` method on a new thread.""" if self.listen_thread and self.listen_thread.is_alive(): logger.warning('AutophonePulseMonitor.start: listen thread already started') return logger.debug('AutophonePulseMonitor.start: listen thread starting') self.listen_thread = threading.Thread(target=self.listen, name='PulseMonitorThread') self.listen_thread.daemon = True self.listen_thread.start() def stop(self): """Stops the pulse monitor listen thread.""" logger.debug('AutophonePulseMonitor stopping') self._stopping.set() self.listen_thread.join() logger.debug('AutophonePulseMonitor stopped') def is_alive(self): return self.listen_thread.is_alive() def listen(self): logger.debug('AutophonePulseMonitor: start shared_lock.acquire') self.shared_lock.acquire() try: consumer = self.connection.Consumer(self.queues, callbacks=[self.handle_message], accept=['json'], auto_declare=False) for queue in self.queues: queue(self.connection).queue_declare(passive=False) queue(self.connection).queue_bind() with consumer: while not self._stopping.is_set(): try: logger.debug('AutophonePulseMonitor shared_lock.release') self.shared_lock.release() self.connection.drain_events(timeout=self.timeout) except socket.timeout: pass except KeyboardInterrupt: raise finally: logger.debug('AutophonePulseMonitor shared_lock.acquire') self.shared_lock.acquire() logger.debug('AutophonePulseMonitor.listen: stopping') except: logger.exception('AutophonePulseMonitor Exception') finally: logger.debug('AutophonePulseMonitor exit shared_lock.release') self.shared_lock.release() self.connection.release() def handle_message(self, data, message): if self._stopping.is_set(): return message.ack() if '_meta' in data and 'payload' in data: self.handle_build(data, message) if (self.treeherder_url and 'action' in data and 'project' in data and 'job_id' in data): self.handle_jobaction(data, message) def handle_build(self, data, message): if self.verbose: logger.debug( 'handle_build:\n' '\tdata : %s\n' '\tmessage: %s' % ( json.dumps(data, sort_keys=True, indent=4), json.dumps(message.__dict__, sort_keys=True, indent=4))) try: build = data['payload']['build'] except (KeyError, TypeError), e: logger.debug('AutophonePulseMonitor.handle_build_event: %s pulse build data' % e) return fields = ( 'appName', # Fennec 'branch', 'buildid', 'comments', 'packageUrl', 'platform', 'robocopApkUrl', 'symbolsUrl', 'testsUrl', 'who' ) required_fields = ( 'appName', # Fennec 'branch', # mozilla-central, ... 'comments', 'packageUrl', 'platform', # android... ) build_data = {} builder_name = build['builderName'] build_data['builder_name'] = builder_name build_data['build_type'] = 'debug' if 'debug' in builder_name else 'opt' for property in build['properties']: property_name = property[0] if property_name in fields and len(property) > 1 and property[1]: build_data[property_name] = type(property[1])(property[1]) for required_field in required_fields: if required_field not in build_data or not build_data[required_field]: return if build_data['appName'] != 'Fennec': return if not build_data['platform'].startswith('android'): return if build_data['branch'] not in self.trees: return if build_data['platform'] not in self.platforms: return if build_data['build_type'] not in self.buildtypes: return if build_data['branch'] == 'try' and 'autophone' not in build_data['comments']: return self.build_callback(build_data)
class test_MemoryTransport(Case): def setup(self): self.c = Connection(transport='memory') self.e = Exchange('test_transport_memory') self.q = Queue('test_transport_memory', exchange=self.e, routing_key='test_transport_memory') self.q2 = Queue('test_transport_memory2', exchange=self.e, routing_key='test_transport_memory2') self.fanout = Exchange('test_transport_memory_fanout', type='fanout') self.q3 = Queue('test_transport_memory_fanout1', exchange=self.fanout) self.q4 = Queue('test_transport_memory_fanout2', exchange=self.fanout) def test_driver_version(self): self.assertTrue(self.c.transport.driver_version()) def test_produce_consume_noack(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer = Consumer(channel, self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume_fanout(self): producer = self.c.Producer() consumer = self.c.Consumer([self.q3, self.q4]) producer.publish( {'hello': 'world'}, declare=consumer.queues, exchange=self.fanout, ) self.assertEqual(self.q3(self.c).get().payload, {'hello': 'world'}) self.assertEqual(self.q4(self.c).get().payload, {'hello': 'world'}) self.assertIsNone(self.q3(self.c).get()) self.assertIsNone(self.q4(self.c).get()) def test_produce_consume(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_memory', compression='zlib') m = self.q(channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get()) def test_drain_events(self): with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) c1 = self.c.channel() c2 = self.c.channel() with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) del(c1) # so pyflakes doesn't complain. del(c2) def test_drain_events_unregistered_queue(self): c1 = self.c.channel() producer = self.c.Producer() consumer = self.c.Consumer([self.q2]) producer.publish( {'hello': 'world'}, declare=consumer.queues, routing_key=self.q2.routing_key, exchange=self.q2.exchange, ) message = consumer.queues[0].get()._raw class Cycle(object): def get(self, timeout=None): return (message, 'foo'), c1 self.c.transport.cycle = Cycle() self.c.drain_events() def test_queue_for(self): chan = self.c.channel() chan.queues.clear() x = chan._queue_for('foo') self.assertTrue(x) self.assertIs(chan._queue_for('foo'), x)
class test_FilesystemTransport(Case): def setUp(self): if sys.platform == "win32": raise SkipTest("Needs win32con module") try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest("filesystem transport: cannot create tempfiles") self.c = Connection( transport="filesystem", transport_options={"data_folder_in": data_folder_in, "data_folder_out": data_folder_out}, ) self.p = Connection( transport="filesystem", transport_options={"data_folder_in": data_folder_out, "data_folder_out": data_folder_in}, ) self.e = Exchange("test_transport_filesystem") self.q = Queue("test_transport_filesystem", exchange=self.e, routing_key="test_transport_filesystem") self.q2 = Queue("test_transport_filesystem2", exchange=self.e, routing_key="test_transport_filesystem2") def test_produce_consume_noack(self): producer = Producer(self.p.channel(), self.e) consumer = Consumer(self.c.channel(), self.q, no_ack=True) for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_filesystem") _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): producer_channel = self.p.channel() consumer_channel = self.c.channel() producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_filesystem") for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_filesystem2") _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({"compressed": True}, routing_key="test_transport_filesystem", compression="zlib") m = self.q(consumer_channel).get() self.assertDictEqual(m.payload, {"compressed": True}) # queue.delete for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_filesystem") self.assertTrue(self.q(consumer_channel).get()) self.q(consumer_channel).delete() self.q(consumer_channel).declare() self.assertIsNone(self.q(consumer_channel).get()) # queue.purge for i in range(10): producer.publish({"foo": i}, routing_key="test_transport_filesystem2") self.assertTrue(self.q2(consumer_channel).get()) self.q2(consumer_channel).purge() self.assertIsNone(self.q2(consumer_channel).get())
class test_FilesystemTransport(Case): def setup(self): try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def test_produce_consume_noack(self): producer = Producer(self.p.channel(), self.e) consumer = Consumer(self.c.channel(), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): producer_channel = self.p.channel() consumer_channel = self.c.channel() producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') self.assertTrue(self.q(consumer_channel).get()) self.q(consumer_channel).delete() self.q(consumer_channel).declare() self.assertIsNone(self.q(consumer_channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') self.assertTrue(self.q2(consumer_channel).get()) self.q2(consumer_channel).purge() self.assertIsNone(self.q2(consumer_channel).get())
def do_consume(user_qs): print("about to listen no queues [%s]" % ", ".join(list(map(lambda x: x, user_qs)))) conn = Connection(amqp_hosts, failover_strategy='round-robin') # try to get a connection no matter what while True: try: conn.ensure_connection(errback=on_ens_conn_err_cb) conn.connect() except Exception as e: print("connection error failed on exception [%s]" % repr(e)) conn.release() continue if conn.connected: break else: print("connection failed in some way, retry") chan = conn.channel() global bound_cons_Q cons_Q = Queue(common.uuid(), queue_arguments=q_expires) bound_cons_Q = cons_Q(chan) bound_cons_Q.declare() # first bind to some control route bound_cons_Q.bind_to(priTopicXchg, routing_key='manage.#') for i in user_qs: if '*' in i or '#' in i: # create the wildcard route_key bind bound_cons_Q.bind_to(priTopicXchg, routing_key=i) else: for j in allQs: if i == j.as_dict()['name']: bound_cons_Q.bind_to(priTopicXchg, routing_key=j.as_dict()['routing_key']) cons = Consumer( chan, accept=['json'], queues=bound_cons_Q, callbacks=[on_msg_cb_1, on_msg_cb_2] ) print("queue set to [%s]" % bound_cons_Q.as_dict(recurse=True)) cons.consume() while True: try: conn.drain_events() except conn.connection_errors + conn.channel_errors as e: print("connection [%s] went down (error[%s]), trying to " "connect to the next one" % (conn.info(), repr(e))) conn.close() conn.release() conn.ensure_connection(errback=on_ens_conn_err_cb) conn.connect() chan = conn.channel() cons_Q.bind(chan) cons = Consumer( chan, accept=['json'], queues=bound_cons_Q, callbacks=[on_msg_cb_1, on_msg_cb_2] ) cons.consume()
result = supervisorConn.supervisor.stopAllProcesses() rpcReply(result, args) def task_restartall(message=None, args=None): result = supervisorConn.system.multicall([ {'methodName':'supervisor.stopAllProcesses'}, {'methodName':'supervisor.startAllProcesses'} ]) rpcReply(result, args) def on_request(body, req): message = json.loads(body) print "Received message ",message sys.stdout.flush() operations = { "list_processes" : list_processes, "task_start" : task_start, "task_stop" : task_stop, "task_restart" : task_restart, "read_log" : read_log } operations[message['operation']](message, req) #lets light this candle consumer = Consumer(rmqConn.channel(), queues = queue, auto_declare=True, callbacks=[on_request]) consumer.consume(no_ack=True) while True: rmqConn.drain_events()
#!/usr/bin/env python from kombu import Connection, Exchange, Queue def process_message(body, message): print body message.ack() nova_exchange = Exchange('nova', 'topic', durable=False) nova_queue = Queue('listener', exchange = nova_exchange, routing_key='#') conn = Connection('amqp://*****:*****@192.168.0.10//') consumer = conn.Consumer(nova_queue, callbacks=[process_message]) consumer.consume() while True: conn.drain_events()
class Dbreader: def __init__(self, broker_cloud, host_influxdb): self.clientDB = InfluxDBClient(host_influxdb, 8086, 'root', 'root', 'Collector_DB') self.clientDB.create_database('Collector_DB') self.producer_connection = Connection(broker_cloud) self.consumer_connection = Connection(broker_cloud) self.exchange = Exchange("IoT", type="direct") def get_item_state(sefl, list_item_global_id): items = [] for item_global_id in list_item_global_id: query_statement = 'SELECT * FROM \"' + item_global_id + '\" ORDER BY time DESC LIMIT 1 ' query_result = sefl.clientDB.query(query_statement) for item in query_result: item_state = { 'item_global_id': item[0]['item_global_id'], 'item_state': item[0]['item_state'], # 'last_changed': covert_time_to_correct_time_zone(item[0]['time']), 'last_changed': item[0]['time'], 'thing_global_id': item[0]['thing_global_id'] } items.append(item_state) return items def api_get_item_state(self, body, message): print("API get_item_state") # Message {'list_item_global_id': [], 'reply_to': " ", } list_item_global_id = json.loads(body)["list_item_global_id"] reply_to = json.loads(body)['reply_to'] items = self.get_item_state(list_item_global_id) message_response = { "items": items } self.producer_connection.ensure_connection() with Producer(self.producer_connection) as producer: producer.publish( json.dumps(message_response), exchange=self.exchange.name, routing_key=reply_to, retry=True ) # print("Done: {}".format(items)) def api_get_item_state_history(self, body, message): print("API get_item_state_history") # Message {'list_item_global_id': [], 'reply_to': " ", } list_global_id = json.loads(body)["list_global_id"] reply_to = json.loads(body)['reply_to'] start_time = json.loads(body)["start_time"] end_time = json.loads(body)["end_time"] scale = json.loads(body)["scale"] items = self.get_item_state_history(list_global_id, start_time, end_time, scale) message_response = { "items": items } self.producer_connection.ensure_connection() with Producer(self.producer_connection) as producer: producer.publish( json.dumps(message_response), exchange=self.exchange.name, routing_key=reply_to, retry=True ) def get_item_state_history(self, list_global_id, start_time, end_time, scale): items = [] print(scale) for global_id in list_global_id: item_global_id = global_id['item_global_id'] thing_global_id = global_id['thing_global_id'] query_statement_type_field = """SHOW FIELD KEYS ON \"Collector_DB\" FROM \"{}\"""".format(item_global_id) query_result_type_field = self.clientDB.query(query_statement_type_field) type_field = list(query_result_type_field.get_points())[0]['fieldType'] if scale == "0s": if type_field == 'integer' or type_field == 'float': query_statement_history = """SELECT * FROM \"{}\" where time >= \'{}\' AND time <= \'{}\'""".format(item_global_id, start_time, end_time) query_result_history = self.clientDB.query(query_statement_history) query_result_history = list(query_result_history.get_points()) query_statement_global = """SELECT MAX(\"item_state\") AS max_state, MIN(\"item_state\") AS min_state, MEAN(\"item_state\") AS average_state FROM \"{}\" where time >= \'{}\' AND time <= \'{}\'""".format(item_global_id, start_time, end_time) query_result_global = self.clientDB.query(query_statement_global) query_result_global = list(query_result_global.get_points()) print(query_result_global[0]['max_state'], query_result_global[0]['min_state'], query_result_global[0]['average_state']) if len(query_result_history) > 0 and len(query_result_global) > 0: item = { 'item_global_id': item_global_id, 'thing_global_id': thing_global_id, 'max_global': query_result_global[0]['max_state'], 'min_global': query_result_global[0]['min_state'], 'average_global': query_result_global[0]['average_state'], 'history': [] } for item_history in query_result_history: item_state = { 'last_changed': item_history['time'], 'item_state': item_history['item_state'], } item['history'].append(item_state) items.append(item) else: query_statement_history = """SELECT * FROM \"{}\" where time >= \'{}\' AND time <= \'{}\'""".format(item_global_id, start_time, end_time) query_result_history = self.clientDB.query(query_statement_history) query_result_history = list(query_result_history.get_points()) if len(query_result_history) > 0: item = { 'item_global_id': item_global_id, 'thing_global_id': thing_global_id, 'history': [] } for item_history in query_result_history: item_state = { 'last_changed': item_history['time'], 'item_state': item_history['item_state'], } item['history'].append(item_state) items.append(item) else: if type_field == 'integer' or type_field == 'float': query_statement_history = """SELECT MODE(\"item_state\") AS item_state, MAX(\"item_state\") AS max_state, MIN(\"item_state\") AS min_state, MEAN(\"item_state\") AS average_state FROM \"{}\" WHERE time >= \'{}\' AND time <= \'{}\' GROUP BY time({}), thing_global_id""".format(item_global_id, start_time, end_time, scale) query_result_history = self.clientDB.query(query_statement_history) query_result_history = list(query_result_history.get_points()) query_statement_global = """SELECT MAX(\"item_state\") AS max_state, MIN(\"item_state\") AS min_state, MEAN(\"item_state\") AS average_state FROM \"{}\" where time >= \'{}\' AND time <= \'{}\'""".format(item_global_id, start_time, end_time) query_result_global = self.clientDB.query(query_statement_global) query_result_global = list(query_result_global.get_points()) if len(query_result_history) > 0 and len(query_result_global) > 0: item = { 'item_global_id': item_global_id, 'thing_global_id': thing_global_id, 'max_global': query_result_global[0]['max_state'], 'min_global': query_result_global[0]['min_state'], 'average_global': query_result_global[0]['average_state'], 'history': [] } for item_history in query_result_history: item_state = { 'last_changed': item_history['time'], 'item_state': item_history['item_state'], 'min_state': item_history['min_state'], 'max_state': item_history['max_state'], 'average_state': item_history['average_state'] } item['history'].append(item_state) items.append(item) else: query_statement_history = """SELECT MODE(\"item_state\") AS item_state FROM \"{}\" WHERE time >= \'{}\' AND time <= \'{}\' GROUP BY time({}), thing_global_id""".format(item_global_id, start_time, end_time, scale) query_result_history = self.clientDB.query(query_statement_history) query_result_history = list(query_result_history.get_points()) if len(query_result_history) > 0: item = { 'item_global_id': item_global_id, 'thing_global_id': thing_global_id, 'history': [] } for item_history in query_result_history: item_state = { 'last_changed': item_history['time'], 'item_state': item_history['item_state'] } item['history'].append(item_state) items.append(item) return items def run(self): queue_get_item_state = Queue(name='dbreader.request.api_get_item_state', exchange=self.exchange, routing_key='dbreader.request.api_get_item_state')#, message_ttl=20) queue_get_item_state_history = Queue(name='dbreader.request.api_get_item_state_history', exchange=self.exchange, routing_key='dbreader.request.api_get_item_state_history')#, message_ttl=20) while 1: try: self.consumer_connection.ensure_connection(max_retries=1) with nested(Consumer(self.consumer_connection, queues=queue_get_item_state, callbacks=[self.api_get_item_state], no_ack=True), Consumer(self.consumer_connection, queues=queue_get_item_state_history, callbacks=[self.api_get_item_state_history], no_ack=True)): while True: self.consumer_connection.drain_events() except (ConnectionRefusedError, exceptions.OperationalError): print('Connection lost') except self.consumer_connection.connection_errors: print('Connection error')
class RabbitmqDaemon(threading.Thread): def __init__(self, actiontecQueue, timecapsuleQueue): threading.Thread.__init__(self) self.settings = ConfigParser.ConfigParser() self.settings.read('../config/site.ini') self.rabbitmqUsername = self.settings.get('rabbitmq', 'username') self.rabbitmqPassword = self.settings.get('rabbitmq', 'password') self.rabbitmqHost = self.settings.get('rabbitmq', 'host') self.rmqConn = Connection('amqp://'+self.rabbitmqUsername+':'+self.rabbitmqPassword+'@'+self.rabbitmqHost+':5672//') self.statusProducer = Producer(self.rmqConn.channel(), exchange = Exchange('actiontec.status', type='fanout'), serializer="json") self.timecapsuleEntries = {} self.discoveredHosts = {} self.timecapsuleQueue = timecapsuleQueue self.actiontecQueue = actiontecQueue def run(self): def list_mac_addresses(message=None, args=None): print "RPC reply with",len(self.discoveredHosts)," cached addresses" sys.stdout.flush() rpcReply(self.discoveredHosts, args) def rpcReply(message, req): rpcProducer.publish(body=message, **dict({'routing_key': req.properties['reply_to'], 'correlation_id': req.properties.get('correlation_id'), 'content_encoding': req.content_encoding})) operations = { "list_mac_addresses" : list_mac_addresses } def on_request(body, req): message = json.loads(body) print "Received message ",message sys.stdout.flush() operations[message['operation']](message, req) rpcProducer= Producer(self.rmqConn.channel(), serializer="json") queue = kombu.Queue( name="actiontec.cmd", exchange=Exchange('actiontec.cmd'), channel=self.rmqConn.channel(), durable=False, exclusive=False, auto_delete=True) consumer = Consumer(self.rmqConn.channel(), queues = queue, auto_declare=True, callbacks=[on_request]) consumer.consume(no_ack=True) while 1: try: self.actiontecEntries = self.actiontecQueue.get(True, 0.1) print "Actiontec cache now contains",len(self.actiontecEntries),"entries" except Queue.Empty as e: self.timecapsuleEntries = self.timecapsuleQueue.get(True, 0.1) # newEntries = self.arpQueue.get(True, 0.1) # for entry in newEntries: # self.entries[entry["mac"]] = entry # print "received",len(self.entries),"hosts from watcher" oldDiscoveredHosts = self.discoveredHosts self.discoveredHosts = {} for mac,details in self.timecapsuleEntries.iteritems(): hostname = "" ip = "" tx = 0 rx = 0 txerr = 0 rxerr = 0 if mac in self.actiontecEntries: hostname = self.actiontecEntries[mac]["hostname"] ip = self.actiontecEntries[mac]["ip"] if mac in oldDiscoveredHosts: now = time.mktime((datetime.datetime.now()).timetuple()) timeDelta = now - oldDiscoveredHosts[mac]["stats"]["timestamp"] rxDelta = int(details["rx"]) - oldDiscoveredHosts[mac]["rawStats"]["rx"] txDelta = int(details["tx"]) - oldDiscoveredHosts[mac]["rawStats"]["tx"] rxerrDelta = int(details["rxerr"]) - oldDiscoveredHosts[mac]["rawStats"]["rxerr"] txerrDelta = int(details["txerr"]) - oldDiscoveredHosts[mac]["rawStats"]["txerr"] tx = int(txDelta // timeDelta) rx = int(rxDelta // timeDelta) rxerr = int(rxerrDelta // timeDelta) txerr = int(txerrDelta // timeDelta) self.discoveredHosts[mac] = { "hostname": hostname, "ip": ip, "mac": mac, "rate": details['rate'], "stats":{ "tx":tx, "rx":rx, "txerr":txerr, "rxerr":rxerr, "signal":int(details['signal']), "noise": int(details['noise']), "timestamp": time.mktime((datetime.datetime.now()).timetuple()), }, "rawStats":{ "tx": details['tx'], "rx": details['rx'], "rates": details['rates'], "time": int(details['time']), "noise": int(details['noise']), "rate": int(details['rate']), "rx": int(details['rx']), "tx": int(details['tx']), "rxerr": int(details['rxerr']), "txerr": int(details['txerr']) } } reply = [] for mac,details in self.discoveredHosts.iteritems(): reply.append({ "hostname":details["hostname"], "ip":details["ip"], "mac":details["mac"], "rate":details["rate"], "stats":details["stats"] }) self.statusProducer.publish(body = reply) except Queue.Empty as e: try: self.rmqConn.drain_events(timeout=0.1) except: None
class test_FilesystemTransport: def setup(self): self.channels = set() try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.channels.add(self.c.default_channel) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.channels.add(self.p.default_channel) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def teardown(self): # make sure we don't attempt to restore messages at shutdown. for channel in self.channels: try: channel._qos._dirty.clear() except AttributeError: pass try: channel._qos._delivered.clear() except AttributeError: pass def _add_channel(self, channel): self.channels.add(channel) return channel def test_produce_consume_noack(self): producer = Producer(self._add_channel(self.p.channel()), self.e) consumer = Consumer(self._add_channel(self.c.channel()), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() assert len(_received) == 10 def test_produce_consume(self): producer_channel = self._add_channel(self.p.channel()) consumer_channel = self._add_channel(self.c.channel()) producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() assert len(_received1) + len(_received2) == 20 # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() assert m.payload == {'compressed': True} # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') assert self.q(consumer_channel).get() self.q(consumer_channel).delete() self.q(consumer_channel).declare() assert self.q(consumer_channel).get() is None # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') assert self.q2(consumer_channel).get() self.q2(consumer_channel).purge() assert self.q2(consumer_channel).get() is None
''' Created on May 19, 2014 @author: xiaoxubeii ''' from kombu import Connection from kombu.messaging import Consumer from entity1 import queue connection = Connection("amqp://*****:*****@xiaoxubeii:5672//") channel = connection.channel() def test(body, message): print body message.ack() consumer = Consumer(channel, queue) consumer.register_callback(test) consumer.consume() while True: connection.drain_events()
class test_PyroTransport: def setup(self): self.c = Connection(transport='pyro', virtual_host="kombu.broker") self.e = Exchange('test_transport_pyro') self.q = Queue('test_transport_pyro', exchange=self.e, routing_key='test_transport_pyro') self.q2 = Queue('test_transport_pyro2', exchange=self.e, routing_key='test_transport_pyro2') self.fanout = Exchange('test_transport_pyro_fanout', type='fanout') self.q3 = Queue('test_transport_pyro_fanout1', exchange=self.fanout) self.q4 = Queue('test_transport_pyro_fanout2', exchange=self.fanout) def test_driver_version(self): assert self.c.transport.driver_version() @pytest.mark.skip("requires running Pyro nameserver and Kombu Broker") def test_produce_consume_noack(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer = Consumer(channel, self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_pyro') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() assert len(_received) == 10 def test_drain_events(self): with pytest.raises(socket.timeout): self.c.drain_events(timeout=0.1) c1 = self.c.channel() c2 = self.c.channel() with pytest.raises(socket.timeout): self.c.drain_events(timeout=0.1) del(c1) # so pyflakes doesn't complain. del(c2) @pytest.mark.skip("requires running Pyro nameserver and Kombu Broker") def test_drain_events_unregistered_queue(self): c1 = self.c.channel() producer = self.c.Producer() consumer = self.c.Consumer([self.q2]) producer.publish( {'hello': 'world'}, declare=consumer.queues, routing_key=self.q2.routing_key, exchange=self.q2.exchange, ) message = consumer.queues[0].get()._raw class Cycle(object): def get(self, callback, timeout=None): return (message, 'foo'), c1 self.c.transport.cycle = Cycle() self.c.drain_events() @pytest.mark.skip("requires running Pyro nameserver and Kombu Broker") def test_queue_for(self): chan = self.c.channel() x = chan._queue_for('foo') assert x assert chan._queue_for('foo') is x
class test_basic(TestCase): def setUp(self): self.c = Connection(transport='ironmq') self.p = Connection(transport='ironmq') self.e = Exchange('test_transport_iron') self.q = Queue('test_transport_iron', exchange=self.e, routing_key='test_transport_iron') self.q2 = Queue('test_transport_iron2', exchange=self.e, routing_key='test_transport_iron2') self.q(self.c.channel()).delete() self.q2(self.c.channel()).delete() def test_produce_consume_noack(self): producer = Producer(self.p.channel(), self.e) consumer = Consumer(self.c.channel(), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_iron') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): producer_channel = self.p.channel() consumer_channel = self.c.channel() producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_iron') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_iron2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression #producer.publish({'compressed': True}, # routing_key='test_transport_iron', # compression='zlib') #m = self.q(consumer_channel).get() #self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_iron') self.assertTrue(self.q(consumer_channel).get()) self.q(consumer_channel).delete() self.q(consumer_channel).declare() self.assertIsNone(self.q(consumer_channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_iron2') self.assertTrue(self.q2(consumer_channel).get()) self.q2(consumer_channel).purge() self.assertIsNone(self.q2(consumer_channel).get())
class _AMQPServerWrapper(object): def __init__(self, amqp_url): self.__connection = Connection(amqp_url) self.__connection.connect() self.__monitors = {} self.__running = True self.__consumer = Consumer(self.__connection, on_message=self.__on_message) self.__consumer_gl = gevent.spawn(self.__consumer_greenlet_main) def __consumer_greenlet_main(self): gevent.sleep(0) while self.__running: self.__consumer.consume() try: self.__connection.drain_events(timeout=0.1) except Exception as ex: # NOQA: assigned but not used (left in for super-duper-low-level-debug) # print("was woken because {}".format(ex)) pass gevent.sleep(0) # print("---loop") def __on_message(self, msg): ct = msg.delivery_info['consumer_tag'] assert ct in self.__monitors, \ "Message from consumer '{}', but we are not monitoring that (list={})".format( msg.delivery_info['consumer_tag'], self.__monitors.keys()) mon = self.__monitors[ct] mon['event_cb'](msg, msg.body) @property def connected(self): return self.__connection.connected def create_add_monitor(self, exchange, routing_key, event_cb, queue_name=None): mname = "ex={} rk={} qn={}".format(exchange, routing_key, queue_name) if mname in self.__monitors: mon = self.__monitors[mname] mon["event_cb"] = event_cb else: if queue_name is None: queue_name = '' exclusive = True else: exclusive = False ex = Exchange(exchange, 'topic') queue = Queue(exchange=ex, routing_key=routing_key, exclusive=exclusive) bound_queue = queue.bind(self.__connection) self.__consumer.add_queue(bound_queue) bound_queue.consume(mname, self.__on_message) mon = { "event_cb": event_cb, "exchange": ex } self.__monitors[mname] = mon return mon['exchange'] def inject(self, exchange, routing_key, payload): prod = Producer(self.__connection, exchange=exchange, routing_key=routing_key) prod.publish(payload) def test_helper_sync_send_msg(self, exchange, ex_rk, send_rk, payload): ex = Exchange(exchange, 'topic') queue = Queue(exchange=ex, routing_key=ex_rk + '.*', exclusive=True, channel=self.__connection) queue.declare() prod = Producer(self.__connection, exchange=ex, routing_key=send_rk) prod.publish(payload) return queue def test_helper_sync_recv_msg(self, queue): for tick in range(10): msg = queue.get() if msg is not None: break time.sleep(1) return msg
Queue_Port = 5672 Queue_Path = '/websocketserver' def callback(body, message): print body connection = Connection(hostname=Queue_Server, port=Queue_Port, userid=Queue_User, password=Queue_PassWord, virtual_host=Queue_Path) channel = connection.channel() smsExchange = Exchange("sys.sms", type='topic', channel=channel, durable=True, delivery_mode=2) task_queue = Queue('test_recv', exchange=smsExchange, routing_key='sms.code', durable=False, channel=channel) consumer = Consumer(channel, task_queue, no_ack=True, callbacks=[callback]) consumer.qos(prefetch_count=1) consumer.consume() while True: connection.drain_events() connection.close()
class Event_Generator_1(Event_Generator_Base): list_event_condition = [ ] # list_event_condition saved as a mysql table: |condition_id | condition_name | condition | def __init__(self, event_generator_name, event_generator_id, description, event_dest_topic): Event_Generator_Base.__init__(self, event_generator_name, event_generator_id, description, event_dest_topic) BROKER_CLOUD = "192.168.43.30" self.producer_connection = Connection("localhost") self.consumer_connection = Connection("192.168.43.30") self.exchange = Exchange("IoT", type="direct") # self.queue_get_states = Queue(name='data_source.to.' + str(self.event_generator_name), exchange=self.exchange, # routing_key='data_source.to.' + str(self.event_generator_name))#, message_ttl=20) self.queue_get_states = Queue(name='rule.request.states', exchange=self.exchange, routing_key='rule.request.states', message_ttl=20) self.db = MySQLdb.connect(host="0.0.0.0", user="******", passwd="root", db="rule") self.cursor = self.db.cursor() def read_event_condition(self): print("reading event condition ...") request = "Select rule_id, rule_content from Rule" try: # Execute the SQL command self.cursor.execute(request) results = self.cursor.fetchall() # self.list_event_condition = result # print ("results: ", results) for result in results: print("result: ", result) # (rule_id, rule_content) = result rule_id = result[0] print("rule_id: ", rule_id) # rule_content = json.dumps(rule_content) rule_content = json.loads(rule_content) print("\nrule content: ", rule_content) trigger_type = rule_content['trigger_type'] trigger_content = rule_content['trigger_content'] print("\n\ntrigger_type: ", trigger_type) print("\n\ntrigger_content: ", trigger_content) # Commit your changes in the database self.db.commit() return rule_id, trigger_type, trigger_content except: # Rollback in case there is any error print("error read_event_condition") self.db.rollback() return None def check_trigger_item_has_given_state(self, trigger_content): print("checking trigger item_has_given_state ...") print(trigger_content) trigger_content_list = json.loads(trigger_content) pre_result = False total_result = False bitwise_operator = trigger_content['trigger_bitwise'] print("bitwise_operator: ", bitwise_operator) for trigger_content in trigger_content_list: result = False timer = 0 trigger_item_id = trigger_content['id'] print("trigger_item_id: ", trigger_item_id) operator = trigger_content['operator'] print("operator: ", operator) value = trigger_content['value'] print("value: ", value) if (value.isdigit() == True): value = float(value) request = "select time from ItemTable where item_id = '%s' order by time asc limit 1" % ( trigger_item_id) print("time request: ", request) last_insert_time = str(datetime.max) try: # Execute the SQL command self.cursor.execute(request) request_result = self.cursor.fetchall() last_insert_time = request_result[0][0] # Commit your changes in the database self.db.commit() except: print("error read time") result = False pre_result = result self.db.rollback() last_insert_time = datetime.strptime(last_insert_time, '%Y-%m-%d %H:%M:%S.%f') check_time = last_insert_time - timedelta( seconds=float(timer.split('s')[0])) request = "select item_state from ItemTable where item_id = '%s' and time >= '%s' order by time desc" % ( trigger_item_id, check_time) try: # Execute the SQL command self.cursor.execute(request) request_result = self.cursor.fetchall() item_state_list = request_result[0] print("item_state_list: ", item_state_list) # Commit your changes in the database self.db.commit() for item_state in item_state_list: print("item_state: ", item_state) print("value: ", value) result = True if (item_state.isdigit() == True): item_state = float(item_state) # elif (item_state == "on"): # item_state = 1 # elif (item_state == "off"): # item_state = 0 if (operator == "GT"): if (item_state <= value): result = False break elif (operator == "GE"): if (item_state < value): result = False break elif (operator == "LT"): if (item_state >= value): result = False break elif (operator == "LE"): if (item_state > value): result = False break elif (operator == "EQ"): if (item_state != value): result = False break elif (operator == "NE"): if (item_state == value): result = False break else: print("operator is not valid") result = False break print("result: ", result) if (bitwise_operator.upper() == "NONE"): total_result = result pre_result = result elif (bitwise_operator.upper() == "AND"): total_result = pre_result and result pre_result = total_result elif (bitwise_operator.upper() == "OR"): total_result = pre_result or result pre_result = total_result else: print("bitwise operator is not pre-defined") except: # Rollback in case there is any error self.db.rollback() # print ("error check_trigger_item_has_given_state") # return None result = False pre_result = result print("total result: ", total_result) return total_result def check_trigger_condition(self, trigger_type, trigger_content, item_id): print("checking trigger condition ...") result = False if (trigger_type == "item_has_given_state"): result = self.check_trigger_item_has_given_state(trigger_content) else: print("trigger_type is not pre-defined") return result def create_event(self, rule_id): print("creating Event ...") request = "select trigger_content from RuleTable where trigger_id = '%s'" % ( rule_id) # print (request) try: # Execute the SQL command self.cursor.execute(request) request_result = self.cursor.fetchall() print(request_result[0][0]) trigger_content = request_result[0][0] trigger_content = json.loads(trigger_content) message = { 'event_id': event_id, 'event_source': event_source, 'trigger_id': trigger_id, 'time': str(datetime.now()) } # print (trigger_content) output_field = trigger_content['outputs'] print(output_field) # Commit your changes in the database self.db.commit() for output in output_field: event_id = output['event_id'] # event_name = output['event_name'] event_source = output['event_source'] description = output['description'] message = { 'event_generator_id': self.event_generator_id, 'event_id': event_id, # 'event_name' : event_name, 'event_source': event_source, 'trigger_id': trigger_id, 'description': description, 'time': str(datetime.now()) } self.producer_connection.ensure_connection() with Producer(self.producer_connection) as producer: producer.publish(json.dumps(message), exchange=self.exchange.name, routing_key='event_generator.to.' + str(self.event_dest_topic), retry=True) print("Send event to Rule Engine: " + 'event_generator.to.' + str(self.event_dest_topic)) except: print("error read trigger_content") def write_item_to_database(self, item_id, item_name, item_type, item_state, time): print("writting item to database ...") request = """INSERT INTO ItemTable(item_id, item_name, item_type, item_state, time) VALUES ("%s", "%s", "%s", "%s", "%s")""" \ % (item_id, item_name, item_type, item_state, time) print(request) try: # Execute the SQL command print(self.cursor.execute(request)) result = self.cursor.fetchall() print("write item %s to database: %s" % (item_id, result)) self.db.commit() return True except: # Rollback in case there is any error self.db.rollback() print("error write item to database") return False def receive_states(self): def handle_notification(body, message): print("\n\n\n") print("Receive state!") message = json.loads(body)["body"]["states"] for i in range(len(message)): item_id = json.loads(body)["body"]["states"][i]["MetricId"] item_name = json.loads( body)["body"]["states"][i]["MetricLocalId"] item_type = json.loads( body)["body"]["states"][i]["DataPoint"]["DataType"] item_state = json.loads( body)["body"]["states"][i]["DataPoint"]["Value"] # time = json.loads(body)[""] time = datetime.now() time = str(time) # Write new item to the database write_result = self.write_item_to_database( item_id, item_name, item_type, item_state, time) #if write new item success to the database, consider the trigger with that item if (write_result == 1): list_event_condition = self.read_event_condition() print("list event condition: ", list_event_condition) for event_condition in list_event_condition: rule_id = event_condition[0] trigger_type = event_condition[1] trigger_content = event_condition[2] # Check the item with each trigger condition result = self.check_trigger_condition( trigger_type, trigger_content, item_id) print("check trigger : ", result) if (result == None): return None # If checkTriggerCondition success, create an event if (result == True): # event_source = item_id # event_name = str(item_name) + "_" + str(datetime.now()) # event_id = randint(1, 1000000) # my_event = Event(event_name, event_id, event_source) self.create_event(rule_id) # End handle_notification try: self.consumer_connection.ensure_connection(max_retries=1) with nested( Consumer(self.consumer_connection, queues=self.queue_get_states, callbacks=[handle_notification], no_ack=True)): while True: self.consumer_connection.drain_events() except (ConnectionRefusedError, exceptions.OperationalError): print('Connection lost') except self.consumer_connection.connection_errors: print('Connection error') def run(self): self.cursor.execute("""CREATE TABLE IF NOT EXISTS ItemTable( item_id VARCHAR(50), item_name VARCHAR(50), item_type VARCHAR(50), item_state VARCHAR(50), time VARCHAR(50), PRIMARY KEY (item_id, time))""") while 1: try: self.receive_states() except (ConnectionRefusedError, exceptions.OperationalError): print('Connection lost') except self.consumer_connection.connection_errors: print('Connection error')
from kombu import Connection, Exchange, Queue, Consumer ''' Sample Kombu consumer to drain messages from AMQP message bus ''' rabbit_url = "amqp://{rabbit_server_ip}:5672/" conn = Connection(rabbit_url) channel = conn.channel() exchange = Exchange('data_exchange', type="direct") queue = Queue(name="data_q", exchange=exchange, routing_key="data_info") def process_message(body, message): print type(body), body print type(message), message.body print("The body is {}".format(body)) message.ack() with Consumer(conn, queues=queue, callbacks=[process_message], accept=["text/plain", "json"]): conn.drain_events(timeout=6000)
#coding:utf-8 from kombu import Connection from kombu.messaging import Consumer,Producer from kombu import Exchange, Queue Queue_User='******' Queue_PassWord='******' Queue_Server='124.207.209.57' Queue_Port=None Queue_Path='/spider' conn=Connection(hostname=Queue_Server,port=Queue_Port,userid=Queue_User,password=Queue_PassWord,virtual_host=Queue_Path) channel=conn.channel() exch=Exchange('weibodownload',type='topic',durable=True,delivery_mode=2,passive=True) #最新地理位置微薄的routing_key='weibo.geo' 每个人历史记录微薄的routing_key='weibo.user_geo' queue=Queue(exchange=exch,routing_key='weibo.user_geo',auto_delete=True) consumer=Consumer(channel=channel,queues=queue,no_ack=True) consumer.qos(prefetch_count=1) def on_response(body, message): print body consumer.register_callback(on_response) consumer.consume() while True: conn.drain_events()
class test_MemoryTransport(TestCase): def setUp(self): self.c = Connection(transport='memory') self.e = Exchange('test_transport_memory') self.q = Queue('test_transport_memory', exchange=self.e, routing_key='test_transport_memory') self.q2 = Queue('test_transport_memory2', exchange=self.e, routing_key='test_transport_memory2') def test_produce_consume_noack(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer = Consumer(channel, self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_memory', compression='zlib') m = self.q(channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get()) def test_drain_events(self): with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) c1 = self.c.channel() c2 = self.c.channel() with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) del (c1) # so pyflakes doesn't complain. del (c2) def test_drain_events_unregistered_queue(self): c1 = self.c.channel() class Cycle(object): def get(self, timeout=None): return ('foo', 'foo'), c1 self.c.transport.cycle = Cycle() with self.assertRaises(KeyError): self.c.drain_events() def test_queue_for(self): chan = self.c.channel() chan.queues.clear() x = chan._queue_for('foo') self.assertTrue(x) self.assertIs(chan._queue_for('foo'), x)
class MozReviewBot(object): def __init__(self, config_path=None, reviewboard_url=None, reviewboard_user=None, reviewboard_password=None, pulse_host=None, pulse_port=None, pulse_userid=None, pulse_password=None, exchange=None, queue=None, routing_key=None, pulse_timeout=None, pulse_ssl=False, repo_root=None, logger=None): if logger is None: self.logger = logging.getLogger('mozreviewbot') else: self.logger = logger # We use options passed into __init__ preferentially. If any of these # are not specified, we next check the configuration file, if any. # Finally, we use environment variables. if config_path and not os.path.isfile(config_path): # ConfigParser doesn't seem to throw if it is unable to find the # config file so we'll explicitly check that it exists. self.logger.error('could not locate config file: %s' % ( config_path)) config_path = None if config_path: try: config = ConfigParser() config.read(config_path) reviewboard_url = (reviewboard_url or config.get('reviewboard', 'url')) reviewboard_user = (reviewboard_user or config.get('reviewboard', 'user')) reviewboard_password = (reviewboard_password or config.get('reviewboard', 'password')) pulse_host = pulse_host or config.get('pulse', 'host') pulse_port = pulse_port or config.get('pulse', 'port') pulse_userid = pulse_userid or config.get('pulse', 'userid') pulse_password = pulse_password or config.get('pulse', 'password') exchange = exchange or config.get('pulse', 'exchange') queue = queue or config.get('pulse', 'queue') routing_key = routing_key or config.get('pulse', 'routing_key') pulse_timeout = pulse_timeout or config.get('pulse', 'timeout') if pulse_ssl is None: pulse_ssl = config.get('pulse', 'ssl') except NoSectionError as e: self.logger.error('configuration file missing section: %s' % e.section) try: repo_root = repo_root or config.get('hg', 'repo_root') except (NoOptionError, NoSectionError): # Subclasses do not need to define repo root if they do not # plan on using the hg functionality. pass # keep config around in case any subclasses would like to extract # options from it. self.config = config else: self.config = None reviewboard_url = reviewboard_url or os.environ.get('REVIEWBOARD_URL') pulse_host = pulse_host or os.environ.get('PULSE_HOST') pulse_port = pulse_port or os.environ.get('PULSE_PORT') self.rbclient = RBClient(reviewboard_url, username=reviewboard_user, password=reviewboard_password) self.api_root = self.rbclient.get_root() self.conn = Connection(hostname=pulse_host, port=pulse_port, userid=pulse_userid, password=pulse_password, ssl=pulse_ssl) self.exchange = Exchange(exchange, type='topic', durable=True) self.queue = Queue(name=queue, exchange=self.exchange, durable=True, routing_key=routing_key, exclusive=False, auto_delete=False) self.pulse_timeout = float(pulse_timeout) self.repo_root = repo_root self.hg = None for DIR in os.environ['PATH'].split(os.pathsep): p = os.path.join(DIR, 'hg') if os.path.exists(p): self.hg = p def _get_available_messages(self): messages = [] def onmessage(body, message): messages.append((body, message)) consumer = self.conn.Consumer([self.queue], callbacks=[onmessage], auto_declare=True) with consumer: try: self.conn.drain_events(timeout=self.pulse_timeout) except socket.timeout: pass return messages def _run_hg(self, hg_args): # TODO: Use hgtool. args = [self.hg] + hg_args env = dict(os.environ) env['HGENCODING'] = 'utf-8' null = open(os.devnull, 'w') # Execute at / to prevent Mercurial's path traversal logic from # kicking in and picking up unwanted config files. return subprocess.check_output(args, stdin=null, stderr=null, env=env, cwd='/') def ensure_hg_repo_exists(self, landing_repo_url, repo_url, pull_rev=None): # TODO: Use the root changeset in each repository as an identifier. # This will enable "forks" to share the same local clone. # The "share" extension now has support for this. # Read hg help -e share for details about "pooled storage." # We should probably deploy that. url = landing_repo_url or repo_url sha1 = hashlib.sha1(url).hexdigest() repo_path = os.path.join(self.repo_root, sha1) if not os.path.exists(repo_path): args = ['clone', url, repo_path] self.logger.debug('cloning %s' % url) self._run_hg(args) self.logger.debug('finished cloning %s' % url) args = ['-R', repo_path, 'pull', repo_url] if pull_rev: args.extend(['-r', pull_rev]) self.logger.debug('pulling %s' % repo_url) self._run_hg(args) self.logger.debug('finished pulling %s' % repo_url) return repo_path def hg_commit_changes(self, repo_path, node, diff_context=None): """Obtain information about what changed in a Mercurial commit. The return value is a tuple of: (set(adds), set(dels), set(mods), None, diff) The first 4 items list what files changed in the changeset. The last item is a unified diff of the changeset. File copies are currently not returned. ``None`` is being used as a placeholder until support is needed. """ part_delim = str(uuid.uuid4()) item_delim = str(uuid.uuid4()) parts = [ '{join(file_adds, "%s")}' % item_delim, '{join(file_dels, "%s")}' % item_delim, '{join(file_mods, "%s")}' % item_delim, '{join(file_copies, "%s")}' % item_delim, ] template = part_delim.join(parts) self._run_hg(['-R', repo_path, 'up', '-C', node]) res = self._run_hg(['-R', repo_path, 'log', '-r', node, '-T', template]) diff_args = ['-R', repo_path, 'diff', '-c', node] if diff_context is not None: diff_args.extend(['-U', str(diff_context)]) diff = self._run_hg(diff_args) adds, dels, mods, copies = res.split(part_delim) adds = set(f for f in adds.split(item_delim) if f) dels = set(f for f in dels.split(item_delim) if f) mods = set(f for f in mods.split(item_delim) if f) # TODO parse the copies. return adds, dels, mods, None, diff def strip_nonpublic_changesets(self, repo_path): """Strip non-public changesets from a repository. Pulling changesets over and over results in many heads in a repository. This makes Mercurial slow. So, we prune non-public changesets/heads to keep repositories fast. """ self._run_hg(['-R', repo_path, '--config', 'extensions.strip=', 'strip', '--no-backup', '-r', 'not public()']) def get_commit_files(self, commit): """Fetches a list of files that were changed by this commit.""" rrid = commit['review_request_id'] diff_revision = commit['diffset_revision'] start = 0 files = [] while True: result = self.api_root.get_files(review_request_id=rrid, diff_revision=diff_revision, start=start) files.extend(result) start += result.num_items if result.num_items == 0 or start >= result.total_results: break return files def handle_available_messages(self): for body, message in self._get_available_messages(): payload = body['payload'] repo_url = payload['repository_url'] landing_repo_url = payload['landing_repository_url'] commits = payload['commits'] # TODO: should we allow process commits to signal that we should # skip acknowledging the message? try: for commit in commits: rrid = commit['review_request_id'] diff_revision = commit['diffset_revision'] review = BatchReview(self.api_root, rrid, diff_revision) self.process_commit(review, landing_repo_url, repo_url, commit) finally: # This prevents the queue from growing indefinitely but # prevents us from fixing whatever caused the exception # and restarting the bot to handle the message. message.ack() def listen_forever(self): while True: self.handle_available_messages() def process_commit(self, review, repo_url, commits): pass