def setUp(self): self.connection = Connection(host="localhost:5672", userid="guest", password="******", virtual_host="/") self.channel = self.connection.channel() self._queue_declare()
def setUp(self): self.connection = Connection(host="localhost:5672", userid="guest", password="******", virtual_host="/") self.channel = self.connection.channel() self.TEST_QUEUE = "pyrabbitmq.testq2"
class test_Delete(unittest.TestCase): def setUp(self): self.connection = Connection(host="localhost:5672", userid="guest", password="******", virtual_host="/") self.channel = self.connection.channel() self.TEST_QUEUE = "pyrabbitmq.testq2" def test_delete(self): """Test that we can declare a channel delete it, and then declare with different properties""" res = self.channel.exchange_declare(self.TEST_QUEUE, "direct") res =self.channel.queue_declare(self.TEST_QUEUE) res = self.channel.queue_bind(self.TEST_QUEUE, self.TEST_QUEUE, self.TEST_QUEUE) # Delete the queue self.channel.queue_delete(self.TEST_QUEUE) # Declare it again x = self.channel.queue_declare(self.TEST_QUEUE, durable=True) self.assertIn("message_count", x) self.assertIn("consumer_count", x) self.assertEqual(x["queue"], self.TEST_QUEUE) self.channel.queue_delete(self.TEST_QUEUE) def test_delete_empty(self): """Test that the queue doesn't get deleted if it is not empty""" self.channel.exchange_declare(self.TEST_QUEUE, "direct") self.channel.queue_declare(self.TEST_QUEUE) self.channel.queue_bind(self.TEST_QUEUE, self.TEST_QUEUE, self.TEST_QUEUE) message = Message("the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8")) self.channel.basic_publish(message, self.TEST_QUEUE, self.TEST_QUEUE) self.assertRaises(ChannelError, self.channel.queue_delete, self.TEST_QUEUE, if_empty=True) #We need to make a new channel after a ChannelError self.channel = self.connection.channel() x = self.channel.basic_get(self.TEST_QUEUE) self.assertTrue(x.body) self.channel.queue_delete(self.TEST_QUEUE, if_empty=True) def tearDown(self): if self.channel: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
def __init__(self, mq_server = None, virtual_host = cfg.CB_CLUSTER_TAG): if mq_server == None: mq_server = cfg.RABBITMQ_IP self.connection = Connection(host= mq_server, userid="guest", password="******", virtual_host = virtual_host) self.manager = Client(mq_server+":55672", "guest", "guest")
def rawTaskPublisher(task, args, queue, broker = cfg.RABBITMQ_IP, vhost = cfg.CB_CLUSTER_TAG, exchange = "", userid="guest", password="******", routing_key = None): # setup broker connection connection = Connection(host = broker, userid=userid, password=password, virtual_host = vhost) channel = connection.channel() # construct task body body={'retries': 0, 'task': task, 'errbacks': None, 'callbacks': None, 'kwargs': {}, 'eta': None, 'args': args, 'id': 'e7cb7ff5-acd3-4060-8f7e-2ef85f810fe5', 'expires': None, 'utc': True} header = {'exclusive': False, 'name': queue, 'headers': {}, 'durable': True, 'delivery_mode': 2, 'no_ack': False, 'priority': None, 'alias': None, 'queue_arguments': None, 'content_encoding': 'binary', 'content_type': 'application/x-python-serialize', 'binding_arguments': None, 'auto_delete': True} # prepare message body = pickle.dumps(body) message = (body, header) if routing_key is None: routing_key = queue # publish! rc = channel.basic_publish(message, exchange = exchange, routing_key = routing_key)
def setUp(self): self.connection = Connection(host='localhost:5672', userid='guest', password='******', virtual_host='/') self.channel = self.connection.channel() self._queue_declare()
def setUp(self): self.connection = Connection(host='localhost:5672', userid='guest', password='******', virtual_host='/') self.channel = self.connection.channel() self.TEST_QUEUE = 'pyrabbitmq.testq2'
def Connect(): conn = Connection(host=hostname, port=port, userid=user, password=password, virtual_host=vhost) channel = conn.channel() channel.exchange_declare(exchange_name, 'topic') while True: s = r.grp['state'] for x in s: arg_body = """Routing Key[%s] ==================================> XKRX-CS-KR-000252,13:30:48.023942,7,290.9,123.19,90.82,79.62,937.15 XKRX-CS-KR-000253,13:30:48.024171,7,28.84,93.29,67.13,234.64,149.7 """ % x arg_rky = s[x] print arg_rky, arg_body channel.basic_publish(arg_body.replace("KR", x), exchange_name, arg_rky) time.sleep(5) channel.close() conn.close()
def __init__(self, mq_server = None): if mq_server == None: mq_server = cfg.RABBITMQ_IP self.connection = Connection(host= mq_server, userid="guest", password="******", virtual_host="/") self.declare("workload") self.declare("workload_template")
def _connect(self): """Creates our AMQP connection""" try: self.connection = Connection(host=self.host) self.channel = self.connection.channel() logging.info('Connected to server: ' + self.host) self.last_connect = int(time.time()) return True except ConnectionError: logging.error('Unable to connect to server') return None
def connect(self): connections['default'].close() rabbitmq = ALCO_SETTINGS['RABBITMQ'] self.amqp = Connection(**rabbitmq) self.redis = redis.Redis(**ALCO_SETTINGS['REDIS']) self.insert_thread = Thread(target=self.inserter_loop) self.insert_thread.start() hostname = '%s:%s' % (rabbitmq['host'], ALCO_SETTINGS['RABBITMQ_API_PORT']) self.rabbit = Client(hostname, rabbitmq['userid'], rabbitmq['password']) self.vhost = rabbitmq['virtual_host']
def Connect(): conn = Connection(host=hostname, port=port, userid=user, password=password, virtual_host=vhost) channel = conn.channel() channel.exchange_declare(exchange_name, 'topic') while True: s = r.grp['state'] for x in s: arg_body = """Routing Key[%s] ==================================> XKRX-CS-KR-000252,13:30:48.023942,7,290.9,123.19,90.82,79.62,937.15 XKRX-CS-KR-000253,13:30:48.024171,7,28.84,93.29,67.13,234.64,149.7 XKRX-CS-KR-000254,13:30:48.024337,7,248.17,118.49,1489.54,118.45,117.42 XKRX-CS-KR-000255,13:30:48.024497,7,70.67,170.82,65.45,152.11,420.7 XKRX-CS-KR-000256,13:30:48.034801,7,160.74,82.36,260.87,104.42,384.35 XKRX-CS-KR-000257,13:30:48.034973,7,123.39,150.31,60.78,201.21,181.55 XKRX-CS-KR-000100,13:30:48.035137,8,166.66,87.45,252.83,82.03,44.02 XKRX-CS-KR-000101,13:30:48.045434,8,114.86,1023.0,37.92,65.76,61.82 XKRX-CS-KR-000102,13:30:48.045586,8,159.16,97.96,60.07,75.29,690.15 XKRX-CS-KR-000103,13:30:48.045730,8,23.52,133.91,44.0,107.83,533.96 XKRX-CS-KR-000104,13:30:48.045901,8,76.62,274.25,166.57,116.48,149.1 XKRX-CS-KR-000250,13:30:48.056203,8,105.32,254.87,158.97,21.0,59.72 XKRX-CS-KR-000251,13:30:48.056364,8,192.7,226.26,76.02,72.7,40.53 XKRX-CS-KR-000252,13:30:48.056520,8,138.58,138.76,89.68,41.78,175.83 XKRX-CS-KR-000253,13:30:48.066883,8,88.67,41.84,126.81,222.26,8.98 XKRX-CS-KR-000254,13:30:48.067103,8,156.14,126.11,46.24,24.03,57.94 XKRX-CS-KR-000255,13:30:48.067259,8,136.01,35.25,25.29,275.88,50.33 XKRX-CS-KR-000256,13:30:48.067416,8,136.89,10.51,197.03,200.62,238.65 XKRX-CS-KR-000257,13:30:48.077776,8,47.36,41.77,101.75,105.99,64.56 XKRX-CS-KR-000100,13:30:48.078006,9,26.76,231.9,104.19,117.87,24.69 XKRX-CS-KR-000101,13:30:48.078187,9,57.14,84.92,73.62,33.72,47.86 XKRX-CS-KR-000102,13:30:48.088561,9,21.85,120.6,538.69,58.24,1685.93 XKRX-CS-KR-000103,13:30:48.088819,9,450.32,417.01,210.68,121.41,27.18 XKRX-CS-KR-000104,13:30:48.088998,9,80.61,69.15,132.51,98.67,226.2 XKRX-CS-KR-000250,13:30:48.089161,9,107.44,11.22,80.1,85.93,125.1 XKRX-CS-KR-000251,13:30:48.099518,9,43.86,51.79,282.43,101.35,946.29 XKRX-CS-KR-000252,13:30:48.099705,9,170.75,242.6,74.15,323.43,28.48 XKRX-CS-KR-000253,13:30:48.099871,9,53.27,36.47,81.75,50.96,46.73 XKRX-CS-KR-000254,13:30:48.110195,9,136.93,17.66,77.64,253.57,66.8 XKRX-CS-KR-000255,13:30:48.110408,9,65.49,72.59,39.59,63.07,74.31 XKRX-CS-KR-000256,13:30:48.110575,9,63.16,44.29,36.04,119.36,21.78 XKRX-CS-KR-000257,13:30:48.110733,9,125.17,54.65,374.91,219.27,136.63 """ % x arg_rky = s[x] print arg_rky, arg_body channel.basic_publish(arg_body.replace("KR", x), exchange_name, arg_rky) #ime.sleep(0.1) channel.close() conn.close()
def run(self): self.conn = Connection(host=hostname, port=port, userid=user, password=password, virtual_host=vhost) self.channel = self.conn.channel() self.channel.exchange_declare(exchange_name, 'topic') queueId = self.channel.queue_declare( exclusive = True ).queue self.channel.queue_bind(queueId, exchange_name, self.routingKey) try: self.channel.basic_consume(queueId, callback=self.callback_rdm) except KeyboardInterrupt: self.channel.close() self.conn.close() while True: self.conn.drain_events()
def _check_connection(cls): if not getattr(thread_data, 'rabbitmq_connected', False) \ and 'oioioi.notifications' in settings.INSTALLED_APPS \ and NotificationHandler.last_connection_check < \ time.time() - NotificationHandler.conn_try_interval: try: o = urlparse(settings.NOTIFICATIONS_RABBITMQ_URL) kwargs = {} if o.hostname: kwargs['host'] = o.hostname if o.port: kwargs['port'] = o.port if o.username: kwargs['userid'] = o.username if o.password: kwargs['password'] = o.password if o.path: kwargs['virtual_host'] = o.path thread_data.conn = Connection(**kwargs) thread_data.rabbitmq_connected = True # pylint: disable=broad-except except Exception: NotificationHandler.last_connection_check = time.time() logger.info("Notifications: Can't connect to RabbitMQ", exc_info=True)
class StateService(object): def __init__(self, routingKey): self.routingKey= routingKey self.conn = None self.channel = None def run(self): self.conn = Connection(host=hostname, port=port, userid=user, password=password, virtual_host=vhost) self.channel = self.conn.channel() self.channel.exchange_declare(exchange_name, 'topic') queueId = self.channel.queue_declare( exclusive = True ).queue self.channel.queue_bind(queueId, exchange_name, self.routingKey) try: self.channel.basic_consume(queueId, callback=self.callback_rdm) except KeyboardInterrupt: self.channel.close() self.conn.close() while True: self.conn.drain_events() def callback_rdm(self, message): #print("Body:'%s', Proeprties:'%s', DeliveryInfo:'%s'" % ( message.body, message.properties, message.delivery_info)) print message.body #message.ack() #channel.basic_publish(message.body, exchange_name, arg_rky) def close(self): self.conn.close()
def _connect(self): """Creates our AMQP connection""" try: self.connection = Connection(host=self.host) self.channel = self.connection.channel() logging.info("Connected to server: " + self.host) self.last_connect = int(time.time()) return True except ConnectionError: logging.error("Unable to connect to server") return None
class StateService(object): def __init__(self, csGroup, routingKey): self.csGroup = csGroup self.routingKey= routingKey self.conn = None self.channel = None def run(self): #arg_nam = sys.argv[1] #arg_rky = sys.argv[2] #arg_body = sys.argv[2] self.conn = Connection(host=hostname, port=port, userid=user, password=password, virtual_host=vhost) self.channel = self.conn.channel() self.channel.exchange_declare(exchange_name, 'topic') queueId = self.channel.queue_declare( exclusive = True ).queue self.channel.queue_bind(queueId, exchange_name, self.routingKey) self.channel.basic_consume(queueId, callback=self.callback) """ try: self.channel.basic_consume(queueId, callback=self.callback) except KeyboardInterrupt: self.channel.close() self.conn.close() """ while True: self.conn.drain_events() def callback(self, message): #print("Body:'%s', Proeprties:'%s', DeliveryInfo:'%s'" % ( message.body, message.properties, message.delivery_info)) print message.body self.channel.basic_publish(message.body, exchange_name, self.csGroup) #message.ack() #message.reject() def close(self): self.conn.close()
def run(self): #arg_nam = sys.argv[1] #arg_rky = sys.argv[2] #arg_body = sys.argv[2] self.conn = Connection(host=hostname, port=port, userid=user, password=password, virtual_host=vhost) self.channel = self.conn.channel() self.channel.exchange_declare(exchange_name, 'topic') queueId = self.channel.queue_declare( exclusive = True ).queue self.channel.queue_bind(queueId, exchange_name, self.routingKey) self.channel.basic_consume(queueId, callback=self.callback) """ try: self.channel.basic_consume(queueId, callback=self.callback) except KeyboardInterrupt: self.channel.close() self.conn.close() """ while True: self.conn.drain_events()
class RabbitOperator: queue_name = None connection = None channel = None def __init__(self, qn): self.queue_name = qn self.connection = Connection() self.channel = self.connection.channel() self.start_matchmaking_queue() def start_matchmaking_queue(self): self.channel.queue_declare(queue=self.queue_name) def check_insert_queue(self): message = self.channel.basic_get(self.queue_name) if message: message.ack() return message.body else: return None def publish(self, body): try: self.channel.basic_publish(exchange='', routing_key=self.queue_name, body=json.dumps(body)) return True except Exception as e: return False def insert_player_into_matchmaking_queue(self, username, rating): return self.publish({ "command": "insert", "username": username, "rating": rating }) def remove_player_from_matchmaking_queue(self, username): return self.publish({"command": "remove", "username": username})
class test_Delete(unittest.TestCase): def setUp(self): self.connection = Connection(host='localhost:5672', userid='guest', password='******', virtual_host='/') self.channel = self.connection.channel() self.TEST_QUEUE = 'pyrabbitmq.testq2' self.channel.queue_delete(self.TEST_QUEUE) def test_delete(self): """Test that we can declare a channel delete it, and then declare with different properties""" self.channel.exchange_declare(self.TEST_QUEUE, 'direct') self.channel.queue_declare(self.TEST_QUEUE) self.channel.queue_bind( self.TEST_QUEUE, self.TEST_QUEUE, self.TEST_QUEUE, ) # Delete the queue self.channel.queue_delete(self.TEST_QUEUE) # Declare it again x = self.channel.queue_declare(self.TEST_QUEUE, durable=True) self.assertEqual(x.queue, self.TEST_QUEUE) self.channel.queue_delete(self.TEST_QUEUE) def test_delete_empty(self): """Test that the queue doesn't get deleted if it is not empty""" self.channel.exchange_declare(self.TEST_QUEUE, 'direct') self.channel.queue_declare(self.TEST_QUEUE) self.channel.queue_bind(self.TEST_QUEUE, self.TEST_QUEUE, self.TEST_QUEUE) message = Message('the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) self.channel.basic_publish(message, self.TEST_QUEUE, self.TEST_QUEUE) with self.assertRaises(ChannelError): self.channel.queue_delete(self.TEST_QUEUE, if_empty=True) # We need to make a new channel after a ChannelError self.channel = self.connection.channel() x = self.channel.basic_get(self.TEST_QUEUE) self.assertTrue(x.body) self.channel.queue_delete(self.TEST_QUEUE, if_empty=True) def tearDown(self): if self.channel and self.connection.connected: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
class test_Channel(unittest.TestCase): def setUp(self): self.connection = Connection(host='localhost:5672', userid='guest', password='******', virtual_host='/') self.channel = self.connection.channel() self.channel.queue_delete(TEST_QUEUE) self._queue_declare() def test_send_message(self): message = Message( 'the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.assertGreater(self.channel.queue_purge(TEST_QUEUE), 2) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) def _queue_declare(self): self.channel.exchange_declare(TEST_QUEUE, 'direct') x = self.channel.queue_declare(TEST_QUEUE) self.assertEqual(x.message_count, x[1]) self.assertEqual(x.consumer_count, x[2]) self.assertEqual(x.queue, TEST_QUEUE) self.channel.queue_bind(TEST_QUEUE, TEST_QUEUE, TEST_QUEUE) def test_basic_get_ack(self): message = Message( 'the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) while True: x = self.channel.basic_get(TEST_QUEUE) if x: break self.assertIs(self.channel, x.channel) self.assertIn('message_count', x.delivery_info) self.assertIn('redelivered', x.delivery_info) self.assertEqual(x.delivery_info['routing_key'], TEST_QUEUE) self.assertEqual(x.delivery_info['exchange'], TEST_QUEUE) self.assertTrue(x.delivery_info['delivery_tag']) self.assertTrue(x.properties['content_type']) self.assertTrue(x.body) x.ack() def test_timeout_burst(self): """Check that if we have a large burst of messages in our queue that we can fetch them with a timeout without needing to receive any more messages.""" message = Message('the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) for i in xrange(100): self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) for i in xrange(100): self.connection.drain_events(timeout=0.2) self.assertEquals(len(messages), 100) def test_timeout(self): """Check that our ``drain_events`` call actually times out if there are no messages.""" message = Message('the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) self.connection.drain_events(timeout=0.1) with self.assertRaises(socket.timeout): self.connection.drain_events(timeout=0.1) self.assertEquals(len(messages), 1) def tearDown(self): if self.channel and self.connection.connected: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
class RabbitHelper(object): def __init__(self, mq_server = None): if mq_server == None: mq_server = cfg.RABBITMQ_IP self.connection = Connection(host= mq_server, userid="guest", password="******", virtual_host=cfg.CB_CLUSTER_TAG) def declare(self, queue, durable = True): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) res = channel.queue_declare(queue = queue, durable = durable, auto_delete = True) channel.close() return res def purge(self, queue): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) channel.queue_delete(queue=queue) channel.close() def consume(self, callback, queue, no_ack = True): channel = self.connection.channel() channel.basic_consume(callback, queue = queue, no_ack = no_ack) channel.start_consuming() channel.close() def qsize(self, queue): size = 0 if queue != None: if not isinstance(queue,str): queue = str(queue) response = self.declare(queue = queue) size = response[1] return size def putMsg(self, queue, body): channel = self.connection.channel() if not isinstance(queue, str): queue = str(queue) rc = channel.basic_publish(exchange = '', routing_key=queue, body = body) channel.close() def getMsg(self, queue, no_ack = False, requeue = False): channel = self.connection.channel() message = channel.basic_get(queue = queue) body = None if message is not None: body = message.body # Handle data receipt acknowldegement if no_ack == False: message.ack() if requeue: self.putMsg(queue, body) channel.close() return body def getJsonMsg(self, queue, no_ack = False, requeue = False): msg = self.getMsg(queue, no_ack, requeue) body = {} if msg is not None: try: body = json.loads(msg) except ValueError: pass return body def close(self): self.connection.close() def __del__(self): self.close()
class Collector(object): def __init__(self, index): self.index = index self.cancelled = False self.transport = self.protocol = None self.messages = [] self.block_size = 1000 self.exchange = "logstash" self.current_date = None self.logger = getLogger('alco.collector.%s' % self.index.name) self.amqp = self.redis = self.conn = self.vhost = self.rabbit = None self.insert_thread = None self.query_queue = queue.Queue() self.result_queue = queue.Queue() self.result_queue.put(None) self.query = self.values_stub = None self.existing = self.included = self.indexed = self.filtered = None def cancel(self): self.cancelled = True def inserter_loop(self): self.conn = connections[ALCO_SETTINGS['SPHINX_DATABASE_NAME']] while not self.cancelled: try: query, args = self.query_queue.get(block=True, timeout=1) except queue.Empty: continue result = self.insert_data(query, args) self.result_queue.put(result) def connect(self): connections['default'].close() rabbitmq = ALCO_SETTINGS['RABBITMQ'] self.amqp = Connection(**rabbitmq) self.redis = redis.Redis(**ALCO_SETTINGS['REDIS']) self.insert_thread = Thread(target=self.inserter_loop) self.insert_thread.start() hostname = '%s:%s' % (rabbitmq['host'], ALCO_SETTINGS['RABBITMQ_API_PORT']) self.rabbit = Client(hostname, rabbitmq['userid'], rabbitmq['password']) self.vhost = rabbitmq['virtual_host'] # noinspection PyUnusedLocal def process_sigint(self, signum, frame): self.logger.info("Got signal %s" % signum) self.cancel() self.logger.info("Futures cancelled, wait for thread") self.insert_thread.join() self.logger.info("Thread done") def __call__(self): signal.signal(signal.SIGINT, self.process_sigint) signal.signal(signal.SIGTERM, self.process_sigint) try: self.logger.debug("Connecting to RabbitMQ") self.connect() self.declare_queue() self.cleanup_bindings() channel = self.amqp.channel() channel.basic_qos(0, 1000, False) channel.basic_consume(self.index.queue_name, callback=self.process_message, no_ack=True) start = time.time() self.logger.debug("Start processing messages") while not self.cancelled: try: self.amqp.drain_events(timeout=1) except (socket.timeout, OSError): pass if time.time() - start > 1: self.push_messages() start = time.time() except KeyboardInterrupt: self.logger.warning("Got SIGINT, exit(0)") finally: self.amqp.close() sys.exit(0) def process_message(self, msg): data = json.loads(six.binary_type(msg.body)) ts = data.pop('@timestamp') data.pop("@version") msg = data.pop('message') seq = data.pop('seq', 0) dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") result = { 'ts': time.mktime(dt.timetuple()), 'ms': dt.microsecond, 'seq': seq, 'message': msg, 'data': data } self.messages.append(result) d = dt.date() if not self.current_date: self.current_date = d if d != self.current_date: self.current_date = d self.push_messages() if len(self.messages) >= self.block_size: self.push_messages() def declare_queue(self): channel = self.amqp.channel() """:type channel: amqp.channel.Channel""" durable = self.index.durable channel.exchange_declare(exchange=self.exchange, type='topic', durable=durable, auto_delete=False) channel.queue_declare(self.index.queue_name, durable=durable, auto_delete=False) for rk in self.get_routing_keys(): channel.queue_bind(self.index.queue_name, exchange=self.exchange, routing_key=rk) def get_routing_keys(self): return map(lambda x: x.strip(), self.index.routing_key.split(',')) def cleanup_bindings(self): self.logger.debug("Checking bindings") queue = self.index.queue_name exchange = self.exchange bindings = self.rabbit.get_queue_bindings(self.vhost, queue) bindings = [b for b in bindings if b['source'] == exchange] allowed = self.get_routing_keys() q = six.moves.urllib.parse.quote for b in bindings: rk = b['routing_key'] if rk in allowed: continue self.logger.debug("Unbind %s with RK=%s" % (queue, rk)) self.rabbit.delete_binding(self.vhost, exchange, q(queue), q(rk)) def push_messages(self): try: request_started.send(None, environ=None) self._push_messages() except Exception as e: self.logger.exception(e) raise finally: request_finished.send(None) def _push_messages(self): messages, self.messages = self.messages, [] if not messages: return message_count = len(messages) self.logger.info("Saving %s events" % message_count) columns = defaultdict(set) suffix = self.current_date.strftime("%Y%m%d") name = "%s_%s" % (self.index.name, suffix) args = [] self.load_index_columns() self.prepare_query(name) pkeys = self.get_primary_keys(messages) seen = set() for pk, data in zip(pkeys, messages): # saving seen columns to LoggerColumn model, collecting unique # values for caching in redis js = data['data'] self.process_js_columns(js, columns, self.included, seen) js_str = json.dumps(js) values = tuple(js.get(c) or '' for c in self.indexed) args.extend((pk, js_str, data['message']) + values) query = self.query + ','.join([self.values_stub] * message_count) self.save_column_values(columns) self.save_new_columns(seen) if self.result_queue.empty(): self.logger.debug("Insert still running, waiting") while not self.cancelled: try: self.result_queue.get(block=True, timeout=1) except queue.Empty: continue self.query_queue.put((query, args)) def insert_data(self, query, args): self.logger.debug("Inserting logs to searchd") result = None for _ in 1, 2, 3: try: c = self.conn.cursor() result = c.execute(query, args) self.logger.debug("%s rows inserted" % c.rowcount) c.close() except ProgrammingError: self.logger.exception( "Can't insert values to index: %s" % query) except DatabaseError as e: self.logger.exception("Sphinx connection error: %s" % e) try: close_old_connections() except Exception as e: self.logger.exception("Can't reconnect: %s" % e) os.kill(os.getpid(), signal.SIGKILL) except Exception: self.logger.exception("Unhandled error in insert_data") else: return result self.logger.error("Can't insert data in 3 tries, exit process") os.kill(os.getpid(), signal.SIGKILL) def save_new_columns(self, seen): self.logger.debug("Check for new columns") for column in seen - set(self.existing): self.logger.debug("Register column %s" % column) self.index.loggercolumn_set.create(name=column) def save_column_values(self, columns): self.logger.debug("Saving values for filtered columns") ts = time.time() for column in self.filtered: values = columns.get(column) if not values: continue key = keys.KEY_COLUMN_VALUES.format(index=self.index.name, column=column) values = {v: ts for v in values} self.redis.zadd(key, **values) def prepare_query(self, name): if self.indexed: self.query = "REPLACE INTO %s (id, js, logline, %s) VALUES " % ( name, ', '.join(self.indexed)) else: self.query = "REPLACE INTO %s (id, js, logline) VALUES " % name sql_col_count = len(self.indexed) + 3 # + jd, js, logline self.values_stub = "(%s)" % ", ".join(["%s"] * sql_col_count) def load_index_columns(self): # all defined columns all_columns = list(self.index.loggercolumn_set.all()) included_columns = [c for c in all_columns if not c.excluded] filtered_columns = [c for c in included_columns if c.filtered] indexed_columns = [c for c in included_columns if c.indexed] self.existing = [c.name for c in all_columns] self.included = [c.name for c in included_columns] self.filtered = [c.name for c in filtered_columns] self.indexed = [c.name for c in indexed_columns] @staticmethod def process_js_columns(js, columns, included, seen): for key, value in list(js.items()): if key in ('pk', 'id', 'ts', 'ms', 'seq', 'model'): # escape fields reserved by Django and ALCO js['%s_x' % key] = js.pop(key) key = '%s_x' % key # save seen columns set if key not in seen: seen.add(key) if key not in included: # discard fields excluded from indexing js.pop(key) continue # save column values set if type(value) not in (bool, int, float, six.text_type): continue if value not in columns[key]: columns[key].add(value) def get_primary_keys(self, messages): """ Generate PK sequence for a list of messages.""" pkeys = [] pk = None for msg in messages: # pk is [timestamp][microseconds][randint] in 10based integer pk = int((msg['ts'] * 10**6 + msg['ms']) * 1000) + randint(0, 1000) pkeys.append(pk) self.logger.debug("first pk is %s" % pk) return pkeys
class RabbitHelper(object): def __init__(self, mq_server = None, virtual_host = cfg.CB_CLUSTER_TAG): if mq_server == None: mq_server = cfg.RABBITMQ_IP self.connection = Connection(host= mq_server, userid="guest", password="******", virtual_host = virtual_host) self.manager = Client(mq_server+":55672", "guest", "guest") def declare(self, queue = None, durable = True): res = None channel = self.connection.channel() if queue: if not isinstance(queue,str): queue = str(queue) res = channel.queue_declare(queue = queue, durable = durable, auto_delete = True) else: # tmp queue res = channel.queue_declare(exclusive = True) channel.close() return res def exchange_declare(self, exchange, type_='direct'): channel = self.connection.channel() channel.exchange_declare(exchange = exchange, type=type_) channel.close() def bind(self, exchange, queue): channel = self.connection.channel() channel.queue_bind(exchange = exchange, queue = queue) channel.close() def delete(self, queue): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) channel.queue_delete(queue=queue) channel.close() def purge(self, queue): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) channel.queue_purge(queue=queue) channel.close() def channel(self): return self.connection.channel(), self.connection def qsize(self, queue): size = 0 if queue != None: if not isinstance(queue,str): queue = str(queue) response = self.declare(queue = queue) size = response[1] return size def broadcastMsg(self, routing_key, body): channel = self.connection.channel() rc = channel.basic_publish(exchange = '', routing_key = routing_key, body = body) channel.close() def getExchange(self, vhost, exchange): return self.manager.get_exchange(vhost, exchange) def numExchangeQueues(self, vhost, exchange): try: ex = self.getExchange(vhost, exchange) return len(ex['outgoing']) except Exception: return 1 # todo: sometimes the broker doesn't return expected response def putMsg(self, routing_key, body, exchange = ''): channel = self.connection.channel() if not isinstance(routing_key, str): routing_key= str(routing_key) rc = channel.basic_publish(exchange = exchange, routing_key = routing_key, body = body) channel.close() def getMsg(self, queue, no_ack = False, requeue = False): channel = self.connection.channel() message = channel.basic_get(queue = queue) body = None if message is not None: body = message.body # Handle data receipt acknowldegement if no_ack == False: message.ack() if requeue: self.putMsg(queue, body) channel.close() return body def getJsonMsg(self, queue, no_ack = False, requeue = False): msg = self.getMsg(queue, no_ack, requeue) body = {} if msg is not None: try: body = json.loads(msg) except ValueError: pass return body def close(self): self.connection.close() def __del__(self): self.close()
class RabbitMQ(object): # pylint: disable=r0902,r0903 """Manages our queue connection""" __name__ = "RabbitMQ" def __init__(self, config_parser, queue_name=None): # Get our configs if queue_name is None: self.queue = config_parser.get("rabbitmq", "queue") else: self.queue = queue_name self.host = config_parser.get("rabbitmq", "host") self.port = int(config_parser.get("rabbitmq", "port")) self.delivery_mode = int(config_parser.get("rabbitmq", "delivery_mode")) buffer_size = int(config_parser.get("rabbitmq", "buffer")) if config_parser.get("rabbitmq", "durable").lower() == "true": self.durable = True else: self.durable = False if config_parser.get("rabbitmq", "auto_delete").lower() == "true": self.auto_delete = True else: self.auto_delete = False if config_parser.get("rabbitmq", "refresh").lower() == "none": self.refresh = int(time.time()) + 31536000 else: self.refresh = int(config_parser.get("rabbitmq", "refresh")) # This is a buffer to manage messages that come in while RabbitMQ may # be unavailable self.buffer = Queue.Queue(maxsize=buffer_size) # And set up our connection self.connection = None self.channel = None self.last_connect = 0 self.processed_count = 0 self._connect() self._declare() def __del__(self): self._close() def __unicode__(self): return self.__str__() def __str__(self): return __name__ @timeout_decorator.timeout(5) def _connect(self): """Creates our AMQP connection""" try: self.connection = Connection(host=self.host) self.channel = self.connection.channel() logging.info("Connected to server: " + self.host) self.last_connect = int(time.time()) return True except ConnectionError: logging.error("Unable to connect to server") return None def _close(self): """Closes the AMQP connection""" if self.connection is not None: self.connection.close() logging.debug("Closed connection") @timeout_decorator.timeout(5) def _declare(self): """Declares the queue used for receiving logs""" logging.debug( "Declaring queue: " + self.queue + ", durable = " + repr(self.durable) + ", auto_delete = " + repr(self.auto_delete) ) try: self.channel.queue_declare(queue=self.queue, durable=self.durable, auto_delete=self.auto_delete) return True except AttributeError: # We raise here as faliing to declare the queue is an immediate # show-stopper that things cannot neatly recover from. raise except ChannelError: # This can happen if we are attempting to redefine a queue with # settings that are not representative of those already present. logging.fatal( "A ChannelError occurred. Make sure that you are " + "not attempting to change any existing queue " "declarations." ) logging.fatal(traceback.format_exc()) raise def _refresh_connection(self): """ This refreshes the AMQP connection after timeouts or DNS changes, which is an absolute must-have for load-balanced servers in an environment such as EC2 where the IP address of the host is not guaranteed to remain constant. RabbitMQ does not like network interruptions, so this is our attempt to handle them with a little bit of grace. """ if int(time.time()) - self.last_connect > self.refresh: logging.info("Connection refresh threshold reached") self._close() result = self._connect() return result else: return True def publish(self, log): """Publishes a log entry into the queue.""" logging.debug("Publishing to " + self.queue + ", message: " + str(log)) connection_result = self._refresh_connection() logging.debug("connection_result: " + str(connection_result)) if connection_result is None: logging.info("Buffering log message in publish") self.buffer.put(log) return True try: # Exchanges are not implemented (yet) self.channel.basic_publish(exchange="", routing_key=self.queue, body=log, delivery_mode=self.delivery_mode) self.processed_count = self.processed_count + 1 return True except ConnectionError: # This may happen if the backend server has gone away. We call # _refresh_connection() to try and get it back. If this fails then # the message will be dropped. In the future it would be nice to # have an internal buffer for keeping these messages until the # queue comes back. logging.debug("A ConnectionError was thrown when attempting to publish a " + "message.") self.last_connect = 0 raise
def __init__(self, qn): self.queue_name = qn self.connection = Connection() self.channel = self.connection.channel() self.start_matchmaking_queue()
def __init__(self, mq_server = None): if mq_server == None: mq_server = cfg.RABBITMQ_IP self.connection = Connection(host= mq_server, userid="guest", password="******", virtual_host=cfg.CB_CLUSTER_TAG)
#!/usr/bin/env python import sys import os from librabbitmq import Connection connection = Connection( host=os.getenv("RABBITMQ_HOST", "localhost"), userid=os.getenv("RABBITMQ_USERID", "guest"), password=os.getenv("RABBITMQ_PASSWORD", "guest"), virtual_host=os.getenv("RABBITMQ_VHOST", "/"), ) channel = connection.channel() channel.queue_bind( os.getenv("RABBITMQ_QUEUE", "dump1090"), os.getenv("RABBITMQ_EXCHANGE", "dump1090"), os.getenv("RABBITMQ_ROUTING_KEY", "dump1090"), ) while True: line = sys.stdin.readline() if not line: break # EOF channel.basic_publish( line.strip(), os.getenv("RABBITMQ_EXCHANGE", "dump1090"), os.getenv("RABBITMQ_ROUTING_KEY", "dump1090"), delivery_mode=2, )
class test_Channel(unittest.TestCase): def setUp(self): self.connection = Connection(host="localhost:5672", userid="guest", password="******", virtual_host="/") self.channel = self.connection.channel() self.channel.queue_delete(TEST_QUEUE) self._queue_declare() def test_send_message(self): message = Message( channel=self.channel, body="the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8"), ) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.assertGreater(self.channel.queue_purge(TEST_QUEUE), 2) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) def _queue_declare(self): self.channel.exchange_declare(TEST_QUEUE, "direct") x = self.channel.queue_declare(TEST_QUEUE) self.assertEqual(x.message_count, x[1]) self.assertEqual(x.consumer_count, x[2]) self.assertEqual(x.queue, TEST_QUEUE) self.channel.queue_bind(TEST_QUEUE, TEST_QUEUE, TEST_QUEUE) def test_basic_get_ack(self): message = Message( channel=self.channel, body="the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8"), ) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) while True: x = self.channel.basic_get(TEST_QUEUE) if x: break self.assertIs(self.channel, x.channel) self.assertIn("message_count", x.delivery_info) self.assertIn("redelivered", x.delivery_info) self.assertEqual(x.delivery_info["routing_key"], TEST_QUEUE) self.assertEqual(x.delivery_info["exchange"], TEST_QUEUE) self.assertTrue(x.delivery_info["delivery_tag"]) self.assertTrue(x.properties["content_type"]) self.assertTrue(x.body) x.ack() def test_timeout_burst(self): """Check that if we have a large burst of messages in our queue that we can fetch them with a timeout without needing to receive any more messages.""" message = Message( channel=self.channel, body="the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8"), ) for i in xrange(100): self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) for i in xrange(100): self.connection.drain_events(timeout=0.2) self.assertEquals(len(messages), 100) def test_timeout(self): """Check that our ``drain_events`` call actually times out if there are no messages.""" message = Message( channel=self.channel, body="the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8"), ) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) self.connection.drain_events(timeout=0.1) with self.assertRaises(socket.timeout): self.connection.drain_events(timeout=0.1) self.assertEquals(len(messages), 1) def tearDown(self): if self.channel and self.connection.connected: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
def start_rabbitmq(): vhost_present = False tries = 1 print("\n##### Setting up RabbitMQ @ {0} #####".format(cfg.RABBITMQ_IP)) rabbitmq_client = get_ssh_client(cfg.RABBITMQ_IP) _, stdout, _ = rabbitmq_client.exec_command( "ps aux|grep rabbitmq|grep -v grep|awk \'{print $2}\'") print("Killing existing RabbitMQ process ...") for pid in stdout: if pid == "": continue rabbitmq_client.exec_command("sudo kill -9 {0}".format(pid)) if cfg.RABBITMQ_LOG_LOCATION is not "": print("Deleting RabbitMQ logs from {0}".format( cfg.RABBITMQ_LOG_LOCATION)) rabbitmq_client.exec_command("rm -rf {0}/*.*".format( cfg.RABBITMQ_LOG_LOCATION)) print("Starting RabbitMQ ...") rabbitmq_client.exec_command( "screen -dmS rabbitmq sh -c \'sudo rabbitmq-server start; exec bash;\'" ) time.sleep(20) _, stdout, _ = rabbitmq_client.exec_command("sudo rabbitmqctl status") for line in stdout.readlines(): sys.stdout.write(line) print("Rabbitmq has been restarted and is now running!") _, stdout, _ = rabbitmq_client.exec_command("sudo rabbitmqctl list_vhosts") for line in stdout.readlines(): if not vhost_present: if cfg.CB_CLUSTER_TAG in line: vhost_present = True sys.stdout.write(line) if not vhost_present: print("Adding vhost {0} and setting permissions".format( cfg.CB_CLUSTER_TAG)) rabbitmq_client.exec_command("sudo rabbitmqctl add_vhost {0}".format( cfg.CB_CLUSTER_TAG)) rabbitmq_client.exec_command( "sudo rabbitmqctl set_permissions -p {0} guest '.*' '.*' '.*'". format(cfg.CB_CLUSTER_TAG)) _, stdout, _ = rabbitmq_client.exec_command( "sudo rabbitmqctl list_vhosts") for line in stdout.readlines(): sys.stdout.write(line) time.sleep(30) while True: try: tries += 1 Connection(host=cfg.RABBITMQ_IP, userid="guest", password="******", virtual_host=cfg.CB_CLUSTER_TAG) print("Connected to RabbitMQ vhost") break except Exception as e: print e if tries <= 5: print("Retrying connection {0}/5 ...".format(tries)) rabbitmq_client.exec_command( "sudo rabbitmqctl delete_vhost {0}".format( cfg.CB_CLUSTER_TAG)) rabbitmq_client.exec_command( "sudo rabbitmqctl add_vhost {0}".format( cfg.CB_CLUSTER_TAG)) rabbitmq_client.exec_command( "sudo rabbitmqctl set_permissions -p {0} guest '.*' '.*' '.*'" .format(cfg.CB_CLUSTER_TAG)) time.sleep(30) continue sys.exit(1) rabbitmq_client.close()
class RabbitMQ(object): # pylint: disable=r0902,r0903 """Manages our queue connection""" __name__ = 'RabbitMQ' def __init__(self, config_parser, queue_name=None): # Get our configs if queue_name is None: self.queue = config_parser.get('rabbitmq', 'queue') else: self.queue = queue_name self.host = config_parser.get('rabbitmq', 'host') self.port = int(config_parser.get('rabbitmq', 'port')) self.delivery_mode = int(config_parser.get('rabbitmq', 'delivery_mode')) buffer_size = int(config_parser.get('rabbitmq', 'buffer')) if config_parser.get('rabbitmq', 'durable').lower() == 'true': self.durable = True else: self.durable = False if config_parser.get('rabbitmq', 'auto_delete').lower() == 'true': self.auto_delete = True else: self.auto_delete = False if config_parser.get('rabbitmq', 'refresh').lower() == 'none': self.refresh = int(time.time()) + 31536000 else: self.refresh = int(config_parser.get('rabbitmq', 'refresh')) # This is a buffer to manage messages that come in while RabbitMQ may # be unavailable self.buffer = Queue.Queue(maxsize=buffer_size) # And set up our connection self.connection = None self.channel = None self.last_connect = 0 self.processed_count = 0 self._connect() self._declare() def __del__(self): self._close() def __unicode__(self): return self.__str__() def __str__(self): return __name__ @timeout_decorator.timeout(5) def _connect(self): """Creates our AMQP connection""" try: self.connection = Connection(host=self.host) self.channel = self.connection.channel() logging.info('Connected to server: ' + self.host) self.last_connect = int(time.time()) return True except ConnectionError: logging.error('Unable to connect to server') return None def _close(self): """Closes the AMQP connection""" if self.connection is not None: self.connection.close() logging.debug('Closed connection') @timeout_decorator.timeout(5) def _declare(self): """Declares the queue used for receiving logs""" logging.debug('Declaring queue: ' + self.queue + ', durable = ' + repr(self.durable) + ', auto_delete = ' + repr(self.auto_delete)) try: self.channel.queue_declare(queue=self.queue, durable=self.durable, auto_delete=self.auto_delete) return True except AttributeError: # We raise here as faliing to declare the queue is an immediate # show-stopper that things cannot neatly recover from. raise except ChannelError: # This can happen if we are attempting to redefine a queue with # settings that are not representative of those already present. logging.fatal('A ChannelError occurred. Make sure that you are ' + 'not attempting to change any existing queue ' 'declarations.') logging.fatal(traceback.format_exc()) raise def _refresh_connection(self): """ This refreshes the AMQP connection after timeouts or DNS changes, which is an absolute must-have for load-balanced servers in an environment such as EC2 where the IP address of the host is not guaranteed to remain constant. RabbitMQ does not like network interruptions, so this is our attempt to handle them with a little bit of grace. """ if int(time.time()) - self.last_connect > self.refresh: logging.info('Connection refresh threshold reached') self._close() result = self._connect() return result else: return True def publish(self, log): """Publishes a log entry into the queue.""" logging.debug('Publishing to ' + self.queue + ', message: ' + str(log)) connection_result = self._refresh_connection() logging.debug('connection_result: ' + str(connection_result)) if connection_result is None: logging.info('Buffering log message in publish') self.buffer.put(log) return True try: # Exchanges are not implemented (yet) self.channel.basic_publish(exchange='', routing_key=self.queue, body=log, delivery_mode=self.delivery_mode) self.processed_count = self.processed_count + 1 return True except ConnectionError: # This may happen if the backend server has gone away. We call # _refresh_connection() to try and get it back. If this fails then # the message will be dropped. In the future it would be nice to # have an internal buffer for keeping these messages until the # queue comes back. logging.debug( 'A ConnectionError was thrown when attempting to publish a ' + 'message.') self.last_connect = 0 raise
class test_Delete(unittest.TestCase): def setUp(self): self.connection = Connection(host="localhost:5672", userid="guest", password="******", virtual_host="/") self.channel = self.connection.channel() self.TEST_QUEUE = "pyrabbitmq.testq2" def test_delete(self): """Test that we can declare a channel delete it, and then declare with different properties""" res = self.channel.exchange_declare(self.TEST_QUEUE, "direct") res = self.channel.queue_declare(self.TEST_QUEUE) res = self.channel.queue_bind(self.TEST_QUEUE, self.TEST_QUEUE, self.TEST_QUEUE) # Delete the queue self.channel.queue_delete(self.TEST_QUEUE) # Declare it again x = self.channel.queue_declare(self.TEST_QUEUE, durable=True) self.assertIn("message_count", x) self.assertIn("consumer_count", x) self.assertEqual(x["queue"], self.TEST_QUEUE) self.channel.queue_delete(self.TEST_QUEUE) def test_delete_empty(self): """Test that the queue doesn't get deleted if it is not empty""" self.channel.exchange_declare(self.TEST_QUEUE, "direct") self.channel.queue_declare(self.TEST_QUEUE) self.channel.queue_bind(self.TEST_QUEUE, self.TEST_QUEUE, self.TEST_QUEUE) message = Message("the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8")) self.channel.basic_publish(message, self.TEST_QUEUE, self.TEST_QUEUE) self.assertRaises(ChannelError, self.channel.queue_delete, self.TEST_QUEUE, if_empty=True) #We need to make a new channel after a ChannelError self.channel = self.connection.channel() x = self.channel.basic_get(self.TEST_QUEUE) self.assertTrue(x.body) self.channel.queue_delete(self.TEST_QUEUE, if_empty=True) def tearDown(self): if self.channel: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
class test_Channel(unittest.TestCase): def setUp(self): self.connection = Connection(host="localhost:5672", userid="guest", password="******", virtual_host="/") self.channel = self.connection.channel() self._queue_declare() def test_send_message(self): message = Message("the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8")) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.assertGreater(self.channel.queue_purge(TEST_QUEUE), 2) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) def _queue_declare(self): self.channel.exchange_declare(TEST_QUEUE, "direct") x = self.channel.queue_declare(TEST_QUEUE) self.assertIn("message_count", x) self.assertIn("consumer_count", x) self.assertEqual(x["queue"], TEST_QUEUE) self.channel.queue_bind(TEST_QUEUE, TEST_QUEUE, TEST_QUEUE) def test_basic_get_ack(self): message = Message("the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8")) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) while True: x = self.channel.basic_get(TEST_QUEUE) if x: break self.assertIs(self.channel, x.channel) self.assertIn("message_count", x.delivery_info) self.assertIn("redelivered", x.delivery_info) self.assertEqual(x.delivery_info["routing_key"], TEST_QUEUE) self.assertEqual(x.delivery_info["exchange"], TEST_QUEUE) self.assertTrue(x.delivery_info["delivery_tag"]) self.assertTrue(x.properties["content_type"]) self.assertTrue(x.body) x.ack() def test_timeout_burst(self): """Check that if we have a large burst of messages in our queue that we can fetch them with a timeout without needing to receive any more messages.""" message = Message("the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8")) for i in xrange(100): self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) for i in xrange(100): self.connection.drain_events(timeout=0.2) self.assertEquals(len(messages), 100) def test_timeout(self): """Check that our ``drain_events`` call actually times out if there are no messages.""" message = Message("the quick brown fox jumps over the lazy dog", properties=dict(content_type="application/json", content_encoding="utf-8")) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) self.connection.drain_events(timeout=0.1) self.assertRaises(socket.timeout, self.connection.drain_events, timeout=0.1) self.assertEquals(len(messages), 1) def tearDown(self): if self.channel: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
class Collector(object): def __init__(self, index): self.index = index self.cancelled = False self.transport = self.protocol = None self.messages = [] self.block_size = 1000 self.exchange = "logstash" self.current_date = None self.logger = getLogger('alco.collector.%s' % self.index.name) self.amqp = self.redis = self.conn = self.vhost = self.rabbit = None self.insert_thread = None self.query_queue = queue.Queue() self.result_queue = queue.Queue() self.result_queue.put(None) self.query = self.values_stub = None self.existing = self.included = self.indexed = self.filtered = None def cancel(self): self.cancelled = True def inserter_loop(self): self.conn = connections[ALCO_SETTINGS['SPHINX_DATABASE_NAME']] while not self.cancelled: try: query, args = self.query_queue.get(block=True, timeout=1) except queue.Empty: continue result = self.insert_data(query, args) self.result_queue.put(result) def connect(self): connections['default'].close() rabbitmq = ALCO_SETTINGS['RABBITMQ'] self.amqp = Connection(**rabbitmq) self.redis = redis.Redis(**ALCO_SETTINGS['REDIS']) self.insert_thread = Thread(target=self.inserter_loop) self.insert_thread.start() hostname = '%s:%s' % (rabbitmq['host'], ALCO_SETTINGS['RABBITMQ_API_PORT']) self.rabbit = Client(hostname, rabbitmq['userid'], rabbitmq['password']) self.vhost = rabbitmq['virtual_host'] # noinspection PyUnusedLocal def process_sigint(self, signum, frame): self.logger.info("Got signal %s" % signum) self.cancel() self.logger.info("Futures cancelled, wait for thread") self.insert_thread.join() self.logger.info("Thread done") def __call__(self): signal.signal(signal.SIGINT, self.process_sigint) signal.signal(signal.SIGTERM, self.process_sigint) try: self.logger.debug("Connecting to RabbitMQ") self.connect() self.declare_queue() self.cleanup_bindings() channel = self.amqp.channel() channel.basic_qos(0, 1000, False) channel.basic_consume(self.index.queue_name, callback=self.process_message, no_ack=True) start = time.time() self.logger.debug("Start processing messages") while not self.cancelled: try: self.amqp.drain_events(timeout=1) except (socket.timeout, OSError): pass if time.time() - start > 1: self.push_messages() start = time.time() except KeyboardInterrupt: self.logger.warning("Got SIGINT, exit(0)") finally: self.amqp.close() sys.exit(0) def process_message(self, msg): data = json.loads(six.binary_type(msg.body)) ts = data.pop('@timestamp') data.pop("@version") msg = data.pop('message') seq = data.pop('seq', 0) dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") result = { 'ts': time.mktime(dt.timetuple()), 'ms': dt.microsecond, 'seq': seq, 'message': msg, 'data': data } self.messages.append(result) d = dt.date() if not self.current_date: self.current_date = d if d != self.current_date: self.current_date = d self.push_messages() if len(self.messages) >= self.block_size: self.push_messages() def declare_queue(self): channel = self.amqp.channel() """:type channel: amqp.channel.Channel""" durable = self.index.durable channel.exchange_declare(exchange=self.exchange, type='topic', durable=durable, auto_delete=False) channel.queue_declare(self.index.queue_name, durable=durable, auto_delete=False) for rk in self.get_routing_keys(): channel.queue_bind(self.index.queue_name, exchange=self.exchange, routing_key=rk) def get_routing_keys(self): return map(lambda x: x.strip(), self.index.routing_key.split(',')) def cleanup_bindings(self): self.logger.debug("Checking bindings") queue = self.index.queue_name exchange = self.exchange bindings = self.rabbit.get_queue_bindings(self.vhost, queue) bindings = [b for b in bindings if b['source'] == exchange] allowed = self.get_routing_keys() q = six.moves.urllib.parse.quote for b in bindings: rk = b['routing_key'] if rk in allowed: continue self.logger.debug("Unbind %s with RK=%s" % (queue, rk)) self.rabbit.delete_binding(self.vhost, exchange, q(queue), q(rk)) def push_messages(self): try: request_started.send(None, environ=None) self._push_messages() except Exception as e: self.logger.exception(e) raise finally: request_finished.send(None) def _push_messages(self): messages, self.messages = self.messages, [] if not messages: return message_count = len(messages) self.logger.info("Saving %s events" % message_count) columns = defaultdict(set) suffix = self.current_date.strftime("%Y%m%d") name = "%s_%s" % (self.index.name, suffix) args = [] self.load_index_columns() self.prepare_query(name) pkeys = self.get_primary_keys(messages) seen = set() for pk, data in zip(pkeys, messages): # saving seen columns to LoggerColumn model, collecting unique # values for caching in redis js = data['data'] self.process_js_columns(js, columns, self.included, seen) js_str = json.dumps(js) values = tuple(js.get(c) or '' for c in self.indexed) args.extend((pk, js_str, data['message']) + values) query = self.query + ','.join([self.values_stub] * message_count) self.save_column_values(columns) self.save_new_columns(seen) if self.result_queue.empty(): self.logger.debug("Insert still running, waiting") while not self.cancelled: try: self.result_queue.get(block=True, timeout=1) except queue.Empty: continue self.query_queue.put((query, args)) def insert_data(self, query, args): self.logger.debug("Inserting logs to searchd") result = None for _ in 1, 2, 3: try: c = self.conn.cursor() result = c.execute(query, args) self.logger.debug("%s rows inserted" % c.rowcount) c.close() except ProgrammingError: self.logger.exception("Can't insert values to index: %s" % query) except DatabaseError as e: self.logger.exception("Sphinx connection error: %s" % e) try: close_old_connections() except Exception as e: self.logger.exception("Can't reconnect: %s" % e) os.kill(os.getpid(), signal.SIGKILL) except Exception: self.logger.exception("Unhandled error in insert_data") else: return result self.logger.error("Can't insert data in 3 tries, exit process") os.kill(os.getpid(), signal.SIGKILL) def save_new_columns(self, seen): self.logger.debug("Check for new columns") for column in seen - set(self.existing): self.logger.debug("Register column %s" % column) self.index.loggercolumn_set.create(name=column) def save_column_values(self, columns): self.logger.debug("Saving values for filtered columns") ts = time.time() for column in self.filtered: values = columns.get(column) if not values: continue key = keys.KEY_COLUMN_VALUES.format(index=self.index.name, column=column) values = {v: ts for v in values} self.redis.zadd(key, **values) def prepare_query(self, name): if self.indexed: self.query = "REPLACE INTO %s (id, js, logline, %s) VALUES " % ( name, ', '.join(self.indexed)) else: self.query = "REPLACE INTO %s (id, js, logline) VALUES " % name sql_col_count = len(self.indexed) + 3 # + jd, js, logline self.values_stub = "(%s)" % ", ".join(["%s"] * sql_col_count) def load_index_columns(self): # all defined columns all_columns = list(self.index.loggercolumn_set.all()) included_columns = [c for c in all_columns if not c.excluded] filtered_columns = [c for c in included_columns if c.filtered] indexed_columns = [c for c in included_columns if c.indexed] self.existing = [c.name for c in all_columns] self.included = [c.name for c in included_columns] self.filtered = [c.name for c in filtered_columns] self.indexed = [c.name for c in indexed_columns] @staticmethod def process_js_columns(js, columns, included, seen): for key, value in list(js.items()): if key in ('pk', 'id', 'ts', 'ms', 'seq', 'model'): # escape fields reserved by Django and ALCO js['%s_x' % key] = js.pop(key) key = '%s_x' % key # save seen columns set if key not in seen: seen.add(key) if key not in included: # discard fields excluded from indexing js.pop(key) continue # save column values set if type(value) not in (bool, int, float, six.text_type): continue if value not in columns[key]: columns[key].add(value) def get_primary_keys(self, messages): """ Generate PK sequence for a list of messages.""" pkeys = [] pk = None for msg in messages: # pk is [timestamp][microseconds][randint] in 10based integer pk = int((msg['ts'] * 10**6 + msg['ms']) * 1000) + randint(0, 1000) pkeys.append(pk) self.logger.debug("first pk is %s" % pk) return pkeys
class test_Channel(unittest.TestCase): def setUp(self): self.connection = Connection(host='localhost:5672', userid='guest', password='******', virtual_host='/') self.channel = self.connection.channel() self._queue_declare() def test_send_message(self): message = Message( 'the quick brown fox jumps over the lazy dog', properties={ 'content_type': 'application/json', 'content_encoding': 'utf-8', }, ) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.assertGreater(self.channel.queue_purge(TEST_QUEUE), 2) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) def _queue_declare(self): self.channel.exchange_declare(TEST_QUEUE, 'direct') x = self.channel.queue_declare(TEST_QUEUE) self.assertIn('message_count', x) self.assertIn('consumer_count', x) self.assertEqual(x['queue'], TEST_QUEUE) self.channel.queue_bind(TEST_QUEUE, TEST_QUEUE, TEST_QUEUE) def test_basic_get_ack(self): message = Message( 'the quick brown fox jumps over the lazy dog', properties={ 'content_type': 'application/json', 'content_encoding': 'utf-8', }, ) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) while True: x = self.channel.basic_get(TEST_QUEUE) if x: break self.assertIs(self.channel, x.channel) self.assertIn('message_count', x.delivery_info) self.assertIn('redelivered', x.delivery_info) self.assertEqual(x.delivery_info['routing_key'], TEST_QUEUE) self.assertEqual(x.delivery_info['exchange'], TEST_QUEUE) self.assertTrue(x.delivery_info['delivery_tag']) self.assertTrue(x.properties['content_type']) self.assertTrue(x.body) x.ack() def test_timeout_burst(self): """Check that if we have a large burst of messages in our queue that we can fetch them with a timeout without needing to receive any more messages.""" message = Message('the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) for i in xrange(100): self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) for i in xrange(100): self.connection.drain_events(timeout=0.2) self.assertEquals(len(messages), 100) def test_timeout(self): """Check that our ``drain_events`` call actually times out if there are no messages.""" message = Message('the quick brown fox jumps over the lazy dog', properties=dict(content_type='application/json', content_encoding='utf-8')) self.channel.basic_publish(message, TEST_QUEUE, TEST_QUEUE) messages = [] def cb(x): messages.append(x) x.ack() self.channel.basic_consume(TEST_QUEUE, callback=cb) self.connection.drain_events(timeout=0.1) self.assertRaises( socket.timeout, self.connection.drain_events, timeout=0.1, ) self.assertEquals(len(messages), 1) def tearDown(self): if self.channel: self.channel.queue_purge(TEST_QUEUE) self.channel.close() if self.connection: try: self.connection.close() except ConnectionError: pass
def get_conn(self): from librabbitmq import Connection return Connection(**self.conn_info)
#!/usr/bin/env python import psycopg2 as dbapi2 import sys import os from librabbitmq import Connection # Make a DB connection try: db = dbapi2.connect (host=os.getenv('DUMP1090_HOST', 'localhost'), database=os.getenv('DUMP1090_DATABASE', 'dump1090'), user=os.getenv('DUMP1090_USER', 'dump1090'), password=os.getenv('DUMP1090_PASSWORD', 'dump1090')) except: print "I am unable to connect to the database" cur = db.cursor() connection = Connection(host=os.getenv('RABBITMQ_HOST', 'localhost'), userid=os.getenv('RABBITMQ_USERID', 'guest'), password=os.getenv('RABBITMQ_PASSWORD', 'guest'), virtual_host=os.getenv('RABBITMQ_VHOST', '/')) channel = connection.channel() channel.queue_bind(os.getenv('RABBITMQ_QUEUE', 'dump1090'), os.getenv('RABBITMQ_EXCHANGE', 'dump1090'), os.getenv('RABBITMQ_ROUTING_KEY', 'dump1090')) def dump_message(message): #print("Body:'%s', Properties:'%s', DeliveryInfo:'%s'" % (message.body, message.properties, message.delivery_info)) field = '{0}'.format(message.body).rstrip().split(',') # Not all valid messages have been handled. For example: # STA,,5,179,400AE7,10103,2008/11/28,14:58:51.153,2008/11/28,14:58:51.153,RM MessageType = field[0] TransmissionType = None if field[1] == '' else field[1] SessionID = None if field[2] == '' else field[2] AircraftID = None if field[3] == '' else field[3] HexIdent = None if field[4] == '' else field[4]
# coding=utf-8 import sys from librabbitmq import Connection connection = Connection(host='localhost', userid='dongwm', password='******', virtual_host='web_develop') channel = connection.channel() channel.exchange_declare('web_develop', 'direct', passive=False, durable=True, auto_delete=False) if len(sys.argv) != 1: msg = sys.argv[1] else: msg = 'hah' channel.basic_publish(msg, 'web_develop', 'xxx_routing_key') connection.close()
class RabbitHelper(object): def __init__(self, mq_server = None, virtual_host = cfg.CB_CLUSTER_TAG): if mq_server == None: mq_server = cfg.RABBITMQ_IP self.connection = Connection(host= mq_server, userid="guest", password="******", virtual_host = virtual_host) def declare(self, queue, durable = True): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) res = channel.queue_declare(queue = queue, durable = durable, auto_delete = True) channel.close() return res def delete(self, queue): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) channel.queue_delete(queue=queue) channel.close() def purge(self, queue): channel = self.connection.channel() if not isinstance(queue,str): queue = str(queue) channel.queue_purge(queue=queue) channel.close() def consume(self, callback, queue, no_ack = True): channel = self.connection.channel() channel.basic_consume(callback, queue = queue, no_ack = no_ack) channel.start_consuming() channel.close() def qsize(self, queue): size = 0 if queue != None: if not isinstance(queue,str): queue = str(queue) response = self.declare(queue = queue) size = response[1] return size def putMsg(self, queue, body): channel = self.connection.channel() if not isinstance(queue, str): queue = str(queue) rc = channel.basic_publish(exchange = '', routing_key=queue, body = body) channel.close() def getMsg(self, queue, no_ack = False, requeue = False): channel = self.connection.channel() message = channel.basic_get(queue = queue) body = None if message is not None: body = message.body # Handle data receipt acknowldegement if no_ack == False: message.ack() if requeue: self.putMsg(queue, body) channel.close() return body def getJsonMsg(self, queue, no_ack = False, requeue = False): msg = self.getMsg(queue, no_ack, requeue) body = {} if msg is not None: try: body = json.loads(msg) except ValueError: pass return body def close(self): self.connection.close() def __del__(self): self.close()
import os from librabbitmq import Connection x = Connection() c = x.channel() c.exchange_declare('getmem') c.queue_declare('getmem') c.queue_bind('getmem', 'getmem', 'getmem') from time import sleep for i in xrange(10000): c.basic_publish('foo', exchange='getmem', routing_key='getmem') if not i % 1000: print('sent %s' % i) for i in xrange(10000): assert c.basic_get('getmem', no_ack=True) if not i % 1000: print(i) os.system('sh -c "ps auxww | grep %d | grep -v grep "' % os.getpid())