def consume_messages(amqp_string):
conn = BrokerConnection(amqp_string)
queue = conn.SimpleQueue("eventstream")
while True:
try:
message = queue.get(block=False, timeout=1)
except Empty:
break
else:
message.ack()
pl = message.payload
log_message = {
'state': pl['state'],
'state_msg': pl['state_msg'],
'host': pl['host'],
'body': pl['body'],
'timestamp': pl['timestamp'],
'html': loader.load("message.html").generate(message=pl)
}
MessageHandler.update_cache(log_message)
MessageHandler.send_updates(log_message)
queue.close()
conn.close()
class FanoutPublisher(PluginBase):
def __init__(self):
if app.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(app.config['AMQP_URL'])
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', app.config['AMQP_URL'], e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = app.config['AMQP_TOPIC']
self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
LOG.info('Configured fanout publisher on topic "%s"', app.config['AMQP_TOPIC'])
def pre_receive(self, alert):
return alert
def post_receive(self, alert):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(), app.config['AMQP_TOPIC'])
LOG.debug('Message: %s', alert.get_body())
self.producer.publish(alert.get_body(), declare=[self.exchange], retry=True)
class Worker(ConsumerMixin):
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks',self.on_message)
self.__amqp_url = kwargs.get('amqp_url',AMQP_URL)
self.__queue = kwargs.get('queue')
self.__conn_retries = kwargs.get('max_retries',2)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__conn_retries, callback=self.on_conn_retry)
def get_consumers(self, consumer, channel):
if not isinstance(self.__callbacks,list):
self.__callbacks = [ self.__callbacks ]
return [consumer(self.__queue, callbacks=self.__callbacks)]
def on_message(self, body, message):
out = 'Received message: %r' % dumps(body)
out += ' properties: %s' % dumps(message.properties)
out += ' delivery_info: %s' % dumps(message.delivery_info)
LOG.info(out,json=True)
message.ack()
def on_conn_retry(self):
LOG.error('Retrying connection for {0}'.format(self.__amqp_url))
def start(self):
LOG.info('Starting AMQP worker {0}'.format(self.__queue))
self.run()
def stop(self):
LOG.info('Stopping AMQP worker {0}'.format(self.__queue))
self.should_stop = True
class MqServer(object):
"""
exchange='E_X7_W2S', queue='Q_X7_W2S',routing_key = 'RK_X7_W2S'
"""
def __init__(self, callback, kwargs):
self.callback = callback
if kwargs:
self.kwargs = kwargs
else:
self.kwargs = MqDict
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
self.conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
self.queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = self.conn.channel()
consumer = Consumer(channel, self.queue, callbacks=[self.callback])
consumer.consume()
def run(self, once=False):
if once:
self.conn.drain_events()
else:
while True:
self.conn.drain_events()
def get(self):
message = self.queue.get(block=True)
message.ack()
return message
def run(self):
# Setup connection
mainLogger.debug('Connecting to Redis on %s %s %s' % (
agentConfig['redis_host'], agentConfig['redis_port'], agentConfig['redis_db'])
)
connection = BrokerConnection(
hostname=agentConfig['redis_host'],
transport="redis",
virtual_host=agentConfig['redis_db'],
port=int(agentConfig['redis_port'])
)
connection.connect()
consumer = Consumer(connection)
while True:
try:
consumer.consume()
except Empty:
mainLogger.debug('No tasks, going to sleep')
# sleep is patched and triggers context switching
# for eventlet
time.sleep(1)
mainLogger.debug('Waiting')
mainLogger.debug('Done & exit')
class RabbitMQHandler(object):
def __init__(self, connection_string, exchange):
self._connection = BrokerConnection(connection_string)
self._connections = set([self._connection]) # set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def _get_producer(self):
producer = producers[self._connection].acquire(block=True, timeout=2)
self._connections.add(producer.connection)
return producer
def publish(self, item, contributor):
with self._get_producer() as producer:
producer.publish(item, exchange=self._exchange, routing_key=contributor, declare=[self._exchange])
def info(self):
if not self._is_active:
return {}
with self._get_producer() as producer:
res = producer.connection.info()
if 'password' in res:
del res['password']
return res
def wait_many(timeout=1):
#: Create connection
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection("amqp://*****:*****@localhost:5672//")
#: SimpleQueue mimics the interface of the Python Queue module.
#: First argument can either be a queue name or a kombu.Queue object.
#: If a name, then the queue will be declared with the name as the queue
#: name, exchange name and routing key.
queue = connection.SimpleQueue("kombu_demo")
while True:
try:
message = queue.get(block=False, timeout=timeout)
except Empty:
break
else:
spawn(message.ack)
print(message.payload)
queue.close()
connection.close()
def main():
filename = "meta"
fptr = open(filename, "r")
amqpurl = fptr.readline().strip()
exchange_name = fptr.readline().strip()
exchange = Exchange(exchange_name, type="direct")
D_queue = Queue(exchange_name, exchange, routing_key=exchange_name, auto_delete=False, exclusive=False)
connection = BrokerConnection(amqpurl)
print amqpurl
channel = connection.channel()
queue = D_queue(channel)
queue.declare()
producer = Producer(channel, exchange, routing_key=exchange_name)
message_count = int(sys.argv[1])
imgsize = int(sys.argv[2])
name = sys.argv[3]
s3url = ""
if 'S3_URL' in os.environ:
s3url = os.environ['S3_URL']
s3id = os.environ['EC2_ACCESS_KEY']
s3pw = os.environ['EC2_SECRET_KEY']
n = datetime.now()
print "XXX starting %s" % (str(n))
msg_list = []
dashi_name = str(uuid.uuid4()).split('-')[0]
for i in range(0, message_count):
msg = {'program': 'python node2.py %d %d %d' % (i, message_count, imgsize),
'rank': i,
's3url': s3url,
's3id': s3id,
's3pw': s3pw,
'testname': name,
'dashiname': dashi_name}
msg_list.append(msg)
random.shuffle(msg_list)
print "Add the messages to the queue..."
for msg in msg_list:
print "%s %d of %d" % (msg['testname'], msg['rank'], message_count)
sys.stdout.flush()
producer.publish(msg,
exchange=exchange,
routing_key=exchange_name,
serializer="json")
dashi = get_dashi_connection(amqpurl, dashi_name)
p_con = get_phantom_con(s3id, s3pw)
wait_till_done(dashi, message_count, p_con, name)
n = datetime.now()
print "XXX done %s" % (str(n))
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
# queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = conn.channel()
self.producer = Producer(channel, exchange, routing_key=self.kwargs["X7_RK"])
def exchange_send(data,exchange):
try:
connection = BrokerConnection()
channel = connection.channel()
producer = Producer(channel, Exchange(exchange, type="fanout"))
producer.publish(data)
channel.close()
connection.close()
except Exception, error:
print(error)
def __init__(self,
server_id,
amqp_host='localhost',
amqp_user ='******',
amqp_password='******',
amqp_vhost='/',
amqp_port=5672,
ssl=False,
threaded=False):
self.logger = logging.getLogger('callme.server')
self.logger.debug('Server ID: %s' % server_id)
self.server_id = server_id
self.threaded = threaded
self.do_run = True
self.is_stopped = True
self.func_dict={}
self.result_queue = queue.Queue()
target_exchange = Exchange("server_"+server_id+"_ex", "direct", durable=False,
auto_delete=True)
self.target_queue = Queue("server_"+server_id+"_queue", exchange=target_exchange,
auto_delete=True, durable=False)
self.connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
try:
self.connection.connect()
except IOError:
self.logger.critical("Connection Error: Probably AMQP User has not enough permissions")
raise ConnectionError("Connection Error: Probably AMQP User has not enough permissions")
self.channel = self.connection.channel()
self.publish_connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
self.publish_channel = self.publish_connection.channel()
# consume
self.consumer = Consumer(self.channel, self.target_queue)
if self.threaded == True:
self.consumer.register_callback(self._on_request_threaded)
else:
self.consumer.register_callback(self._on_request)
self.consumer.consume()
self.logger.debug('Init done')
class Messaging(object):
amqp_opts = {
'amqp_queue': '', # do not send to queue by default
'amqp_topic': 'notify',
'amqp_url': 'amqp://*****:*****@localhost:5672//', # RabbitMQ
# 'amqp_url': 'mongodb://*****:*****@' # AWS SQS (must define amqp_queue)
# 'amqp_sqs_region': 'eu-west-1' # required if SQS is used
}
def __init__(self):
config.register_opts(Messaging.amqp_opts)
if CONF.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = None
self.connect()
def connect(self):
if not CONF.amqp_url:
return
if CONF.amqp_url.startswith('sqs://'):
CONF.amqp_url = 'sqs://' + CONF.amqp_url[6:].replace('/', '%2F')
if CONF.amqp_sqs_region:
transport_options = {'region': CONF.amqp_sqs_region}
else:
transport_options = {}
self.connection = BrokerConnection(
CONF.amqp_url,
transport_options=transport_options
)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', CONF.amqp_url, e)
sys.exit(1)
LOG.info('Connected to broker %s', CONF.amqp_url)
def disconnect(self):
return self.connection.release()
def is_connected(self):
return self.connection.connected
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks', self.on_message)
self.__amqp_url = kwargs.get('amqp_url', AMQP_URL)
self.__queue = kwargs.get('queue')
self.__max_retries = kwargs.get('max_retries', 2)
self.__max_error = kwargs.get('max_error', 3)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__max_retries,
errback=self.on_connection_error,
callback=self.on_conn_retry)
class Audit:
def __init__(self, hostname='localhost', port='5672',
userid='', password='', virtual_host='graylog',
exchange=None):
self.hostname = hostname
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.connection = BrokerConnection(virtual_host=virtual_host)
self.exchange_setup = exchange or ExchangeSetup()
def custom_exchange(self, exchange, exchange_type, routing_key, queue):
"""Broker exchange can be set after the object has been instantiated.
Args:
exchange (str): Exchange name
exchange_type (str): AMQP exchange type, see your broker manual
routing_key (str)
queue (str)
"""
self.exchange_setup.exchange = exchange
self.exchange_setup.exchange_type = exchange_type
self.exchange_setup.routing_key = routing_key
self.exchange_setup.queue = queue
def log(self, message):
"""Pushes argument object to message broker.
Args:
message (json/gelp): Message can depend on third-party log software
Graylog uses gelp format: https://www.graylog.org/resources/gelf/
"""
if (type(message) is not str) or (message == ''):
print 'Unable to log empty message'
return False
if len(message) > 8192:
print 'Message size too large'
return False
self.connection.connect()
channel = self.connection.channel()
exchange = Exchange(self.exchange_setup.exchange,
type=self.exchange_setup.exchange_type)
bound_exchange = exchange(channel)
bound_exchange.declare()
# example_message = '{"short_message":"Kombu", "host":"example.org"}'
message = bound_exchange.Message(message)
bound_exchange.publish(message, routing_key=self.exchange_setup.routing_key)
self.connection.release()
def _consume(cnx: BrokerConnection, timesup: int) -> None:
try:
cnx.drain_events(timeout=timesup)
except kombu_exceptions.TimeoutError:
pass
except(kombu_exceptions.ChannelLimitExceeded,
kombu_exceptions.ConnectionLimitExceeded,
kombu_exceptions.OperationalError,
kombu_exceptions.NotBoundError,
kombu_exceptions.MessageStateError,
kombu_exceptions.LimitExceeded) as err:
raise ChannelFailureException("Error connecting to RabbitMQ, see inner exception for details", err)
def CallServer(self, method, args=None):
try:
LOG.debug(_("strBroker : %s "), self._strBroker)
connection = BrokerConnection(self._strBroker)
# create the response channel
respQueueName = self._respQueueName + str(uuid())
respconnection = BrokerConnection(self._strBroker)
respQueue = respconnection.SimpleQueue(respQueueName,
queue_opts={
'durable': False,
'auto_delete': True
},
exchange_opts={
'delivery_mode': 1,
'auto_delete': True,
'durable': False
})
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
payload = {
"RespQueue": respQueueName,
"Source": self._strBroker,
'Method': method,
'args': args
}
producer.publish(payload,
exchange=self._exchange,
serializer="json",
routing_key=self._routing_key)
# wait for the response
resp_message = respQueue.get(block=True, timeout=1)
resp_message.ack()
respQueue.close()
#respQueue.delete()
except:
LOG.debug(_("Exception caught : %s"), sys.exc_info()[0])
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
if type(resp_message.payload['Result']).__name__ in ('list', 'tuple'):
nElems = len(resp_message.payload['Result'])
if resp_message.payload['Result'][nElems - 1] == -128:
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
elif type(resp_message.payload['Result']).__name__ == 'int':
if resp_message.payload['Result'] == -128:
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
else:
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
return resp_message.payload['Result']
def _connect(glance_api_cfg):
# We use BrokerConnection rather than Connection as RHEL 6 has an ancient
# version of kombu library.
conn = BrokerConnection(hostname=glance_api_cfg['host'],
port=glance_api_cfg['port'],
userid=glance_api_cfg['userid'],
password=glance_api_cfg['password'],
virtual_host=glance_api_cfg['virtual_host'])
exchange = Exchange(glance_api_cfg['exchange'],
type='topic',
durable=False,
channel=conn.channel())
return conn, exchange
def _connect(glance_api_cfg):
# We use BrokerConnection rather than Connection as RHEL 6 has an ancient
# version of kombu library.
conn = BrokerConnection(hostname=glance_api_cfg['host'],
port=glance_api_cfg['port'],
userid=glance_api_cfg['userid'],
password=glance_api_cfg['password'],
virtual_host=glance_api_cfg['virtual_host'])
exchange = Exchange(glance_api_cfg['exchange'],
type='topic',
durable=False,
channel=conn.channel())
return conn, exchange
def _consume(cnx: BrokerConnection, timesup: int) -> None:
try:
cnx.drain_events(timeout=timesup)
except kombu_exceptions.TimeoutError:
pass
except (kombu_exceptions.ChannelLimitExceeded,
kombu_exceptions.ConnectionLimitExceeded,
kombu_exceptions.OperationalError,
kombu_exceptions.NotBoundError,
kombu_exceptions.MessageStateError,
kombu_exceptions.LimitExceeded) as err:
raise ChannelFailureException(
"Error connecting to RabbitMQ, see inner exception for details",
err)
def __init__(self,
connection_string,
exchange_name,
exchange_type="topic"):
self._connection = BrokerConnection(connection_string)
self._connections = {self._connection
} # set of connection for the heartbeat
self._exchange = Exchange(exchange_name,
durable=True,
delivery_mode=2,
type=exchange_type,
auto_delete=False,
no_declare=False)
monitor_heartbeats(self._connections)
class AMQPWorker(ConsumerMixin):
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks', self.on_message)
self.__amqp_url = kwargs.get('amqp_url', AMQP_URL)
self.__queue = kwargs.get('queue')
self.__max_retries = kwargs.get('max_retries', 2)
self.__max_error = kwargs.get('max_error', 3)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__max_retries,
errback=self.on_connection_error,
callback=self.on_conn_retry)
def get_consumers(self, consumer, channel):
if not isinstance(self.__callbacks, list):
self.__callbacks = [self.__callbacks]
return [consumer(self.__queue, callbacks=self.__callbacks)]
def on_message(self, body, message):
out = {
'message': body,
'properties': message.properties,
'delivery_info': message.delivery_info
}
LOG.info(out, json=True)
message.ack()
def on_conn_retry(self):
LOG.error('Retrying connection for {0}'.format(self.__amqp_url))
def on_connection_error(self, exc, interval):
if self.__max_error:
LOG.warning(
'Connection error, retrying in {0} seconds (retry={1})'.format(
interval, self.__max_error))
self.__max_error -= 1
else:
LOG.error('max connection errors exceeded.')
stop()
def start(self):
LOG.info('Starting AMQP worker {0}'.format(self.__queue))
self.should_stop = False
self.run()
def stop(self):
LOG.info('Stopping AMQP worker {0}'.format(self.__queue))
self.should_stop = True
def connect_rabbitmq():
message = 'hello, this function is connection to RabbitMQ'
print message
print current_app.config['CELERY_BROKER_URL']
conn = BrokerConnection(current_app.config['CELERY_BROKER_URL'],
heartbeat=int(10))
pdf_request.apply_async(['hello'], connection=conn)
conn.release()
return jsonify({
'error_code':
0,
'error_desc':
'Cannot call to IAPI server, check connection again'
})
def add_query(query):
query = query.lower()
queries = fetch_queries()
result = True
slots = getattr(settings, 'KRAL_SLOTS', 1)
if query in queries:
queries.remove(query)
else:
try:
connection = BrokerConnection();
channel = connection.channel();
Exchange(query, type="fanout")(channel).declare()
print('Exchange declared for: %s' % query)
except Exception,error:
print(error)
class FanoutPublisher(PluginBase):
def __init__(self, name=None):
if app.config['DEBUG']:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(AMQP_URL)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL,
e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = AMQP_TOPIC
self.exchange = Exchange(name=self.exchange_name,
type='fanout',
channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
super(FanoutPublisher, self).__init__(name)
LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
def pre_receive(self, alert):
return alert
def post_receive(self, alert):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(),
AMQP_TOPIC)
try:
body = alert.serialize # alerta >= 5.0
# update body's datetime-related fields with utc-aware values
body.update({
key: body[key].replace(tzinfo=pytz.utc)
for key in ['createTime', 'lastReceiveTime', 'receiveTime']
})
except Exception:
body = alert.get_body() # alerta < 5.0
LOG.debug('Message: %s', body)
self.producer.publish(body, declare=[self.exchange], retry=True)
def status_change(self, alert, status, text):
return
def test_migrate(self, name="testcelery"):
x = BrokerConnection("memory://foo")
y = BrokerConnection("memory://foo")
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, "direct")
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish("foo", exchange=name, routing_key=name)
Producer(x).publish("bar", exchange=name, routing_key=name)
Producer(x).publish("baz", exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, "foo")
self.assertEqual(yq.get().body, "bar")
self.assertEqual(yq.get().body, "baz")
Producer(x).publish("foo", exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish("baz", exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate)
self.assertTrue(migrate.called)
with patch("kombu.transport.virtual.Channel.queue_declare") as qd:
def effect(*args, **kwargs):
if kwargs.get("passive"):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y)
x = BrokerConnection("memory://")
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertFalse(callback.called)
def __init__(self, connection: Connection, configuration: BrightsideConsumerConfiguration, logger: logging.Logger=None) -> None:
self._exchange = Exchange(connection.exchange, type=connection.exchange_type, durable=connection.is_durable)
self._routing_key = configuration.routing_key
self._amqp_uri = connection.amqp_uri
self._queue_name = configuration.queue_name
self._routing_key = configuration.routing_key
self._prefetch_count = configuration.prefetch_count
self._is_durable = configuration.is_durable
self._heartbeat = connection.heartbeat
self._connect_timeout = connection.connect_timeout
self._message_factory = ArameMessageFactory()
self._logger = logger or logging.getLogger(__name__)
self._conn = None
consumer_arguments = {}
if configuration.is_ha is True:
consumer_arguments = {"x-ha-policy": "all"}
self._is_long_running_handler = configuration.is_long_runing_handler
self._queue = Queue(self._queue_name, exchange=self._exchange, routing_key=self._routing_key,
durable=self._is_durable, consumer_arguments=consumer_arguments)
self._msg = None # Kombu Message
self._message = None # Brightside Message
self._establish_connection(BrokerConnection(hostname=self._amqp_uri, connect_timeout=self._connect_timeout, heartbeat=self._heartbeat))
self._establish_channel()
self._establish_consumer()
def DjangoBrokerConnection():
return BrokerConnection(
"amqp://%s:%s@%s:%s/%s" %
(settings.BROKER_USER, settings.BROKER_PASSWORD, settings.BROKER_HOST
or "localhost", settings.BROKER_PORT or "5672", settings.BROKER_VHOST
or "/"))
def publish(start=False, startfs=False, stop=False, close=False):
body = json.dumps(close_json)
if start == True:
body = json.dumps(start_json)
elif stop == True:
body = json.dumps(stop_json)
elif startfs == True:
body = json.dumps(startfs_json)
conn = BrokerConnection(hostname=rabbit_host, port=5672, userid=rabbit_user, password=rabbit_password, virtual_host=rabbit_vhost, heartbeat=4)
channel = conn.channel()
exchange = Exchange(rabbit_exchange, type='topic', durable=False)
producer = Producer(exchange=exchange, channel=channel, routing_key=rabbit_routingkey)
producer.publish(body)
def receive(self, timeout: int) -> Message:
def _consume(cnx, timesup):
try:
cnx.drain_events(timeout=timesup)
except kombu_exceptions.TimeoutError:
pass
def _consume_errors(exc, interval):
self._logger.error('Draining error: %s, will retry triggering in %s seconds', exc, interval, exc_info=True)
def _ensure_consumer():
self._queue = Queue(self._queue_name, exchange=self._exchange, routing_key=self._routing_key)
def _read_message(body, message: KombuMessage) -> Message:
self._logger.debug("Monitoring event received at: %s headers: %s payload: %s", datetime.utcnow().isoformat(), message.headers, message.payload)
return self._message_factory.create(message)
# read the next batch number of monitoring messages from the control bus
# evaluate for color coding (error is red)
# print to stdout
_ensure_consumer()
connection = BrokerConnection(hostname=self._amqp_uri)
with connections[connection].acquire(block=True) as conn:
self._logger.debug('Got connection: %s', conn.as_uri())
with Consumer(conn, [self._queue], callbacks=[_read_message], accept=['json', 'text/plain']) as consumer:
consumer.qos(prefetch_count = 1)
ensure_kwargs = self.RETRY_OPTIONS.copy()
ensure_kwargs['errback'] = _consume_errors
safe_drain = conn.ensure(consumer, _consume, **ensure_kwargs)
safe_drain(conn, timeout)
def _consume(cnx: BrokerConnection, timesup: int) -> None:
try:
cnx.drain_events(timeout=timesup)
except kombu_exceptions.TimeoutError:
self._logger.debug("Time out reading from queue %s", self._queue_name)
cnx.heartbeat_check()
except(kombu_exceptions.ChannelLimitExceeded,
kombu_exceptions.ConnectionLimitExceeded,
kombu_exceptions.OperationalError,
kombu_exceptions.NotBoundError,
kombu_exceptions.MessageStateError,
kombu_exceptions.LimitExceeded) as err:
raise ChannelFailureException("Error connecting to RabbitMQ, see inner exception for details", err)
except (OSError, IOError, ConnectionError) as socket_err:
self._reset_connection()
raise ChannelFailureException("Error connecting to RabbitMQ, see inner exception for details", socket_err)
def ready(self):
super(MessagingAppConfig, self).ready()
from kombu import pools
from kombu import BrokerConnection
global connections
global producers
global url
global task_serializer
global broker_transport_options
global broker_socket_timeout
global connection
connections = pools.Connections(limit=100)
producers = pools.Producers(limit=connections.limit)
# run in-memory if broker is not available
# see producer code for synchronous queue
url = getattr(settings, 'BROKER_URL', 'memory://')
task_serializer = getattr(settings, 'CELERY_TASK_SERIALIZER', 'pickle')
broker_transport_options = getattr(settings,
'BROKER_TRANSPORT_OPTIONS',
{'socket_timeout': 10})
broker_socket_timeout = getattr(broker_transport_options,
'socket_timeout', 10)
connection = BrokerConnection(url,
connect_timeout=broker_socket_timeout)
def send_message(self, workername, message):
if self._connected == False: return
qname = Queue(workername,
Exchange("exchange:%s" % workername, type='fanout'))
with BrokerConnection(self._transport_url) as conn:
with conn.SimpleQueue(qname) as queue:
queue.put(message)
def assertConnection(self, cls):
x = cls(app=self.app)
self.assertTrue(x.connection)
conn = BrokerConnection(transport="memory")
x = cls(app=self.app, connection=conn)
self.assertIs(x.connection, conn)
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
self.is_connected = True
if not is_active:
self.is_connected = False
return
self._connection = BrokerConnection(connection_string)
self._connections = set([self._connection
]) #set of connection for the heartbeat
self._exchange = Exchange(exchange,
durable=True,
delivry_mode=2,
type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
class MqReader(object):
def __init__(self, kwargs):
if kwargs:
self.kwargs = kwargs
else:
self.kwargs = MqDict
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
self.conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = self.conn.channel()
self.bound_queue = queue(channel)
# consumer = Consumer(channel, self.queue, callbacks=[self.callback])
# consumer.consume()
def get(self):
message = self.bound_queue.get()
if message is None:
return None
message.ack()
return message
def initconn(self, kwargs):
hostname = 'localhost'
user_id = 'poll_cloud'
password = '******'
virtual_host = 'test',
port = 5672
connect_timeout = 10
heartbeat = 0
self.conn = BrokerConnection(
hostname=kwargs.get('hostname') or hostname,
userid=kwargs.get('userid') or user_id,
password=kwargs.get('password') or password,
port=kwargs.get('port') or port,
virtual_host=kwargs.get('virtual_host') or virtual_host,
connect_timeout=kwargs.get('connect_timeout') or connect_timeout,
heartbeat=kwargs.get('heartbeat') or heartbeat)
def receive(self, timeout: int) -> BrightsideMessage:
self._message = BrightsideMessage(BrightsideMessageHeader(uuid4(), "", BrightsideMessageType.none), BrightsideMessageBody(""))
def _consume(cnx: BrokerConnection, timesup: int) -> None:
try:
cnx.drain_events(timeout=timesup)
except kombu_exceptions.TimeoutError as te:
pass
def _consume_errors(exc, interval: int)-> None:
self._logger.error('Draining error: %s, will retry triggering in %s seconds', exc, interval, exc_info=True)
def _read_message(body: str, msg: KombuMessage) -> None:
self._logger.debug("Monitoring event received at: %s headers: %s payload: %s", datetime.utcnow().isoformat(), msg.headers, body)
self._message = self._message_factory.create_message(msg)
msg.ack()
connection = BrokerConnection(hostname=self._amqp_uri)
with connections[connection].acquire(block=True) as conn:
self._logger.debug('Got connection: %s', conn.as_uri())
with Consumer(conn, queues=[self._queue], callbacks=[_read_message]) as consumer:
consumer.qos(prefetch_count=1)
ensure_kwargs = self.RETRY_OPTIONS.copy()
ensure_kwargs['errback'] = _consume_errors
safe_drain = conn.ensure(consumer, _consume, **ensure_kwargs)
safe_drain(conn, timeout)
return self._message
class test_StdChannel(TestCase):
def setUp(self):
self.conn = BrokerConnection("memory://")
self.channel = self.conn.channel()
self.channel.queues.clear()
self.conn.connection.state.clear()
def test_Consumer(self):
q = Queue("foo")
print(self.channel.queues)
cons = self.channel.Consumer(q)
self.assertIsInstance(cons, Consumer)
self.assertIs(cons.channel, self.channel)
def test_Producer(self):
prod = self.channel.Producer()
self.assertIsInstance(prod, Producer)
self.assertIs(prod.channel, self.channel)
def test_interface_list_bindings(self):
with self.assertRaises(NotImplementedError):
StdChannel().list_bindings()
def test_interface_after_reply_message_received(self):
self.assertIsNone(StdChannel().after_reply_message_received(
Queue("foo")))
def test_migrate(self, name='testcelery'):
x = BrokerConnection('memory://foo')
y = BrokerConnection('memory://foo')
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes('foo'))
self.assertEqual(yq.get().body, ensure_bytes('bar'))
self.assertEqual(yq.get().body, ensure_bytes('baz'))
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertTrue(callback.called)
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate)
self.assertTrue(migrate.called)
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise StdChannelError()
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y)
x = BrokerConnection('memory://')
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback)
self.assertFalse(callback.called)
class Publisher(object):
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
self.is_connected = True
if not is_active:
self.is_connected = False
return
self._connection = BrokerConnection(connection_string)
self._connections = set([self._connection])#set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def _get_producer(self):
producer = producers[self._connection].acquire(block=True, timeout=2)
self._connections.add(producer.connection)
return producer
def publish(self, item, contributor):
if not self._is_active:
return
with self._get_producer() as producer:
try:
producer.publish(item, exchange=self._exchange, routing_key=contributor, declare=[self._exchange])
self.is_connected = True
except socket.error:
self.is_connected = False
logging.getLogger(__name__).debug('Impossible to publish message !')
raise
def info(self):
result = {
"is_active": self._is_active,
"is_connected": self.is_connected
}
if not self._is_active:
return result
with self._get_producer() as producer:
res = producer.connection.info()
if 'password' in res:
del res['password']
for key, value in res.items():
result[key] = value
return result
class RabbitMQHandler(object):
def __init__(self, connection_string, exchange):
self._connection = BrokerConnection(connection_string)
self._connections = {self._connection
} # set of connection for the heartbeat
self._exchange = Exchange(exchange,
durable=True,
delivery_mode=2,
type="topic")
monitor_heartbeats(self._connections)
@retry(wait_fixed=200, stop_max_attempt_number=3)
def publish(self, item, contributor_id):
with self._connection.channel() as channel:
with Producer(channel) as producer:
producer.publish(item,
exchange=self._exchange,
routing_key=contributor_id,
declare=[self._exchange])
def info(self):
info = self._connection.info()
info.pop("password", None)
return info
def connect(self):
self._connection.connect()
def close(self):
for c in self._connections:
c.release()
def listen_load_realtime(self, queue_name, max_retries=10):
log = logging.getLogger(__name__)
route = "task.load_realtime.*"
log.info("listening route {} on exchange {}...".format(
route, self._exchange))
rt_queue = Queue(queue_name,
routing_key=route,
exchange=self._exchange,
durable=False)
RTReloader(connection=self._connection,
rpc_queue=rt_queue,
exchange=self._exchange,
max_retries=max_retries).run()
def __init__(self, app, broker, interval=1):
# self.interval = interval
start_http_server(8000)
self.app = app
self.state = app.events.State()
self.broker_conn = BrokerConnection(broker)
self.gateway = 'localhost:9091'
self.create_metric()
def recv_message(**kwargs):
connection = BrokerConnection('amqp://%(mq_user)s:%(mq_password)s@'
'%(mq_host)s:%(mq_port)s//' % kwargs)
with connection as conn:
try:
SomeConsumer(conn).run()
except KeyboardInterrupt:
LOG.warning('Quitting %s' % __name__)
class AMQPWorker(ConsumerMixin):
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks',self.on_message)
self.__amqp_url = kwargs.get('amqp_url',AMQP_URL)
self.__queue = kwargs.get('queue')
self.__max_retries = kwargs.get('max_retries',2)
self.__max_error = kwargs.get('max_error',3)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__max_retries,
errback=self.on_connection_error, callback=self.on_conn_retry)
def get_consumers(self, consumer, channel):
if not isinstance(self.__callbacks,list):
self.__callbacks = [ self.__callbacks ]
return [consumer(self.__queue, callbacks=self.__callbacks)]
def on_message(self, body, message):
out = {
'message':body,
'properties':message.properties,
'delivery_info': message.delivery_info
}
LOG.info(out, json=True)
message.ack()
def on_conn_retry(self):
LOG.error('Retrying connection for {0}'.format(self.__amqp_url))
def on_connection_error(self, exc, interval):
if self.__max_error:
LOG.warning('Connection error, retrying in {0} seconds (retry={1})'.format(interval, self.__max_error))
self.__max_error -= 1
else:
LOG.error('max connection errors exceeded.')
stop()
def start(self):
LOG.info('Starting AMQP worker {0}'.format(self.__queue))
self.should_stop = False
self.run()
def stop(self):
LOG.info('Stopping AMQP worker {0}'.format(self.__queue))
self.should_stop = True
class KombuLogger(object):
def __init__(self, host="localhost", user="******", password="******", vhost="/", exchange="analytics"):
self.connection = BrokerConnection(host, user, password, vhost)
self.channel = self.connection.channel()
self.exchange = Exchange(exchange, "topic", durable=True, auto_delete=False)
self.producer = Producer(self.channel, exchange=self.exchange, serializer="json")
def write(self, event, timestamp, attributes):
self.producer.publish({"event": event, "ts": timestamp, "attr": attributes}, routing_key=event)
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks',self.on_message)
self.__amqp_url = kwargs.get('amqp_url',AMQP_URL)
self.__queue = kwargs.get('queue')
self.__conn_retries = kwargs.get('max_retries',2)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__conn_retries, callback=self.on_conn_retry)
def initconn(self, kwargs):
HOSTNAME = 'localhost'
USERID = 'poll_cloud'
PASSWORK = ''
VIRTUAL_HOST = 'test'
PORT = 5672
CONNECT_TIMEOUT = 5
HEARTBEAT = 0
self.conn = BrokerConnection(
hostname=kwargs.get('hostname') or HOSTNAME,
userid=kwargs.get('userid') or USERID,
password=kwargs.get('password') or PASSWORK,
virtual_host=kwargs.get('virtual_host') or VIRTUAL_HOST,
port=kwargs.get('port') or PORT,
connect_timeout=kwargs.get('connect_timeout') or CONNECT_TIMEOUT,
heartbeat=kwargs.get('heartbeat') or HEARTBEAT,
)
def consumer(self):
with BrokerConnection('localhost') as conn:
with conn.SimpleQueue(self.queue) as queue:
while True:
message = queue.get()
message.ack()
yield message.payload
message = queue.get()
message.ack()
def _establish_connection(self, conn: BrokerConnection) -> None:
"""
We don't use a pool here. We only have one consumer connection per process, so
we get no value from a pool, and we want to use a heartbeat to keep the consumer
collection alive, which does not work with a pool
:return: the connection to the transport
"""
try:
self._logger.debug("Establishing connection.")
self._conn = conn.ensure_connection(max_retries=3)
self._logger.debug('Got connection: %s', conn.as_uri())
except kombu_exceptions.OperationalError as oe:
self._logger.error("Error connecting to RMQ, could not retry %s", oe)
# Try to clean up the mess
if self._conn is not None:
self._conn.close()
else:
conn.close()
def __init__(self,
connection: ArameConnection,
logger: logging.Logger = None) -> None:
self._amqp_uri = connection.amqp_uri
self._cnx = BrokerConnection(hostname=connection.amqp_uri)
self._exchange = Exchange(connection.exchange,
type=connection.exchange_type,
durable=connection.is_durable)
self._logger = logger or logging.getLogger(__name__)
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
self.conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
self.queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = self.conn.channel()
consumer = Consumer(channel, self.queue, callbacks=[self.callback])
consumer.consume()
def main():
conn = BrokerConnection("amqp://*****:*****@localhost:5672//")
queue = conn.SimpleQueue("eventstream")
queue_options = {'serializer': 'json', 'compression': 'zlib'}
host = socket.gethostname()
while True:
queue.put({
'state': random.choice(bootstrap_labels),
'state_msg': 'Hello',
'host': host,
'timestamp': datetime.datetime.now().isoformat(),
'body': 'foobar'}, **queue_options)
time.sleep(random.random() / 8.0)
queue.close()
conn.close()
def return_hello(*args, **kw):
from test_django2 import tasks
tasks.delay_hello.delay()
from kombu import BrokerConnection
tasks.delay_hello2.apply_async(
(), connection=BrokerConnection(
"amqp://*****:*****@localhost:5672/broker1"))
return HttpResponse("hello world")
def setUp(self):
self.connection = BrokerConnection(transport="pika")
try:
self.connection.connect()
except socket.error:
self.connected = False
else:
self.connected = True
self.exchange = Exchange("tamqplib", "direct")
self.queue = Queue("tamqplib", self.exchange, "tamqplib")
def __init__(self, connection_string, exchange, is_active=True):
self._is_active = is_active
if not is_active:
return
self._connection = BrokerConnection(connection_string)
self._connections = set([self._connection])#set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
self._connection.connect()
monitor_heartbeats(self._connections)
def setUp(self):
self.connection = BrokerConnection(backend_cls="pika")
try:
self.connection.connect()
except socket.error:
self.connected = False
else:
self.connected = True
self.exchange = Exchange("tamqplib", "direct")
self.binding = Binding("tamqplib", self.exchange, "tamqplib")
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (self.user,self.password,self.host,self.port,self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def __init__(self, hostname='localhost', port='5672',
userid='', password='', virtual_host='graylog',
exchange=None):
self.hostname = hostname
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.connection = BrokerConnection(virtual_host=virtual_host)
self.exchange_setup = exchange or ExchangeSetup()
def __connect(self):
self.__connection = BrokerConnection(self.url)
self.channel = self.get_channel()
self.rpc_factory = rpc.RpcFactory(self.channel)
self.publisher_factory = publisher.PublisherFactory(self.channel)
self.consumer_factory = consumer.ConsumerFactory(self.channel)
self.__running = True
Connection.connection = self