class Connection(object):
def initconn(self, kwargs):
HOSTNAME = 'localhost'
USERID = 'poll_cloud'
PASSWORK = ''
VIRTUAL_HOST = 'test'
PORT = 5672
CONNECT_TIMEOUT = 5
HEARTBEAT = 0
self.conn = BrokerConnection(
hostname=kwargs.get('hostname') or HOSTNAME,
userid=kwargs.get('userid') or USERID,
password=kwargs.get('password') or PASSWORK,
virtual_host=kwargs.get('virtual_host') or VIRTUAL_HOST,
port=kwargs.get('port') or PORT,
connect_timeout=kwargs.get('connect_timeout') or CONNECT_TIMEOUT,
heartbeat=kwargs.get('heartbeat') or HEARTBEAT,
)
def producerFunc(self, exchange='amq.direct', ex_type="direct"):
self.chan = self.conn.channel()
self.exchange = Exchange(exchange, type=ex_type)
self.producer = Producer(self.chan, self.exchange)
def consumerFunc(self, exchange='amq.direct', ex_type="direct"):
self.chan = self.conn.channel()
self.exchange = Exchange(exchange, type=ex_type)
def main():
cfg = {
'hostname': 'localhost',
'userid': 'guest',
'password': '******',
'virtual_host': '/',
'port': 5672
}
transport = 'pika'
#transport = 'librabbitmq'
connection = BrokerConnection(transport=transport, **cfg)
connection.connect()
cfg = {
'name': 'simple-test-1',
'auto_delete': True,
'durable': False,
'delivery_mode': 'transient'
}
channel = connection.channel()
exchange = Exchange(channel=channel, **cfg)
#exchange = exchange_def(channel)
routing_key = 'simple-test-1-route'
queue = Queue(exchange=exchange, routing_key=routing_key, **cfg)
channel = connection.channel()
producer = Producer(channel=channel,
exchange=exchange,
routing_key=routing_key)
channel = connection.channel()
consumer = Consumer(channel=channel, queues=[queue], callbacks=[receive])
consumer.consume()
def serve_forever():
while True:
#print 'drain'
#gevent.sleep(0.0001)
connection.drain_events(timeout=1)
def publish_forever():
while True:
producer.publish(loremIpsum)
gevent.sleep(0.0001)
#g1, g2 = gevent.spawn(publish_forever), gevent.spawn(serve_forever)
g2 = gevent.spawn(serve_forever)
g1 = gevent.spawn(publish_forever)
gevent.joinall([g1, g2])
class test_StdChannel(TestCase):
def setUp(self):
self.conn = BrokerConnection("memory://")
self.channel = self.conn.channel()
self.channel.queues.clear()
self.conn.connection.state.clear()
def test_Consumer(self):
q = Queue("foo")
print(self.channel.queues)
cons = self.channel.Consumer(q)
self.assertIsInstance(cons, Consumer)
self.assertIs(cons.channel, self.channel)
def test_Producer(self):
prod = self.channel.Producer()
self.assertIsInstance(prod, Producer)
self.assertIs(prod.channel, self.channel)
def test_interface_list_bindings(self):
with self.assertRaises(NotImplementedError):
StdChannel().list_bindings()
def test_interface_after_reply_message_received(self):
self.assertIsNone(StdChannel().after_reply_message_received(
Queue("foo")))
class FanoutPublisher(PluginBase):
def __init__(self, name=None):
if app.config['DEBUG']:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(AMQP_URL)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL, e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = AMQP_TOPIC
self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
super(FanoutPublisher, self).__init__(name)
LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
def pre_receive(self, alert, **kwargs):
return alert
def post_receive(self, alert, **kwargs):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(), AMQP_TOPIC)
body = alert.get_body(history=self.get_config('AMQP_SEND_ALERT_HISTORY', default=DEFAULT_AMQP_SEND_ALERT_HISTORY, type=bool, **kwargs))
LOG.debug('Message: %s', body)
self.producer.publish(body, declare=[self.exchange], retry=True)
def status_change(self, alert, status, text, **kwargs):
return
class RabbitMQHandler(object):
def __init__(self, connection_string, exchange):
self._connection = BrokerConnection(connection_string)
self._connections = {self._connection} # set of connection for the heartbeat
self._exchange = Exchange(exchange, durable=True, delivry_mode=2, type='topic')
monitor_heartbeats(self._connections)
@retry(wait_fixed=200, stop_max_attempt_number=3)
def publish(self, item, contributor):
with self._connection.channel() as channel:
with Producer(channel) as producer:
producer.publish(item, exchange=self._exchange, routing_key=contributor, declare=[self._exchange])
def info(self):
return self._connection.info()
def listen_load_realtime(self, queue_name, max_retries=10):
log = logging.getLogger(__name__)
route = 'task.load_realtime.*'
log.info('listening route {} on exchange {}...'.format(route, self._exchange))
rt_queue = Queue(queue_name, routing_key=route, exchange=self._exchange, durable=False)
RTReloader(connection=self._connection,
rpc_queue=rt_queue,
exchange=self._exchange,
max_retries=max_retries).run()
class MqServer(object):
"""
exchange='E_X7_W2S', queue='Q_X7_W2S',routing_key = 'RK_X7_W2S'
"""
def __init__(self, callback, kwargs):
self.callback = callback
if kwargs:
self.kwargs = kwargs
else:
self.kwargs = MqDict
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
self.conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
self.queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = self.conn.channel()
consumer = Consumer(channel, self.queue, callbacks=[self.callback])
consumer.consume()
def run(self, once=False):
if once:
self.conn.drain_events()
else:
while True:
self.conn.drain_events()
def get(self):
message = self.queue.get(block=True)
message.ack()
return message
class FanoutPublisher(PluginBase):
def __init__(self):
if app.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(app.config['AMQP_URL'])
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', app.config['AMQP_URL'], e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = app.config['AMQP_TOPIC']
self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
LOG.info('Configured fanout publisher on topic "%s"', app.config['AMQP_TOPIC'])
def pre_receive(self, alert):
return alert
def post_receive(self, alert):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(), app.config['AMQP_TOPIC'])
LOG.debug('Message: %s', alert.get_body())
self.producer.publish(alert.get_body(), declare=[self.exchange], retry=True)
class MqReader(object):
def __init__(self, kwargs):
if kwargs:
self.kwargs = kwargs
else:
self.kwargs = MqDict
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
self.conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = self.conn.channel()
self.bound_queue = queue(channel)
# consumer = Consumer(channel, self.queue, callbacks=[self.callback])
# consumer.consume()
def get(self):
message = self.bound_queue.get()
if message is None:
return None
message.ack()
return message
def main():
filename = "meta"
fptr = open(filename, "r")
amqpurl = fptr.readline().strip()
exchange_name = fptr.readline().strip()
exchange = Exchange(exchange_name, type="direct")
D_queue = Queue(exchange_name, exchange, routing_key=exchange_name, auto_delete=False, exclusive=False)
connection = BrokerConnection(amqpurl)
print amqpurl
channel = connection.channel()
queue = D_queue(channel)
queue.declare()
producer = Producer(channel, exchange, routing_key=exchange_name)
message_count = int(sys.argv[1])
imgsize = int(sys.argv[2])
name = sys.argv[3]
s3url = ""
if 'S3_URL' in os.environ:
s3url = os.environ['S3_URL']
s3id = os.environ['EC2_ACCESS_KEY']
s3pw = os.environ['EC2_SECRET_KEY']
n = datetime.now()
print "XXX starting %s" % (str(n))
msg_list = []
dashi_name = str(uuid.uuid4()).split('-')[0]
for i in range(0, message_count):
msg = {'program': 'python node2.py %d %d %d' % (i, message_count, imgsize),
'rank': i,
's3url': s3url,
's3id': s3id,
's3pw': s3pw,
'testname': name,
'dashiname': dashi_name}
msg_list.append(msg)
random.shuffle(msg_list)
print "Add the messages to the queue..."
for msg in msg_list:
print "%s %d of %d" % (msg['testname'], msg['rank'], message_count)
sys.stdout.flush()
producer.publish(msg,
exchange=exchange,
routing_key=exchange_name,
serializer="json")
dashi = get_dashi_connection(amqpurl, dashi_name)
p_con = get_phantom_con(s3id, s3pw)
wait_till_done(dashi, message_count, p_con, name)
n = datetime.now()
print "XXX done %s" % (str(n))
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
# queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = conn.channel()
self.producer = Producer(channel, exchange, routing_key=self.kwargs["X7_RK"])
class KombuLogger(object):
def __init__(self, host="localhost", user="******", password="******", vhost="/", exchange="analytics"):
self.connection = BrokerConnection(host, user, password, vhost)
self.channel = self.connection.channel()
self.exchange = Exchange(exchange, "topic", durable=True, auto_delete=False)
self.producer = Producer(self.channel, exchange=self.exchange, serializer="json")
def write(self, event, timestamp, attributes):
self.producer.publish({"event": event, "ts": timestamp, "attr": attributes}, routing_key=event)
def exchange_send(data,exchange):
try:
connection = BrokerConnection()
channel = connection.channel()
producer = Producer(channel, Exchange(exchange, type="fanout"))
producer.publish(data)
channel.close()
connection.close()
except Exception, error:
print(error)
class Audit:
def __init__(self, hostname='localhost', port='5672',
userid='', password='', virtual_host='graylog',
exchange=None):
self.hostname = hostname
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.connection = BrokerConnection(virtual_host=virtual_host)
self.exchange_setup = exchange or ExchangeSetup()
def custom_exchange(self, exchange, exchange_type, routing_key, queue):
"""Broker exchange can be set after the object has been instantiated.
Args:
exchange (str): Exchange name
exchange_type (str): AMQP exchange type, see your broker manual
routing_key (str)
queue (str)
"""
self.exchange_setup.exchange = exchange
self.exchange_setup.exchange_type = exchange_type
self.exchange_setup.routing_key = routing_key
self.exchange_setup.queue = queue
def log(self, message):
"""Pushes argument object to message broker.
Args:
message (json/gelp): Message can depend on third-party log software
Graylog uses gelp format: https://www.graylog.org/resources/gelf/
"""
if (type(message) is not str) or (message == ''):
print 'Unable to log empty message'
return False
if len(message) > 8192:
print 'Message size too large'
return False
self.connection.connect()
channel = self.connection.channel()
exchange = Exchange(self.exchange_setup.exchange,
type=self.exchange_setup.exchange_type)
bound_exchange = exchange(channel)
bound_exchange.declare()
# example_message = '{"short_message":"Kombu", "host":"example.org"}'
message = bound_exchange.Message(message)
bound_exchange.publish(message, routing_key=self.exchange_setup.routing_key)
self.connection.release()
def _connect(glance_api_cfg):
# We use BrokerConnection rather than Connection as RHEL 6 has an ancient
# version of kombu library.
conn = BrokerConnection(hostname=glance_api_cfg['host'],
port=glance_api_cfg['port'],
userid=glance_api_cfg['userid'],
password=glance_api_cfg['password'],
virtual_host=glance_api_cfg['virtual_host'])
exchange = Exchange(glance_api_cfg['exchange'],
type='topic',
durable=False,
channel=conn.channel())
return conn, exchange
def _connect(glance_api_cfg):
# We use BrokerConnection rather than Connection as RHEL 6 has an ancient
# version of kombu library.
conn = BrokerConnection(hostname=glance_api_cfg['host'],
port=glance_api_cfg['port'],
userid=glance_api_cfg['userid'],
password=glance_api_cfg['password'],
virtual_host=glance_api_cfg['virtual_host'])
exchange = Exchange(glance_api_cfg['exchange'],
type='topic',
durable=False,
channel=conn.channel())
return conn, exchange
def add_query(query):
query = query.lower()
queries = fetch_queries()
result = True
slots = getattr(settings, 'KRAL_SLOTS', 1)
if query in queries:
queries.remove(query)
else:
try:
connection = BrokerConnection();
channel = connection.channel();
Exchange(query, type="fanout")(channel).declare()
print('Exchange declared for: %s' % query)
except Exception,error:
print(error)
def main():
cfg = {'hostname':'localhost', 'userid':'guest', 'password':'******', 'virtual_host':'/', 'port':5672}
transport = 'pika'
#transport = 'librabbitmq'
connection = BrokerConnection(transport=transport, **cfg)
connection.connect()
cfg = {'name':'simple-test-1', 'auto_delete':True, 'durable':False, 'delivery_mode':'transient'}
channel = connection.channel()
exchange = Exchange(channel=channel, **cfg)
#exchange = exchange_def(channel)
routing_key = 'simple-test-1-route'
queue = Queue(exchange=exchange, routing_key=routing_key, **cfg)
channel = connection.channel()
producer = Producer(channel=channel, exchange=exchange, routing_key=routing_key)
channel = connection.channel()
consumer = Consumer(channel=channel, queues=[queue], callbacks=[receive])
consumer.consume()
def serve_forever():
while True:
#print 'drain'
#gevent.sleep(0.0001)
connection.drain_events(timeout=1)
def publish_forever():
while True:
producer.publish(loremIpsum)
gevent.sleep(0.0001)
#g1, g2 = gevent.spawn(publish_forever), gevent.spawn(serve_forever)
g2 = gevent.spawn(serve_forever)
g1 = gevent.spawn(publish_forever)
gevent.joinall([g1, g2])
class FanoutPublisher(PluginBase):
def __init__(self, name=None):
if app.config['DEBUG']:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(AMQP_URL)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL,
e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = AMQP_TOPIC
self.exchange = Exchange(name=self.exchange_name,
type='fanout',
channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
super(FanoutPublisher, self).__init__(name)
LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
def pre_receive(self, alert):
return alert
def post_receive(self, alert):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(),
AMQP_TOPIC)
try:
body = alert.serialize # alerta >= 5.0
# update body's datetime-related fields with utc-aware values
body.update({
key: body[key].replace(tzinfo=pytz.utc)
for key in ['createTime', 'lastReceiveTime', 'receiveTime']
})
except Exception:
body = alert.get_body() # alerta < 5.0
LOG.debug('Message: %s', body)
self.producer.publish(body, declare=[self.exchange], retry=True)
def status_change(self, alert, status, text):
return
def publish(start=False, startfs=False, stop=False, close=False):
body = json.dumps(close_json)
if start == True:
body = json.dumps(start_json)
elif stop == True:
body = json.dumps(stop_json)
elif startfs == True:
body = json.dumps(startfs_json)
conn = BrokerConnection(hostname=rabbit_host, port=5672, userid=rabbit_user, password=rabbit_password, virtual_host=rabbit_vhost, heartbeat=4)
channel = conn.channel()
exchange = Exchange(rabbit_exchange, type='topic', durable=False)
producer = Producer(exchange=exchange, channel=channel, routing_key=rabbit_routingkey)
producer.publish(body)
class test_Message(TestCase):
def setUp(self):
self.conn = BrokerConnection("memory://")
self.channel = self.conn.channel()
self.message = Message(self.channel, delivery_tag=313)
def test_ack_respects_no_ack_consumers(self):
self.channel.no_ack_consumers = set(["abc"])
self.message.delivery_info["consumer_tag"] = "abc"
ack = self.channel.basic_ack = Mock()
self.message.ack()
self.assertNotEqual(self.message._state, "ACK")
self.assertFalse(ack.called)
def test_ack_missing_consumer_tag(self):
self.channel.no_ack_consumers = set(["abc"])
self.message.delivery_info = {}
ack = self.channel.basic_ack = Mock()
self.message.ack()
ack.assert_called_with(self.message.delivery_tag)
def test_ack_not_no_ack(self):
self.channel.no_ack_consumers = set()
self.message.delivery_info["consumer_tag"] = "abc"
ack = self.channel.basic_ack = Mock()
self.message.ack()
ack.assert_called_with(self.message.delivery_tag)
def test_ack_log_error_when_no_error(self):
ack = self.message.ack = Mock()
self.message.ack_log_error(Mock(), KeyError)
ack.assert_called_with()
def test_ack_log_error_when_error(self):
ack = self.message.ack = Mock()
ack.side_effect = KeyError("foo")
logger = Mock()
self.message.ack_log_error(logger, KeyError)
ack.assert_called_with()
self.assertTrue(logger.critical.called)
self.assertIn("Couldn't ack", logger.critical.call_args[0][0])
class FanoutPublisher(PluginBase):
def __init__(self, name=None):
if app.config['DEBUG']:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(AMQP_URL)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL, e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = AMQP_TOPIC
self.exchange = Exchange(name=self.exchange_name, type='fanout', channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
super(FanoutPublisher, self).__init__(name)
LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
def pre_receive(self, alert):
return alert
def post_receive(self, alert):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(), AMQP_TOPIC)
try:
body = alert.serialize # alerta >= 5.0
# update body's datetime-related fields with utc-aware values
body.update({key: body[key].replace(tzinfo=pytz.utc) for key in ['createTime', 'lastReceiveTime', 'receiveTime']})
except Exception:
body = alert.get_body() # alerta < 5.0
LOG.debug('Message: %s', body)
self.producer.publish(body, declare=[self.exchange], retry=True)
def status_change(self, alert, status, text):
return
class FanoutPublisher(PluginBase):
def __init__(self):
if app.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(app.config['AMQP_URL'])
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s',
app.config['AMQP_URL'], e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = app.config['AMQP_TOPIC']
self.exchange = Exchange(name=self.exchange_name,
type='fanout',
channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
LOG.info('Configured fanout publisher on topic "%s"',
app.config['AMQP_TOPIC'])
def pre_receive(self, alert):
return alert
def post_receive(self, alert):
LOG.info('Sending message %s to AMQP topic "%s"', alert.get_id(),
app.config['AMQP_TOPIC'])
LOG.debug('Message: %s', alert.get_body())
self.producer.publish(alert.get_body(),
declare=[self.exchange],
retry=True)
def status_change(self, alert, status, text):
return
def publish_cmd():
parser = get_parser("publish")
(options, args) = parser.parse_args(sys.argv[2:])
if len(args) < 1:
parser.error("You must provide a queue hostname to publish to!")
exchange = Exchange(options.exchange, type="fanout", delivery_mode="transient")
queue = Queue(options.queue, exchange)
connection = BrokerConnection(hostname=args[0],
userid=options.userid,
password=options.password,
virtual_host=options.vhost)
channel = connection.channel()
producer = Producer(channel, exchange)
pods = pod.load_pod_dir(dirname=options.directory,
timeout=options.timeout,
# The first arg is the queue hostname, the rest
# will be pod names
filter_fn=pod.get_pod_filter(options.pod))
pods.update(pod.get_barker_metadata())
producer.publish(pods, serializer="json")
return 0
class Connection(object):
def initconn(self, kwargs):
hostname = 'localhost'
user_id = 'poll_cloud'
password = '******'
virtual_host = 'test',
port = 5672
connect_timeout = 10
heartbeat = 0
self.conn = BrokerConnection(
hostname=kwargs.get('hostname') or hostname,
userid=kwargs.get('userid') or user_id,
password=kwargs.get('password') or password,
port=kwargs.get('port') or port,
virtual_host=kwargs.get('virtual_host') or virtual_host,
connect_timeout=kwargs.get('connect_timeout') or connect_timeout,
heartbeat=kwargs.get('heartbeat') or heartbeat)
def consumerFunc(self, exchange='amq.direct', type='direct'):
self.chan = self.conn.channel()
self.exchange = Exchange(exchange=exchange, type=type)
def queue_drain():
filename = "meta"
fptr = open(filename, "r")
amqpurl = fptr.readline().strip()
exchange_name = fptr.readline().strip()
exchange = Exchange(exchange_name, type="direct")
D_queue = Queue(exchange_name, exchange, routing_key=exchange_name, exclusive=False)
connection = BrokerConnection(amqpurl)
channel = connection.channel()
queue = D_queue(channel)
queue.declare()
consumer = Consumer(channel, queue, callbacks=[work])
consumer.qos(prefetch_size=0, prefetch_count=1, apply_global=False)
consumer.consume(no_ack=False)
print "about to drain"
for i in range(0, 30):
try:
connection.drain_events(timeout=1)
except socket.timeout, ex:
pass
class Messaging(object):
amqp_opts = {
'amqp_queue': 'alerts',
'amqp_topic': 'notify',
'amqp_url': 'amqp://*****:*****@localhost:5672//', # RabbitMQ
# 'amqp_url': 'mongodb://localhost:27017/kombu', # MongoDB
# 'amqp_url': 'redis://localhost:6379/', # Redis
}
def __init__(self):
config.register_opts(Messaging.amqp_opts)
self.connection = None
self.channel = None
self.connect()
def connect(self):
if not CONF.amqp_url:
return
self.connection = BrokerConnection(CONF.amqp_url)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', CONF.amqp_url, e)
sys.exit(1)
self.channel = self.connection.channel()
LOG.info('Connected to broker %s', CONF.amqp_url)
def disconnect(self):
return self.connection.release()
def is_connected(self):
return self.connection.connected
def run(self):
print "exchange = %s, queue = %s, routing_key = %s, amqpurl = %s" % (self.testname, self.testname, self.testname, self.amqpurl)
exchange = Exchange(self.testname, type="direct")
D_queue = Queue(self.testname, exchange, routing_key=self.testname, exclusive=False)
connection = BrokerConnection(self.amqpurl)
#u = self.amqpurl.replace('amqp', 'http')
#parts = urlparse.urlparse(u)
#connection = Connection(host=parts.hostname, userid=parts.username, password=parts.password, port=parts.port, heartbeat=30)
channel = connection.channel()
queue = D_queue(channel)
queue.declare()
consumer = Consumer(channel, queue, callbacks=[self.work])
consumer.qos(prefetch_size=0, prefetch_count=1, apply_global=False)
self.done = False
consumer.consume(no_ack=False)
print "about to drain"
while not self.done:
connection.drain_events()
self.real_work()
def publish_cmd():
parser = get_parser("publish")
(options, args) = parser.parse_args(sys.argv[2:])
if len(args) < 1:
parser.error("You must provide a queue hostname to publish to!")
exchange = Exchange(options.exchange,
type="fanout",
delivery_mode="transient")
queue = Queue(options.queue, exchange)
connection = BrokerConnection(hostname=args[0],
userid=options.userid,
password=options.password,
virtual_host=options.vhost)
channel = connection.channel()
producer = Producer(channel, exchange)
pods = pod.load_pod_dir(
dirname=options.directory,
timeout=options.timeout,
# The first arg is the queue hostname, the rest
# will be pod names
filter_fn=pod.get_pod_filter(options.pod))
pods.update(pod.get_barker_metadata())
producer.publish(pods, serializer="json")
return 0
class event2amqp():
def __init__(self,host,port,user,password,virtual_host, exchange_name,identifier,maxqueuelength,queue_dump_frequency):
self.host = host
self.port = port
self.user = user
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.identifier = identifier
self.maxqueuelength = maxqueuelength
self.queue_dump_frequency = queue_dump_frequency
self.connection_string = None
self.connection = None
self.channel = None
self.producer = None
self.exchange = None
self.queue = deque([])
self.tickage = 0
self.load_queue()
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (self.user,self.password,self.host,self.port,self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def connect(self):
logger.info("[Canopsis] connection with : %s" % self.connection_string)
try:
self.connection.connect()
if not self.connected():
return False
else:
self.get_channel()
self.get_exchange()
self.create_producer()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def disconnect(self):
try:
if self.connected():
self.connection.release()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def connected(self):
try:
if self.connection.connected:
return True
else:
return False
except:
return False
def get_channel(self):
try:
self.channel = self.connection.channel()
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def get_exchange(self):
try:
self.exchange = Exchange(self.exchange_name , "topic", durable=True, auto_delete=False)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def create_producer(self):
try:
self.producer = Producer(
channel=self.channel,
exchange=self.exchange,
routing_key=self.virtual_host
)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def postmessage(self,message,retry=False):
# process enqueud events if possible
self.pop_events()
if message["source_type"] == "component":
key = "%s.%s.%s.%s.%s" % (
message["connector"],
message["connector_name"],
message["event_type"],
message["source_type"],
message["component"]
)
else:
key = "%s.%s.%s.%s.%s[%s]" % (
message["connector"],
message["connector_name"],
message["event_type"],
message["source_type"],
message["component"],
message["resource"]
)
# connection management
if not self.connected():
logger.error("[Canopsis] Create connection")
self.create_connection()
self.connect()
# publish message
if self.connected():
logger.info("[Canopsis] using routing key %s" % key)
logger.info("[Canopsis] sending %s" % str(message))
try:
self.producer.revive(self.channel)
self.producer.publish(body=message, compression=None, routing_key=key, exchange=self.exchange_name)
return True
except:
logger.error("[Canopsis] Not connected, going to queue messages until connection back")
self.queue.append({"key":key,"message":message})
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
# logger.error(str(traceback.format_exc()))
return False
else:
errmsg="[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (str(len(self.queue)),str(self.maxqueuelength))
logger.info(errmsg)
#enqueue_cano_event(key,message)
if len(self.queue) < int(self.maxqueuelength):
self.queue.append({"key":key,"message":message})
logger.info("[Canopsis] Queue length : %d" % len(self.queue))
return True
else:
logger.error("[Canopsis] Maximum retention for event queue %s reached" % str(self.maxqueuelength))
return False
def errback(self,exc,interval):
logger.warning("Couldn't publish message: %r. Retry in %ds" % (exc, interval))
def pop_events(self):
if self.connected():
while len(self.queue) > 0:
item = self.queue.pop()
try:
logger.info("[Canopsis] Pop item from queue [%s] : %s" % (str(len(self.queue)),str(item)))
self.producer.revive(self.channel)
self.producer.publish(body=item["message"], compression=None, routing_key=item["key"], exchange=self.exchange_name)
except:
self.queue.append(item)
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
else:
return False
def hook_tick(self, brok):
self.tickage += 1
# queue retention saving
if self.tickage >= int(self.queue_dump_frequency) and len(self.queue) > 0:
# flush queue to disk if queue age reach queue_dump_frequency
self.save_queue()
self.tickage = 0
return True
def save_queue(self):
retentionfile="%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] saving to %s" % retentionfile)
filehandler = open(retentionfile, 'w')
pickle.dump(self.queue, filehandler)
filehandler.close()
return True
def load_queue(self):
retentionfile="%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] loading from %s" % retentionfile)
filehandler = open(retentionfile, 'r')
try:
self.queue = pickle.load(filehandler)
except:
pass
return True
class Server(object):
"""
This Server class is used to provide an RPC server
:keyword server_id: Id of the server
:keyword amqp_host: The host of where the AMQP Broker is running.
:keyword amqp_user: The username for the AMQP Broker.
:keyword amqp_password: The password for the AMQP Broker.
:keyword amqp_vhost: The virtual host of the AMQP Broker.
:keyword amqp_port: The port of the AMQP Broker.
:keyword ssl: Use SSL connection for the AMQP Broker.
:keyword threaded: Use of multithreading. If set to true RPC call-execution
will processed parallel (one thread per call) which dramatically improves
performance.
"""
def __init__(self,
server_id,
amqp_host='localhost',
amqp_user ='******',
amqp_password='******',
amqp_vhost='/',
amqp_port=5672,
ssl=False,
threaded=False):
self.logger = logging.getLogger('callme.server')
self.logger.debug('Server ID: %s' % server_id)
self.server_id = server_id
self.threaded = threaded
self.do_run = True
self.is_stopped = True
self.func_dict={}
self.result_queue = queue.Queue()
target_exchange = Exchange("server_"+server_id+"_ex", "direct", durable=False,
auto_delete=True)
self.target_queue = Queue("server_"+server_id+"_queue", exchange=target_exchange,
auto_delete=True, durable=False)
self.connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
try:
self.connection.connect()
except IOError:
self.logger.critical("Connection Error: Probably AMQP User has not enough permissions")
raise ConnectionError("Connection Error: Probably AMQP User has not enough permissions")
self.channel = self.connection.channel()
self.publish_connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
self.publish_channel = self.publish_connection.channel()
# consume
self.consumer = Consumer(self.channel, self.target_queue)
if self.threaded == True:
self.consumer.register_callback(self._on_request_threaded)
else:
self.consumer.register_callback(self._on_request)
self.consumer.consume()
self.logger.debug('Init done')
def _on_request(self, body, message):
"""
This method is automatically called when a request is incoming. It
processes the incomming rpc calls in a serial manner (no multithreading)
:param body: the body of the amqp message already unpickled by kombu
:param message: the plain amqp kombu.message with aditional information
"""
self.logger.debug('Got Request')
rpc_req = body
if not isinstance(rpc_req, RpcRequest):
self.logger.debug('Not an RpcRequest Instance')
return
self.logger.debug('Call func on Server %s' %self.server_id)
try:
self.logger.debug('corr_id: %s' % message.properties['correlation_id'])
self.logger.debug('Call func with args %s' % repr(rpc_req.func_args))
result = self.func_dict[rpc_req.func_name](*rpc_req.func_args)
self.logger.debug('Result: %s' % repr(result))
self.logger.debug('Build respnse')
rpc_resp = RpcResponse(result)
except Exception as e:
self.logger.debug('exception happened')
rpc_resp = RpcResponse(e, exception_raised=True)
message.ack()
self.logger.debug('Publish respnse')
# producer
src_exchange = Exchange(message.properties['reply_to'], "direct", durable=False,
auto_delete=True)
self.producer = Producer(self.publish_channel, src_exchange, auto_declare=False)
self.producer.publish(rpc_resp, serializer="pickle",
correlation_id=message.properties['correlation_id'])
self.logger.debug('acknowledge')
def _on_request_threaded(self, body, message):
"""
This method is automatically called when a request is incoming and
`threaded` set to `True`. It processes the incomming rpc calls in
a parallel manner (one thread for each request). A seperate Publisher
thread is used to send back the results.
:param body: the body of the amqp message already unpickled by kombu
:param message: the plain amqp kombu.message with aditional information
"""
self.logger.debug('Got Request')
rpc_req = body
if not isinstance(rpc_req, RpcRequest):
self.logger.debug('Not an RpcRequest Instance')
return
message.ack()
self.logger.debug('acknowledge')
def exec_func(body, message, result_queue):
self.logger.debug('Call func on Server %s' %self.server_id)
try:
self.logger.debug('corr_id: %s' % message.properties['correlation_id'])
self.logger.debug('Call func with args %s' % repr(rpc_req.func_args))
result = self.func_dict[rpc_req.func_name](*rpc_req.func_args)
self.logger.debug('Result: %s' % repr(result))
self.logger.debug('Build respnse')
rpc_resp = RpcResponse(result)
except Exception as e:
self.logger.debug('exception happened')
rpc_resp = RpcResponse(e, exception_raised=True)
result_queue.put(ResultSet(rpc_resp,
message.properties['correlation_id'],
message.properties['reply_to']))
p = Thread(target=exec_func,
name=message.properties['correlation_id'],
args=(body, message, self.result_queue))
p.start()
def register_function(self, func, name):
"""
Registers a function as rpc function so that is accessible from the
proxy.
:param func: The function we want to provide as rpc method
:param name: The name with which the function is visible to the clients
"""
self.func_dict[name] = func
def start(self):
"""
Starts the server. If `threaded` is `True` also starts the Publisher
thread.
"""
self.is_stopped = False
if self.threaded == True:
self.pub_thread = Publisher(self.result_queue, self.publish_channel)
self.pub_thread.start()
while self.do_run:
try:
self.logger.debug("drain_events: %s" % repr(self.do_run))
self.connection.drain_events(timeout=1)
except socket.timeout:
self.logger.debug("do_run: %s" % repr(self.do_run))
except:
self.logger.debug("interrupt exception" )
if self.threaded == True:
self.pub_thread.stop()
self.consumer.cancel()
self.connection.close()
self.publish_connection.close()
self.is_stopped = True
return
if self.threaded == True:
self.pub_thread.stop()
self.logger.debug("Normal Exit" )
self.consumer.cancel()
self.connection.close()
self.publish_connection.close()
self.logger.debug("All closed" )
self.is_stopped = True
def stop(self):
"""
Stops the server.
"""
self.logger.debug('Stop server')
self.do_run = False
while not self.is_stopped:
self.logger.debug('wait for stop')
sleep(0.1)
class Connection:
connection = None
def __init__(self, url):
self.url = url
self.__connection = None
self.__running = True
self.channel = None
self.sleep_time = 10
self.reconnect(url)
@staticmethod
def get_instance():
return Connection.connection
def __connect(self):
self.__connection = BrokerConnection(self.url)
self.channel = self.get_channel()
self.rpc_factory = rpc.RpcFactory(self.channel)
self.publisher_factory = publisher.PublisherFactory(self.channel)
self.consumer_factory = consumer.ConsumerFactory(self.channel)
self.__running = True
Connection.connection = self
def get_broker_connection(self):
if self.__connection is None:
self.reconnect(self.url)
return self.__connection
def get_channel(self):
if self.channel is None:
self.channel = self.get_new_channel()
return self.channel
def get_new_channel(self):
if self.__connection is None:
self.reconnect(self.url)
return self.__connection.channel()
def get_rpc_factory(self):
return self.rpc_factory
def reconnect(self, url=None):
cc.acquire()
if self.__connection is not None:
self.release()
if url is not None:
self.url = url
logger.debug("reconnect connection")
attempt = 0
while True:
try:
self.__connect()
cc.release()
return
except Exception as e:
logging.exception(e)
logging.debug("retry again in %s s" % self.sleep_time)
time.sleep(self.sleep_time)
cc.release()
def drain_events(self):
self.__connection.drain_events()
def release(self):
Connection.connection = None
self.__running = False
self.__connection.release()
self.__connection.close()
self.channel = None
self.__connection = None
class Proxy(object):
"""
This Proxy class is used to handle the communication with the rpc server
:keyword server_id: Default id of the Server (can be declared later see :func:`use_server`)
:keyword amqp_host: The host of where the AMQP Broker is running.
:keyword amqp_user: The username for the AMQP Broker.
:keyword amqp_password: The password for the AMQP Broker.
:keyword amqp_vhost: The virtual host of the AMQP Broker.
:keyword amqp_port: The port of the AMQP Broker.
:keyword ssl: Use SSL connection for the AMQP Broker.
:keyword timeout: Default timeout for calls in seconds
"""
timeout = 0
response = None
def __init__(self,
server_id = None,
amqp_host='localhost',
amqp_user ='******',
amqp_password='******',
amqp_vhost='/',
amqp_port=5672,
ssl=False,
timeout=0):
self.logger = logging.getLogger('callme.proxy')
self.timeout = 0
self.is_received = False
self.connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
self.channel = self.connection.channel()
self.timeout = timeout
my_uuid = gen_unique_id()
self.reply_id = "client_"+amqp_user+"_ex_" + my_uuid
self.logger.debug("Queue ID: %s" %self.reply_id)
src_exchange = Exchange(self.reply_id, "direct", durable=False
,auto_delete=True)
src_queue = Queue("client_"+amqp_user+"_queue_"+my_uuid, exchange=src_exchange,
auto_delete=True,
durable=False)
# must declare in advance so reply message isn't
# published before.
src_queue(self.channel).declare()
consumer = Consumer(channel=self.channel, queues=src_queue, callbacks=[self._on_response])
consumer.consume()
def _on_response(self, body, message):
"""
This method is automatically called when a response is incoming and
decides if it is the message we are waiting for - the message with the
result
:param body: the body of the amqp message already unpickled by kombu
:param message: the plain amqp kombu.message with aditional information
"""
if self.corr_id == message.properties['correlation_id'] and \
isinstance(body, RpcResponse):
self.response = body
self.is_received = True
message.ack()
def use_server(self, server_id=None, timeout=None):
"""Use the specified server and set an optional timeout for the method
call
Typical use:
>>> my_proxy.use_server('fooserver').a_remote_func()
:keyword server_id: The server id where the call will be made.
:keyword timeout: set or overrides the call timeout in seconds
:rtype: Return `self` to cascade further calls
"""
if server_id != None:
self.server_id = server_id
if timeout !=None:
self.timeout = timeout
return self
def __request(self, methodname, params):
"""
The remote-method-call execution function
:param methodname: name of the method that should be executed
:param params: parameter for the remote-method
:type methodname: string
:type param: list of parameters
:rtype: result of the method
"""
self.logger.debug('Request: ' + repr(methodname) + '; Params: '+ repr(params))
target_exchange = Exchange("server_"+self.server_id+"_ex", "direct", durable=False,
auto_delete=True)
self.producer = Producer(channel=self.channel, exchange=target_exchange,
auto_declare=False)
rpc_req = RpcRequest(methodname, params)
self.corr_id = str(uuid.uuid4())
self.logger.debug('RpcRequest build')
self.logger.debug('corr_id: %s' % self.corr_id)
self.producer.publish(rpc_req, serializer="pickle",
reply_to=self.reply_id,
correlation_id=self.corr_id)
self.logger.debug('Producer published')
self._wait_for_result()
self.logger.debug('Result: %s' % repr(self.response.result))
res = self.response.result
self.response.result = None
self.is_received = False
if self.response.exception_raised:
raise res
else:
return res
def _wait_for_result(self):
"""
Waits for the result from the server, checks every second if a timeout
occurred. If a timeout occurs a `socket.timout` exception will be raised.
"""
seconds_elapsed = 0
while not self.is_received:
try:
self.logger.debug('drain events... timeout=%d, counter=%d'
% (self.timeout, seconds_elapsed))
self.connection.drain_events(timeout=1)
except socket.timeout:
if self.timeout > 0:
seconds_elapsed = seconds_elapsed + 1
if seconds_elapsed > self.timeout:
raise socket.timeout()
def __getattr__(self, name):
"""
This method is invoked, if a method is being called, which doesn't exist on Proxy.
It is used for RPC, to get the function which should be called on the Server.
"""
# magic method dispatcher
self.logger.debug('Recursion: ' + name)
return _Method(self.__request, name)
class event2amqp():
def __init__(self, host, port, user, password, virtual_host, exchange_name,
identifier, maxqueuelength, queue_dump_frequency):
self.host = host
self.port = port
self.user = user
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.identifier = identifier
self.maxqueuelength = maxqueuelength
self.queue_dump_frequency = queue_dump_frequency
self.connection_string = None
self.connection = None
self.channel = None
self.producer = None
self.exchange = None
self.queue = deque([])
self.tickage = 0
self.load_queue()
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (
self.user, self.password, self.host, self.port, self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def connect(self):
logger.info("[Canopsis] connection with: %s" % self.connection_string)
try:
self.connection.connect()
if not self.connected():
return False
else:
self.get_channel()
self.get_exchange()
self.create_producer()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def disconnect(self):
try:
if self.connected():
self.connection.release()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def connected(self):
try:
if self.connection.connected:
return True
else:
return False
except:
return False
def get_channel(self):
try:
self.channel = self.connection.channel()
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def get_exchange(self):
try:
self.exchange = Exchange(self.exchange_name,
"topic",
durable=True,
auto_delete=False)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def create_producer(self):
try:
self.producer = Producer(channel=self.channel,
exchange=self.exchange,
routing_key=self.virtual_host)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def postmessage(self, message, retry=False):
# process enqueud events if possible
self.pop_events()
if message["source_type"] == "component":
key = "%s.%s.%s.%s.%s" % (
message["connector"], message["connector_name"],
message["event_type"], message["source_type"],
message["component"])
else:
key = "%s.%s.%s.%s.%s[%s]" % (
message["connector"], message["connector_name"],
message["event_type"], message["source_type"],
message["component"], message["resource"])
# connection management
if not self.connected():
logger.error("[Canopsis] Create connection")
self.create_connection()
self.connect()
# publish message
if self.connected():
logger.debug("[Canopsis] using routing key %s" % key)
logger.debug("[Canopsis] sending %s" % str(message))
try:
self.producer.revive(self.channel)
self.producer.publish(body=message,
compression=None,
routing_key=key,
exchange=self.exchange_name)
return True
except:
logger.error(
"[Canopsis] Not connected, going to queue messages until connection back"
)
self.queue.append({"key": key, "message": message})
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
# logger.error(str(traceback.format_exc()))
return False
else:
errmsg = "[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (
str(len(self.queue)), str(self.maxqueuelength))
logger.error(errmsg)
#enqueue_cano_event(key,message)
if len(self.queue) < int(self.maxqueuelength):
self.queue.append({"key": key, "message": message})
logger.debug("[Canopsis] Queue length: %d" % len(self.queue))
return True
else:
logger.error(
"[Canopsis] Maximum retention for event queue %s reached" %
str(self.maxqueuelength))
return False
def errback(self, exc, interval):
logger.warning("Couldn't publish message: %r. Retry in %ds" %
(exc, interval))
def pop_events(self):
if self.connected():
while len(self.queue) > 0:
item = self.queue.pop()
try:
logger.debug("[Canopsis] Pop item from queue [%s]: %s" %
(str(len(self.queue)), str(item)))
self.producer.revive(self.channel)
self.producer.publish(body=item["message"],
compression=None,
routing_key=item["key"],
exchange=self.exchange_name)
except:
self.queue.append(item)
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
else:
return False
def hook_tick(self, brok):
self.tickage += 1
# queue retention saving
if self.tickage >= int(self.queue_dump_frequency) and len(
self.queue) > 0:
# flush queue to disk if queue age reach queue_dump_frequency
self.save_queue()
self.tickage = 0
return True
def save_queue(self):
retentionfile = "%s/canopsis.dat" % os.getcwd() #:fixme: use path.join
logger.info("[Canopsis] saving to %s" % retentionfile)
filehandler = open(retentionfile, 'w')
pickle.dump(self.queue, filehandler)
filehandler.close()
return True
def load_queue(self):
retentionfile = "%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] loading from %s" % retentionfile)
filehandler = open(retentionfile, 'r')
try:
self.queue = pickle.load(filehandler)
except:
pass
return True
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print("Received message: %r" % (body, ))
print(" properties:\n%s" % (pretty(message.properties), ))
print(" delivery_info:\n%s" % (pretty(message.delivery_info), ))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
connection = BrokerConnection(hostname="localhost",
userid="guest",
password="******",
virtual_host="/")
channel = connection.channel()
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
consumer = Consumer(channel, queue, callbacks=[handle_message])
consumer.consume()
#: This waits for a single event. Note that this event may not
#: be a message, or a message that is to be delivered to the consumers
#: channel, but any event received on the connection.
connection.drain_events()
class SimpleBase(TestCase):
abstract = True
def Queue(self, name, *args, **kwargs):
q = name
if not isinstance(q, Queue):
q = self.__class__.__name__
if name:
q = "%s.%s" % (q, name)
return self._Queue(q, *args, **kwargs)
def _Queue(self, *args, **kwargs):
raise NotImplementedError()
def setUp(self):
if not self.abstract:
self.connection = BrokerConnection(transport="memory")
self.q = self.Queue(None, no_ack=True)
def tearDown(self):
if not self.abstract:
self.q.close()
self.connection.close()
def test_produce__consume(self):
if self.abstract:
return
q = self.Queue("test_produce__consume", no_ack=True)
q.put({"hello": "Simple"})
self.assertEqual(q.get(timeout=1).payload, {"hello": "Simple"})
with self.assertRaises(Empty):
q.get(timeout=0.1)
def test_produce__basic_get(self):
if self.abstract:
return
q = self.Queue("test_produce__basic_get", no_ack=True)
q.put({"hello": "SimpleSync"})
self.assertEqual(q.get_nowait().payload, {"hello": "SimpleSync"})
with self.assertRaises(Empty):
q.get_nowait()
q.put({"hello": "SimpleSync"})
self.assertEqual(q.get(block=False).payload, {"hello": "SimpleSync"})
with self.assertRaises(Empty):
q.get(block=False)
def test_clear(self):
if self.abstract:
return
q = self.Queue("test_clear", no_ack=True)
for i in range(10):
q.put({"hello": "SimplePurge%d" % (i, )})
self.assertEqual(q.clear(), 10)
def test_enter_exit(self):
if self.abstract:
return
q = self.Queue("test_enter_exit")
q.close = Mock()
self.assertIs(q.__enter__(), q)
q.__exit__()
q.close.assert_called_with()
def test_qsize(self):
if self.abstract:
return
q = self.Queue("test_clear", no_ack=True)
for i in range(10):
q.put({"hello": "SimplePurge%d" % (i, )})
self.assertEqual(q.qsize(), 10)
self.assertEqual(len(q), 10)
def test_autoclose(self):
if self.abstract:
return
channel = self.connection.channel()
q = self.Queue("test_autoclose", no_ack=True, channel=channel)
q.close()
def test_custom_Queue(self):
if self.abstract:
return
n = self.__class__.__name__
exchange = Exchange("%s-test.custom.Queue" % (n, ))
queue = Queue("%s-test.custom.Queue" % (n, ),
exchange,
"my.routing.key")
q = self.Queue(queue)
self.assertEqual(q.consumer.queues[0], queue)
q.close()
def test_bool(self):
if self.abstract:
return
q = self.Queue("test_nonzero")
self.assertTrue(q)
import boto
from kombu import BrokerConnection, Exchange, Queue, Consumer
connection = BrokerConnection()
connection.connect()
channel = connection.channel()
exchange = Exchange(name="android", type="fanout", channel=channel, durable=True)
exchange.declare()
channel = connection.channel()
queue = Queue(name="filr", exchange=exchange, durable=True, auto_delete=False, channel=channel, routing_key="filr")
queue.declare()
def fetch(b, m):
print b, m
consumer = Consumer(channel=connection.channel(), queues=queue, auto_declare=False, callbacks=[fetch])
consumer.consume(no_ack=False)
while True:
connection.drain_events()
pass
# execfile('.private-settings')
# sdb = boto.connect_sdb(key_id, sec_key)
# domain = sdb.create_domain('android')
# item = domain.new_item('kral_step1')
import boto
from kombu import BrokerConnection, Exchange, Queue, Consumer
connection = BrokerConnection()
connection.connect()
channel = connection.channel()
exchange = Exchange(
name="android",
type="fanout",
channel=channel,
durable=True,
)
exchange.declare()
channel = connection.channel()
queue = Queue(
name='filr',
exchange=exchange,
durable=True,
auto_delete=False,
channel=channel,
routing_key='filr',
)
queue.declare();
def fetch(b,m):
print b,m
consumer = Consumer(
channel=connection.channel(),
class test_pika(unittest.TestCase):
def purge(self, names):
chan = self.connection.channel()
map(chan.queue_purge, names)
def setUp(self):
self.connection = BrokerConnection(transport="pika")
try:
self.connection.connect()
except socket.error:
self.connected = False
else:
self.connected = True
self.exchange = Exchange("tamqplib", "direct")
self.queue = Queue("tamqplib", self.exchange, "tamqplib")
def test_produce__consume(self):
if not self.connected:
raise SkipTest("Broker not running.")
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
producer.publish({"foo": "bar"}, routing_key="tamqplib")
chan1.close()
chan2 = self.connection.channel()
consumer = Consumer(chan2, self.queue)
message = consumeN(self.connection, consumer)
self.assertDictEqual(message[0], {"foo": "bar"})
chan2.close()
self.purge(["tamqplib"])
def test_produce__consume_multiple(self):
if not self.connected:
raise SkipTest("Broker not running.")
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
b1 = Queue("pyamqplib.b1", self.exchange, "b1")
b2 = Queue("pyamqplib.b2", self.exchange, "b2")
b3 = Queue("pyamqplib.b3", self.exchange, "b3")
producer.publish("b1", routing_key="b1")
producer.publish("b2", routing_key="b2")
producer.publish("b3", routing_key="b3")
chan1.close()
chan2 = self.connection.channel()
consumer = Consumer(chan2, [b1, b2, b3])
messages = consumeN(self.connection, consumer, 3)
self.assertItemsEqual(messages, ["b1", "b2", "b3"])
chan2.close()
self.purge(["pyamqplib.b1", "pyamqplib.b2", "pyamqplib.b3"])
def test_timeout(self):
if not self.connected:
raise SkipTest("Broker not running.")
chan = self.connection.channel()
self.purge([self.queue.name])
consumer = Consumer(chan, self.queue)
self.assertRaises(socket.timeout, self.connection.drain_events,
timeout=0.3)
consumer.cancel()
def test_basic_get(self):
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
chan2 = self.connection.channel()
queue = Queue("amqplib_basic_get", self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
for i in range(50):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def tearDown(self):
if self.connected:
self.connection.close()
class RabbitMQHandler(object):
def __init__(self,
connection_string,
exchange_name,
exchange_type="topic"):
self._connection = BrokerConnection(connection_string)
self._connections = {self._connection
} # set of connection for the heartbeat
self._exchange = Exchange(exchange_name,
durable=True,
delivery_mode=2,
type=exchange_type,
auto_delete=False,
no_declare=False)
monitor_heartbeats(self._connections)
@retry(wait_fixed=200, stop_max_attempt_number=3)
def publish(self, item, contributor_id):
with self._connection.channel() as channel:
with Producer(channel) as producer:
producer.publish(
item,
exchange=self._exchange,
routing_key=contributor_id,
declare=[self._exchange],
content_type="plain/text",
)
def info(self):
info = self._connection.info()
info.pop("password", None)
return info
def check_connection(self, force=False):
"""
Trying to connect is the best way to check that the connection works
if force is set to True, we will force the connection again
"""
try:
# we need to refresh the connection to be notified as soon as rabbitmq stopped working
if force:
self._connection._establish_connection()
self._connection.ensure_connection(interval_start=0, max_retries=1)
return True
except Exception:
return False
def connect(self):
self._connection.connect()
def close(self):
for c in self._connections:
c.release()
def listen_load_realtime(self, queue_name, max_retries=10):
log = logging.getLogger(__name__)
route = "task.load_realtime.*"
log.info("listening route {} on exchange {}...".format(
route, self._exchange))
rt_queue = Queue(queue_name,
routing_key=route,
exchange=self._exchange,
durable=False)
RTReloader(connection=self._connection,
rpc_queue=rt_queue,
exchange=self._exchange,
max_retries=max_retries).run()
