def declare_exchange(self, name, type='direct', queues=None, **options):
"""Create or update exchange
:param name: name of exchange
:type name: str
:param type: type of exchange - direct, fanout, topic, match
:type type: str
:param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...]
:type queues: list, None or tuple
:param options: additional options for Exchange creation
"""
if queues is None:
queues = [] # pragma: no cover
with self.connections[self.connection].acquire() as conn:
exchange = Exchange(name, type=type, channel=conn, **options)
exchange.declare()
self.exchanges[name] = exchange
for q_name, routing_key in queues:
queue = Queue(name=q_name, channel=conn)
queue.declare()
queue.bind_to(exchange=name, routing_key=routing_key)
self.logger.debug(
'Queue "%s" with routing_key "%s" was bond to exchange "%s"',
q_name, routing_key if routing_key else q_name, name)
def send(self,
body,
routing_key,
exchange_name=None,
exchange_type=None,
headers=None,
log_flag=None):
exchange = Exchange(name=exchange_name or self.send_exchange_name,
type=exchange_type or self.send_exchange_type,
auto_delete=False,
durable=True)
while True:
try:
self.wait_send_queue.join()
self.wait_send_queue.put(body, block=False)
break
except Exception as E:
logger.info('wait lock')
pass
try:
channel = self.send_connection.default_channel
exchange.declare(channel=channel)
self.consumer.producer.publish(
body=self.wait_send_queue.get(),
exchange=exchange,
routing_key=routing_key,
retry=True,
headers=headers,
)
logger.info(log_flag, 'send data: %s', body)
except Exception as E:
pass
finally:
self.wait_send_queue.task_done()
def run_pulse_listener(username, password, timeout, no_send):
"""Run a Pulse message queue listener."""
connection = Connection(
hostname='pulse.mozilla.org',
port=5671,
ssl=True,
userid=username,
password=password,
)
# Connect and pass in our own low value for retries so the connection
# fails fast if there is a problem.
connection.ensure_connection(
max_retries=1
) # Retries must be >=1 or it will retry forever.
with closing(connection):
hgpush_exchange = Exchange(config.PULSE_EXCHANGE, 'topic', channel=connection)
# Pulse queue names need to be prefixed with the username
queue_name = f'queue/{username}/{config.PULSE_QUEUE_NAME}'
queue = Queue(
queue_name,
exchange=hgpush_exchange,
routing_key=config.PULSE_QUEUE_ROUTING_KEY,
durable=True,
exclusive=False,
auto_delete=False,
channel=connection,
)
# Passing passive=True will assert that the exchange exists but won't
# try to declare it. The Pulse server forbids declaring exchanges.
hgpush_exchange.declare(passive=True)
# Queue.declare() also declares the exchange, which isn't allowed by
# the Pulse server. Use the low-level Queue API to only declare the
# queue itself.
queue.queue_declare()
queue.queue_bind()
callback = partial(process_push_message, no_send=no_send)
# Pass auto_declare=False so that Consumer does not try to declare the
# exchange. Declaring exchanges is not allowed by the Pulse server.
with connection.Consumer(
queue, callbacks=[callback], auto_declare=False
) as consumer:
if no_send:
log.info('transmission of ping data has been disabled')
log.info('message acks has been disabled')
log.info('reading messages')
try:
connection.drain_events(timeout=timeout)
except socket.timeout:
log.info('message queue is empty, nothing to do')
log.info('done')
def runner(k):
msg_ = {
"timestamp": 1,
"orig_path": [],
"communities": [],
"service": "a",
"type": "A",
"path": [8, 3, 2, 1, ord(k)],
"prefix": "10.0.0.0/8",
"peer_asn": 8,
}
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange("bgp-update",
channel=connection,
type="direct",
durable=False)
exchange.declare()
with Producer(connection) as producer:
for i in range(1000):
msg_["timestamp"] = i
msg_["key"] = "{}-{}".format(k, i)
producer.publish(msg_,
exchange=exchange,
routing_key="update",
serializer="json")
class Worker:
def __init__(self, connection):
self.connection = connection
# Time in secs to gather entries to perform a bulk operation
self.time_to_wait = float(os.getenv("BULK_TIMER", 1))
self.correlation_id = None
self.db_clock_exchange = Exchange(
"db-clock",
type="direct",
channel=connection,
durable=False,
delivery_mode=1,
)
self.db_clock_exchange.declare()
signal_loading("clock", True)
log.info("started")
signal_loading("clock", False)
self._db_clock_send()
def _db_clock_send(self):
with Producer(self.connection) as producer:
while True:
time.sleep(self.time_to_wait)
producer.publish(
{"op": "bulk_operation"},
exchange=self.db_clock_exchange,
routing_key="pulse",
retry=True,
priority=3,
serializer="ujson",
)
def runner():
with open(sys.argv[1]) as json_file:
data = json.load(json_file)
for i in range(7):
msg_ = {
"timestamp": data['bgp_update'][i]['timestamp'],
"orig_path": data['bgp_update'][i]['orig_path'],
"communities": data['bgp_update'][i]['communities'],
"service": data['bgp_update'][i]['service'],
"type": data['bgp_update'][i]['type'],
"path": data['bgp_update'][i]['path'],
"prefix": data['bgp_update'][i]['prefix'],
"peer_asn": data['bgp_update'][i]['peer_asn'],
}
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange("bgp-update",
channel=connection,
type="direct",
durable=False)
exchange.declare()
with Producer(connection) as producer:
msg_["key"] = "{}".format(i + 50)
print(msg_)
producer.publish(msg_,
exchange=exchange,
routing_key="update",
serializer="json")
time.sleep(15)
class Worker():
def __init__(self, connection):
self.connection = connection
self.time_to_wait = 1 # Time in secs to gather entries to perform a bulk operation
self.db_clock_exchange = Exchange('db-clock',
type='direct',
channel=connection,
durable=False,
delivery_mode=1)
self.db_clock_exchange.declare()
log.info('started')
self._db_clock_send()
def _get_module_status(self, module):
server = ServerProxy('http://{}:{}/RPC2'.format(
SUPERVISOR_HOST, SUPERVISOR_PORT))
try:
return any([
x['name'] for x in server.supervisor.getAllProcessInfo()
if x['group'] == module and x['state'] == 20
])
except BaseException:
return False
def _db_clock_send(self):
with Producer(self.connection) as producer:
while True:
time.sleep(self.time_to_wait)
producer.publish({'op': 'bulk_operation'},
exchange=self.db_clock_exchange,
routing_key='pulse',
retry=True,
priority=3)
def _create_or_verify_model_exchange(self, connection):
"""
Create or verify existence of model exchange on AMQP server.
"""
model_exchange = Exchange(
self.model_exchange_name, 'topic', connection, durable=True)
model_exchange.declare()
return model_exchange
class ExaBGP():
def __init__(self, prefixes, host):
self.host = host
self.prefixes = prefixes
self.sio = None
signal.signal(signal.SIGTERM, self.exit)
signal.signal(signal.SIGINT, self.exit)
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
def start(self):
with Connection(RABBITMQ_HOST) as connection:
self.connection = connection
self.exchange = Exchange('bgp-update',
channel=connection,
type='direct',
durable=False)
self.exchange.declare()
try:
self.sio = SocketIO('http://' + self.host,
namespace=BaseNamespace)
def exabgp_msg(bgp_message):
msg = {
'type': bgp_message['type'],
'communities': bgp_message.get('communities', []),
'timestamp': float(bgp_message['timestamp']),
'path': bgp_message.get('path', []),
'service': 'exabgp|{}'.format(self.host),
'prefix': bgp_message['prefix'],
'peer_asn': int(bgp_message['peer_asn'])
}
if mformat_validator(msg):
with Producer(connection) as producer:
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(msg,
exchange=self.exchange,
routing_key='update',
serializer='json')
else:
log.warning('Invalid format message: {}'.format(msg))
self.sio.on('exa_message', exabgp_msg)
self.sio.emit('exa_subscribe', {'prefixes': self.prefixes})
self.sio.wait()
except KeyboardInterrupt:
self.exit()
except Exception:
log.exception('exception')
def exit(self):
print('Exiting ExaBGP')
if self.sio is not None:
self.sio.disconnect()
self.sio.wait()
def create_exchange(name, channel=None, _type="direct", declare=False):
exchange = Exchange(name,
channel=channel,
type=_type,
durable=False,
delivery_mode=1)
if declare:
exchange.declare()
return exchange
def declare_topic(self, topic_name):
with self.connection as _conn:
_conn.connect()
channel = _conn.channel()
topic = Exchange(
name=f"{self.config.get_event_name_prefix()}{topic_name}",
type="topic",
channel=channel)
topic.declare()
def downstream_exchange_declare(self, name, type_, upstream=None):
if not upstream:
upstream = self._default_exchange
with Connection(self._url) as connection:
channel = connection.default_channel
exchange = Exchange(name, type_).bind(channel)
exchange.declare()
upstream.bind(channel).declare()
exchange.bind_to(upstream, routing_key='#')
def test_assert_is_bound(self):
exchange = Exchange('foo', 'direct')
with pytest.raises(NotBoundError):
exchange.declare()
conn = get_conn()
chan = conn.channel()
exchange.bind(chan).declare()
assert 'exchange_declare' in chan
def test_assert_is_bound(self):
exchange = Exchange('foo', 'direct')
with self.assertRaises(NotBoundError):
exchange.declare()
conn = get_conn()
chan = conn.channel()
exchange.bind(chan).declare()
self.assertIn('exchange_declare', chan)
def test_assert_is_bound(self):
exchange = Exchange('foo', 'direct')
with self.assertRaises(NotBoundError):
exchange.declare()
conn = get_conn()
chan = conn.channel()
exchange.bind(chan).declare()
self.assertIn('exchange_declare', chan)
def test_assert_is_bound(self):
exchange = Exchange('foo', 'direct')
with pytest.raises(NotBoundError):
exchange.declare()
conn = get_conn()
chan = conn.channel()
exchange.bind(chan).declare()
assert 'exchange_declare' in chan
def create_exchange(broker_connection, exchange_name):
exchange = Exchange(
exchange_name,
durable=True,
delivery_mode=2,
type="fanout",
auto_delete=False,
no_declare=False,
)
exchange.declare(channel=broker_connection.channel())
def init_ack():
'''初始化返回exchange和queue'''
connection = Connection(CONF.rabbit.connection)
channel = connection.channel()
exchange = Exchange(CONF.region_id + '_ack', 'direct', channel)
exchange.declare()
queue = Queue(CONF.region_id + '_ack',
exchange=exchange,
routing_key=CONF.region_id + '_ack',
channel=channel)
queue.declare()
def connect(self):
self.close_channel()
try:
self.channel = self.connection.channel()
exchange = Exchange(name='policymq.direct',
type='direct',
durable=False)
exchange.declare(channel=self.channel)
self.producer = Producer(channel=self.channel)
except Exception as e:
logger.exception(e)
pass
def __init__(self, connection, logger):
self.connection = connection
self.logger = logger
from gooutsafe import app
exchange = Exchange(
app.config.get('RABMQ_SEND_EXCHANGE_NAME'),
type='topic',
channel=connection.channel()
)
exchange.declare(nowait=False)
self.queues = [Queue('RestaurantDeletionQueue', exchange, routing_key='RESTAURANT_DELETION')]
def receive(exchange_name, routing_key):
def bind_and_wait(connection, queue):
queue.declare(channel=connection.default_channel)
bind_queue = queue.bind(connection.default_channel)
recv_cnt = 0
start = time.time()
while recv_cnt < LIMIT_UPDATES:
if bind_queue.get():
recv_cnt += 1
if recv_cnt % 1000 == 0:
with open("{}-{}".format(exchange_name, routing_key),
"w") as f:
print(
"[!] Throughput for {} on {}:{} = {} msg/s".format(
recv_cnt,
exchange_name,
routing_key,
recv_cnt / (time.time() - start),
))
f.write(str(int(recv_cnt / (time.time() - start))))
stop = time.time()
print("[!] Throughput for {} on {}:{} = {} msg/s".format(
recv_cnt, exchange_name, routing_key, recv_cnt / (stop - start)))
with open("{}-{}".format(exchange_name, routing_key), "w") as f:
f.write(str(int(recv_cnt / (stop - start))))
print("[+] Receiving {} on {}:{}".format(LIMIT_UPDATES, exchange_name,
routing_key))
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange(
exchange_name,
channel=connection,
type="direct",
durable=False,
delivery_mode=1,
)
exchange.declare()
queue = Queue(
"{}".format(uuid()),
exchange=exchange,
routing_key=routing_key,
durable=False,
auto_delete=True,
max_priority=1,
consumer_arguments={"x-priority": 1},
channel=connection.default_channel,
)
bind_and_wait(connection, queue)
print("[+] Exit recv")
def connect_to_amqp(sysconfig):
"""
Connect to an AMQP Server, and return the connection and Exchange.
:param sysconfig: The slickqaweb.model.systemConfiguration.amqpSystemConfiguration.AMQPSystemConfiguration instance
to use as the source of information of how to connect.
:return: (connection, exchange) on success, exception on error
"""
assert isinstance(sysconfig, AMQPSystemConfiguration)
configuration = dict()
configuration['AMQP'] = dict()
if hasattr(sysconfig, 'hostname') and sysconfig.hostname is not None:
configuration['AMQP']['hostname'] = sysconfig.hostname
else:
raise AMQPConnectionError(message="No hostname defined for AMQP connection.")
if hasattr(sysconfig, 'port') and sysconfig.port is not None:
configuration['AMQP']['port'] = sysconfig.port
if hasattr(sysconfig, 'username') and sysconfig.username is not None:
configuration['AMQP']['username'] = sysconfig.username
if hasattr(sysconfig, 'password') and sysconfig.password is not None:
configuration['AMQP']['password'] = sysconfig.password
if hasattr(sysconfig, 'virtualHost') and sysconfig.virtualHost is not None:
configuration['AMQP']['virtual host'] = sysconfig.virtualHost
if hasattr(sysconfig, 'exchangeName') and sysconfig.exchangeName is not None:
configuration['AMQP']['exchange'] = sysconfig.exchangeName
else:
raise AMQPConnectionError(message="No exchange defined for AMQP connection.")
logger = logging.getLogger("slickqaweb.amqpcon.connect_to_amqp")
url = str.format("amqp://{hostname}:{port}", **dict(list(configuration['AMQP'].items())))
if 'virtual host' in configuration['AMQP'] and configuration['AMQP']['virtual host'] != '':
url = str.format("{}/{}", url, configuration['AMQP']['virtual host'])
logger.debug("AMQPConnection configured with url %s", url)
exchange = Exchange(configuration['AMQP'].get('exchange', "amqp.topic"), type='topic', durable=True)
logger.debug("AMQPConnection is using exchange %s", exchange)
connection = None
if 'username' in configuration['AMQP'] and 'password' in configuration['AMQP']:
username = configuration['AMQP']['username']
password = configuration['AMQP']['password']
logger.debug("Using username %s and password %s to connect to AMQP Broker", username, password)
connection = Connection(url, userid=username, password=password)
else:
connection = Connection(url)
# typically connection connect on demand, but we want to flush out errors before proceeding
connection.connect()
exchange = exchange(connection)
exchange.declare()
return (connection, exchange)
def init(self):
try:
self.channel = channel = self.connection.channel()
exchange = Exchange(name='policymq.direct', type='direct', durable=False)
exchange.declare(channel=channel)
queue = Queue(name=self.queueName, durable=False)
queue.queue_declare(channel=channel)
queue.bind_to(exchange=exchange, routing_key='ALL_', channel=channel)
queue.bind_to(exchange=exchange, routing_key=self.queueName, channel=channel)
consumer = Consumer(channel=channel, queues=[queue], callbacks=[self._consume])
consumer.consume()
return True
except Exception as e:
logger.exception(e)
return False
def publish_message(body, routing_key):
try:
with Connection(os.environ['MESSAGE_BROKER_URL']) as conn:
with conn.channel() as channel:
exchange = Exchange(os.environ['MESSAGE_BROKER_EXCHANGE'],
type='topic', channel=channel, durable=True)
exchange.declare()
producer = Producer(channel=channel,
exchange=exchange,
routing_key=routing_key,
serializer='json')
producer.publish(body=body, retry=True)
except Exception as ex:
# TODO: Handle this better
logging.error(f"Something went wrong in publish_message - {ex.__class__} - {ex}")
def declare_queues(self, queues_names):
with self.connection as _conn:
_conn.connect()
channel = _conn.channel()
for queue_name in queues_names:
topic = Exchange(
name=f"TOPIC/{queue_name}", type="topic", channel=channel
)
topic.declare()
queue = Queue(name=queue_name, channel=channel)
queue.declare()
queue.bind_to(exchange=f"TOPIC/{queue_name}")
def get_mq_connection():
config_read = ConfigParser.RawConfigParser()
config_read.read('config.ini')
server=config_read.get('global','rabbit_mq')
from kombu import Connection, Producer, Exchange, Queue
connection=Connection('amqp://*****:*****@%s:5672//'%server)
channel = connection.channel();
exchange = Exchange("billing_collector", 'direct', channel)
exchange.declare()
queue = Queue("billing_collector", exchange=exchange, routing_key="billing_collector", channel=channel)
queue.declare()
return(connection, exchange);
def _declare_exchange(self, name, type, retry=False, retry_policy={}):
ex = Exchange(name,
type=type,
durable=self.durable,
auto_delete=self.auto_delete)(self.channel)
if retry:
return self.connection.ensure(ex, ex.declare, **retry_policy)
return ex.declare()
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url, exchange_name=None):
self.url = url
self.connect()
self.exchange_name = exchange_name if exchange_name else celery_queue(
"socket_notification")
self.channel = self.connection.channel()
self.socket_exchange = Exchange(self.exchange_name,
type="fanout",
channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""Test if connection is open.
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info("Connecting to broker {}".format(self.url))
self.connection = Connection(self.url, heartbeat=WS_HEART_BEAT)
self.connection.connect()
logger.info("Connected to broker {}".format(self.url))
def _close(self):
if hasattr(self, "connection") and self.connection:
logger.info("Closing connecting to broker {}".format(self.url))
self.connection.release()
self.connection = None
logger.info("Connection closed to broker {}".format(self.url))
def close(self):
self._close()
def append_to_topic(self, topic_name, queue_name, routing_key=None):
with self.connection as _conn:
_conn.connect()
channel = _conn.channel()
if not topic_name.startswith(self.config.get_event_name_prefix()):
topic_name = f"{self.config.get_event_name_prefix()}{topic_name}"
topic = Exchange(name=f"{topic_name}",
type="topic",
channel=channel)
topic.declare()
queue = Queue(name=queue_name,
channel=channel,
routing_key=routing_key)
queue.declare()
queue.bind_to(exchange=f"{topic_name}", routing_key=routing_key)
class KWriteQueue(object):
def __init__(self, channel, exchange, **kwargs):
self._exchange_declare = kwargs.get("exchange_declare", False)
if isinstance(exchange, Queue):
self.exchange = exchange.exchange
elif isinstance(exchange, basestring):
self.exchange = Exchange(exchange,
type="fanout") # , durable=True)
else:
assert isinstance(exchange, Exchange)
self.exchange = exchange
self.channel = maybe_channel(channel)
self.exchange.maybe_bind(self.channel)
if self._exchange_declare:
self.exchange.declare()
self.producer = messaging.Producer(channel,
self.exchange,
serializer='json',
routing_key='',
compression=None,
auto_declare=False)
def __enter__(self):
return self
def __exit__(self, *exc_info):
self.close()
def put(self,
message,
serializer=None,
headers=None,
compression=None,
routing_key=None,
**kwargs):
self.producer.publish(message,
content_type="application/octet-stream",
serializer=serializer,
routing_key=routing_key,
headers=headers,
compression=compression,
**kwargs)
def parse_ripe_ris(connection, prefix, host):
exchange = Exchange('bgp-update',
channel=connection,
type='direct',
durable=False)
exchange.declare()
def on_ris_msg(msg):
try:
producer = Producer(connection)
normalize_ripe_ris(msg)
if mformat_validator(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(msg,
exchange=exchange,
routing_key='update',
serializer='json')
else:
log.warning('Invalid format message: {}'.format(msg))
except Exception:
log.exception('exception')
with SocketIO('http://stream-dev.ris.ripe.net/stream2',
wait_for_connection=False) as socket_io:
socket_io.on('ris_message', on_ris_msg)
socket_io.emit(
'ris_subscribe', {
'host': host,
'type': 'UPDATE',
'prefix': prefix,
'moreSpecific': True,
'lessSpecific': False,
'socketOptions': {
'includeBody': False,
'explodePrefixes': True,
}
})
socket_io.wait()
def test_connection(host, port, user_id, password, virt_host, exchange_name,
queue_name):
"""
Test a connection to an exchange on a virtual host
"""
connection = None
connected = False
success = False
try:
# Connect to the virtual host - will raise exception if it fails.
connection = Connection(host, user_id, password, virt_host, port)
connection.connect()
connected = connection.connected
if connected:
# Check whether exchange exists - will raise exception if it fails.
exchange = Exchange(exchange_name,
channel=connection,
type='topic',
durable=False,
passive=True)
exchange.declare()
# Check whether the queue exists - will raise exception if it
# fails.
rpc_receive_queue = Queue(queue_name,
durable=True,
exchange=exchange,
channel=connection)
rpc_receive_queue.queue_declare(passive=True)
success = True
except Exception as e:
DLOG.info("Unable to connect to virt_host %s, exchange %s, error: %s" %
(virt_host, exchange_name, e))
finally:
if connected:
connection.close()
return success
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url):
self.url = url
self.connect()
self.channel = self.connection.channel()
self.socket_exchange = Exchange(exchange_name,
type='fanout',
channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""Test if connection is open.
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info('Connecting to broker {}'.format(self.url))
self.connection = Connection(self.url)
self.connection.connect()
logger.info('Connected to broker {}'.format(self.url))
def _close(self):
if hasattr(self, 'connection') and self.connection:
logger.info('Closing connecting to broker {}'.format(self.url))
self.connection.release()
self.connection = None
logger.info('Connection closed to broker {}'.format(self.url))
def close(self):
self._close()
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url, exchange_name=None):
self.url = url
self.connect()
self.exchange_name = exchange_name if exchange_name else celery_queue('socket_notification')
self.channel = self.connection.channel()
self.socket_exchange = Exchange(self.exchange_name, type='fanout', channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""Test if connection is open.
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info('Connecting to broker {}'.format(self.url))
self.connection = Connection(self.url, heartbeat=WS_HEART_BEAT)
self.connection.connect()
logger.info('Connected to broker {}'.format(self.url))
def _close(self):
if hasattr(self, 'connection') and self.connection:
logger.info('Closing connecting to broker {}'.format(self.url))
self.connection.release()
self.connection = None
logger.info('Connection closed to broker {}'.format(self.url))
def close(self):
self._close()
class SocketBrokerClient:
"""
Base class for web socket notification using broker (redis or rabbitmq)
"""
connection = None
def __init__(self, url):
self.url = url
self.connect()
self.channel = self.connection.channel()
self.socket_exchange = Exchange(exchange_name, type="fanout", channel=self.channel)
self.socket_exchange.declare()
def open(self):
"""
True if connected else false
:return bool:
"""
return self.connection and self.connection.connected
def connect(self):
self._close()
logger.info("Connecting to broker {}".format(self.url))
self.connection = Connection(self.url)
self.connection.connect()
logger.info("Connected to broker {}".format(self.url))
def _close(self):
if hasattr(self, "connection") and self.connection:
logger.info("Closing connecting to broker {}".format(self.url))
self.connection.release()
self.connection = None
logger.info("Connection closed to broker {}".format(self.url))
def close(self):
self._close()
def declare_exchange(self, name, type='direct', queues=None, **options):
"""Create or update exchange
:param name: name of exchange
:type name: str
:param type: type of exchange - direct, fanout, topic, match
:type type: str
:param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...]
:type queues: list, None or tuple
:param options: additional options for Exchange creation
"""
if queues is None:
queues = []
with connections[self.connection].acquire() as conn:
exchange = Exchange(name, type=type, channel=conn, **options)
exchange.declare()
self.exchanges[name] = exchange
for q_name, routing_key in queues:
queue = Queue(name=q_name, channel=conn)
queue.declare()
queue.bind_to(exchange=name, routing_key=routing_key)
self.logger.debug('Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name,
routing_key if routing_key else q_name, name)
class KWriteQueue(object):
def __init__(self, channel, exchange, **kwargs):
self._exchange_declare = kwargs.get("exchange_declare", False)
if isinstance(exchange, Queue):
self.exchange = exchange.exchange
elif isinstance(exchange, basestring):
self.exchange = Exchange(exchange, type="fanout") # , durable=True)
else:
assert isinstance(exchange, Exchange)
self.exchange = exchange
self.channel = maybe_channel(channel)
self.exchange.maybe_bind(self.channel)
if self._exchange_declare:
self.exchange.declare()
self.producer = messaging.Producer(channel, self.exchange,
serializer='json',
routing_key='',
compression=None,
auto_declare=False)
def __enter__(self):
return self
def __exit__(self, *exc_info):
self.close()
def put(self, message, serializer=None, headers=None, compression=None,
routing_key=None, **kwargs):
self.producer.publish(message,
content_type="application/octet-stream",
serializer=serializer,
routing_key=routing_key,
headers=headers,
compression=compression,
**kwargs)
def connect(self):
if not self.connection:
logging.info("Connecting to server %s", self.amqp_address)
self.connection = self.create_connection()
else:
return
self.channel = self.connection.channel()
self.channel.basic_qos(0, self.prefetch_count, False)
for qname, params in self.queues.iteritems():
if "exchange" in params:
exchange = Exchange(params["exchange"], **self.exchanges.get(params["exchange"], {}))
exchange = exchange(self.channel)
exchange.declare()
self.declared_exchanges[params["exchange"]] = exchange
queue_params = params.copy()
del queue_params['exchange']
self.declared_queues[qname] = Queue(qname, exchange=exchange, **queue_params)
else:
self.declared_queues[qname] = Queue(qname, **params)
self.declared_queues[qname](self.channel).declare()
def _declare_exchange(self, name, type, retry=False, retry_policy={}):
ex = Exchange(name, type=type, durable=self.durable,
auto_delete=self.auto_delete)(self.channel)
if retry:
return self.connection.ensure(ex, ex.declare, **retry_policy)
return ex.declare()
class Publisher(KombuConfReader):
def __init__(self, config):
self._log = log.getLogger()
KombuConfReader.__init__(self, config)
self.connection = Connection(self.broker_url)
try:
self._init_amqp()
except Exception as exc:
self._log.error('Publisher fail in init connection: %s' % exc)
raise
def _init_amqp(self):
"""Init AMQP objects after connection"""
self.producer = self.connection.Producer()
self.exchange = Exchange(
self.exchange_name,
channel=self.connection.channel(),
type=self.exchange_type,
durable=self.exchange_is_durable)
self.queue = Queue(
self.queue_name,
self.exchange,
channel=self.connection.channel(),
durable=self.queue_is_durable,
routing_key=self.routing_key,
queue_arguments=self.queue_args)
# We declare object to broker. this way only we can
# ensure to publish to an existing queue and routing_key
# AMQP work this way, not a library principle
self.exchange.declare()
self.queue.declare()
def switch_connection(self):
"""Switch AMQP connection from url to backup_url and vice versa"""
self._log.warn('Switching AMQP connection from %s' %
self.connection.as_uri())
if (self.connection.hostname in self.broker_url
and self.broker_backup_url):
self.connection = Connection(self.broker_backup_url)
elif self.connection.hostname in self.broker_backup_url:
self.connection = Connection(self.broker_url)
else:
raise URLError('Invalid current URI to switch connection : %s' %
self.connection.as_uri())
self._init_amqp()
def _publish(self, msg):
"""Publish message ensuring connection is available"""
publish = self.connection.ensure(
self.producer, self.producer.publish, max_retries=3)
publish(msg, exchange=self.exchange,
routing_key=self.routing_key,
serializer=self.serializer,
compression=self.compression)
return True
def publish(self, msg):
"""
must return True/False if message is well publish or not
"""
try:
return self._publish(msg)
except Exception as exc:
try:
self.switch_connection()
return self._publish(msg)
except Exception as exc:
self._log.error('Publish fail when switching connection: %s' %
exc)
return False
def handle_message(body, message):
print('Received message: %r' % (body, ))
print(' properties:\n%s' % (pretty(message.properties), ))
print(' delivery_info:\n%s' % (pretty(message.delivery_info), ))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('pyamqp://*****:*****@localhost:5672//') as connection:
# The configuration of the message flow is as follows:
# gateway_kombu_exchange -> internal_kombu_exchange -> kombu_demo queue
gateway_exchange = Exchange('gateway_kombu_demo')(connection)
exchange = Exchange('internal_kombu_demo')(connection)
gateway_exchange.declare()
exchange.declare()
exchange.bind_to(gateway_exchange, routing_key='kombu_demo')
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: This waits for a single event. Note that this event may not
#: be a message, or a message that is to be delivered to the consumers
#: channel, but any event received on the connection.
recv = eventloop(connection)
while True:
def process_media(body, message):
print '*'*10
print body
print '*'*10
message.ack()
# Connection
conn = Connection('amqp://guest@localhost//')
channel = conn.channel()
media_exchange = Exchange('media', 'topic', channel=channel, durable=True)
video_queue = Queue('video', channel=channel, exchange=media_exchange, routing_key='video')
try:
media_exchange.declare()
except amqp.exceptions.PreconditionFailed, e:
print 'zhe...'
print e
exit()
# produce
producer = conn.Producer(serializer='json', auto_declare=False)
producer.publish('name',
exchange = media_exchange, routing_key='video',
declare=[video_queue])
# # consume
# with conn.Consumer(video_queue, callbacks=[process_media]) as consumer:
# while True:
import boto
from kombu import BrokerConnection, Exchange, Queue, Consumer
connection = BrokerConnection()
connection.connect()
channel = connection.channel()
exchange = Exchange(name="android", type="fanout", channel=channel, durable=True)
exchange.declare()
channel = connection.channel()
queue = Queue(name="filr", exchange=exchange, durable=True, auto_delete=False, channel=channel, routing_key="filr")
queue.declare()
def fetch(b, m):
print b, m
consumer = Consumer(channel=connection.channel(), queues=queue, auto_declare=False, callbacks=[fetch])
consumer.consume(no_ack=False)
while True:
connection.drain_events()
pass
# execfile('.private-settings')
# sdb = boto.connect_sdb(key_id, sec_key)
# domain = sdb.create_domain('android')
# item = domain.new_item('kral_step1')
import datetime
import threading
from twisted.internet import protocol, reactor, defer
from kombu import Exchange, Queue, Consumer, Connection
from kombu.messaging import Producer
from kombu.transport.base import Message
from kombu.common import eventloop, drain_consumer
from kombu.async import Hub
connection = Connection('amqp://*****:*****@localhost//')
channel = connection.channel()
# 定义了一个exchange
task_exchange = Exchange('tasks', channel=channel, type='topic', durable=False)
task_exchange.declare()
# 在这里进行了exchange和queue的绑定，并且指定了这个queue的routing_key
task_queue = Queue('piap', task_exchange, channel=channel,
routing_key='suo_piao.#', durable=False)
task_queue2 = Queue('piap2.*abc.#', task_exchange, channel=channel,
routing_key='suo_piao.#', durable=False)
task_queue3 = Queue('piap3', task_exchange, channel=channel,
routing_key='suo_piao.abc.#', durable=False)
task_queue4 = Queue('piap4', task_exchange, channel=channel,
routing_key='abc.#', durable=False)
task_queues = []
for x in xrange(1,10):
tmpQueue = Queue('testFlood'+str(x), task_exchange, channel=channel,
routing_key='abc.*.'+str(x), durable=False)
tmpQueue.declare()
