def get_consumers(self, Consumer, channel):
assert self.mq_config
# Declaring ourselves rather than use auto-declare.
log.debug("Declaring %s exchange", self.mq_config.exchange)
self.exchange(channel).declare()
queues = []
for queue_name, routing_keys in self.mq_config.queues.items():
queue = Queue(name=queue_name,
exchange=self.exchange,
channel=channel,
durable=True)
log.debug("Declaring queue %s", queue_name)
queue.declare()
for routing_key in routing_keys:
log.debug("Binding queue %s to %s", queue_name, routing_key)
queue.bind_to(exchange=self.exchange,
routing_key=routing_key)
queues.append(queue)
consumer = Consumer(queues=queues,
callbacks=[self._callback],
auto_declare=False)
consumer.qos(prefetch_count=1, apply_global=True)
return [consumer]
class DelayedRetryQueue(Queue):
attrs = Queue.attrs + (('retry_delay', int), )
def __init__(self, *args, retry_delay=None, **kwargs):
super(DelayedRetryQueue, self).__init__(*args, **kwargs)
self.retry_delay = retry_delay or 1
retry_exchange_name = '{}-retry'.format(self.exchange.name)
retry_queue_name = '{}-retry'.format(self.name)
routing_key = self.routing_key
retry_exchange = Exchange(retry_exchange_name,
self.exchange.type,
durable=self.exchange.durable,
auto_delete=self.exchange.auto_delete)
self.retry_queue = Queue(
name=retry_queue_name,
exchange=retry_exchange,
routing_key=routing_key,
durable=self.durable,
auto_delete=self.auto_delete,
message_ttl=self.retry_delay,
queue_arguments={'x-dead-letter-exchange': self.exchange.name})
if self.queue_arguments is None:
self.queue_arguments = {}
self.queue_arguments['x-dead-letter-exchange'] = retry_exchange_name
def when_bound(self):
super(DelayedRetryQueue, self).when_bound()
self.retry_queue = self.retry_queue(self.channel)
def declare(self, *args, **kwargs):
self.retry_queue.declare(*args, **kwargs)
return super(DelayedRetryQueue, self).declare(*args, **kwargs)
def _get_or_create_queue(self, queue_name):
queue = Queue(name=queue_name,
exchange=self.exchange,
durable=True,
auto_delete=False)
queue.declare(channel=self.connection.channel())
return queue
def listen_events(cls, routing_key, exchange=BUS_EXCHANGE_NAME):
exchange = Exchange(exchange, type=BUS_EXCHANGE_TYPE)
with Connection(BUS_URL) as conn:
queue = Queue(BUS_QUEUE_NAME, exchange=exchange, routing_key=routing_key, channel=conn.channel())
queue.declare()
queue.purge()
cls.bus_queue = queue
def send_address_to_queue(message):
logger.info('Starting to send a message to {} queue'.format(
ADDRESS_PRODUCER_QUEUE))
with Connection(settings.BROKER_URL) as connection:
logger.info('Connected into the broker with success')
connection.connect()
channel = connection.channel()
exchange = Exchange(ADDRESS_EXCHANGE, type='direct')
producer = Producer(
channel=channel,
routing_key=ADDRESS_PRODUCER_ROUTING_KEY,
exchange=exchange,
)
queue = Queue(
name=ADDRESS_PRODUCER_QUEUE,
routing_key=ADDRESS_PRODUCER_ROUTING_KEY,
exchange=exchange,
)
from api.order.serializers import AddressSerializer
address_serializer = AddressSerializer(message).data
queue.maybe_bind(connection)
queue.declare()
producer.publish(address_serializer)
connection.close()
def run(rabbit_url):
print rabbit_url
conn = Connection(rabbit_url)
conn.ensure_connection()
conn.connect()
channel = conn.channel()
exchange = Exchange(config.EXCHANGE_NAME, type='direct')
producer = Producer(exchange=exchange,
channel=channel,
routing_key=config.ROUTING_KEY)
queue = Queue(name=config.QUEUE_NAME,
exchange=exchange,
routing_key=config.ROUTING_KEY)
queue.maybe_bind(conn)
queue.declare()
index = 0
while True:
try:
time.sleep(1)
print 'producer'
index += 1
producer.publish("send message -- %s" % index)
except socket.timeout:
pass
def declare_exchange(self, name, type='direct', queues=None, **options):
"""Create or update exchange
:param name: name of exchange
:type name: str
:param type: type of exchange - direct, fanout, topic, match
:type type: str
:param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...]
:type queues: list, None or tuple
:param options: additional options for Exchange creation
"""
if queues is None:
queues = [] # pragma: no cover
with self.connections[self.connection].acquire() as conn:
exchange = Exchange(name, type=type, channel=conn, **options)
exchange.declare()
self.exchanges[name] = exchange
for q_name, routing_key in queues:
queue = Queue(name=q_name, channel=conn)
queue.declare()
queue.bind_to(exchange=name, routing_key=routing_key)
self.logger.debug(
'Queue "%s" with routing_key "%s" was bond to exchange "%s"',
q_name, routing_key if routing_key else q_name, name)
class EventsPublisher():
def __init__(self):
self.rabbit_url = 'amqp://localhost:5672/'
self.conn = Connection(self.rabbit_url)
self.channel = self.conn.channel()
self.exchange = Exchange(name='gateway-exchange', type='fanout')
self.producer = Producer(exchange=self.exchange,
channel=self.channel,
routing_key='gateway')
self.queue = Queue(name='gateway-queue',
exchange=self.exchange,
routing_key='gateway')
self.queue.maybe_bind(self.conn)
self.queue.declare()
def publish(self, body):
# body = {
# # "id": "bb2cdchl52n4orsopmtg",
# "status": 1,
# "location": {
# "type": "Point",
# "coordinates": [2.2861460, 48.8268020],
# }
# }
self.producer.publish(body, serializer='json')
logging.info('*** Event published is: {}'.format(body))
def _send_command(self,
payload,
server_routing_key,
properties=None,
declare_queue=True):
if properties is None:
properties = {}
self.reply = None
logger.debug("Using connection: {!r}".format(CONN_DICT))
logger.info("Declaring queue %s." % self._client_queue)
queue = Queue(self._client_queue,
channel=self._conn,
durable=self._exchange.durable,
exchange=self._exchange,
routing_key=self._client_queue)
if declare_queue:
queue.declare()
self._queue = queue
with producers[self._conn].acquire(block=True) as producer:
producer.publish(payload,
serializer='json',
exchange=self._exchange,
declare=[self._exchange],
routing_key=server_routing_key,
**properties)
logger.info("Published {!r} to exchange "
"{!r}".format(payload, self._exchange))
def test_task_succeed(self):
"""What happens when a task succeeds.
Expect it to be acked from the queue.
"""
from kombu import Exchange
e = Exchange('', type='direct')
# declare queue
consumer_queue = Queue('test.task.succeed',
e,
channel=self._connection,
routing_key='test.task.succeed')
client_queue = Queue('', e, durable=False, channel=self._connection)
consumer_queue.declare()
client_queue.declare()
self.queues.append(client_queue)
q = Qurator(task_exchange=e, queue='test.task.succeed')
@q.task
def succeed(data):
return None
client = RpcClient(exchange=e)
client.task('succeed', {'x': 1},
server_routing_key='test.task.succeed')
curr_queues = q.queues['succeed']
@q.rpc
def still_around(body, message):
print("Message: {!r}".format(message))
self._connection.drain_events(timeout=1)
def _register_retry_queues(self, **_: Any) -> None:
"""
Initializes a set of AMQP primitives to implement broker-based delays.
"""
channel = self.broker_connection().default_channel
for queue in self.conf.task_queues:
retry_queue = Queue(
name=f'{queue.name}.retry',
routing_key=f'{queue.routing_key}.retry',
exchange=queue.exchange,
queue_arguments={
"x-dead-letter-exchange": "",
"x-dead-letter-routing-key": queue.name
}
)
retry_queue.declare(channel=channel)
retry_queue.maybe_bind(channel=channel)
archived_queue = Queue(
name=f'{queue.name}.archived',
routing_key=f'{queue.routing_key}.archived',
exchange=queue.exchange,
queue_arguments={
"x-message-ttl": defaults.AMQP_EVENTS_ARCHIVED_MESSAGE_TTL,
"x-max-length": defaults.AMQP_EVENTS_ARCHIVED_QUEUE_LENGTH,
"x-queue-mode": "lazy"
})
archived_queue.declare(channel=channel)
archived_queue.maybe_bind(channel=channel)
def init_adversary():
global __username, __my_type, __connection, __global_exchange, __producer, __backchannel, __backch_producer, __adv_exchange, __adv_queue #, __my_queue
__username = '******'
__my_type = 'adversary'
#adversary has broadcast-only access to the regular exchange
__connection = Connection('amqp://')
__connection.connect()
__global_exchange = Exchange('broadcast',
type='fanout',
durable=False,
delivery_mode=1)
__global_exchange.maybe_bind(__connection)
__producer = __connection.Producer(__connection)
__backchannel = Connection('amqp://')
__backchannel.connect()
__adv_exchange = Exchange('adversary', durable=False, delivery_mode=1)
__adv_exchange.maybe_bind(__backchannel)
__backch_producer = __backchannel.Producer(__backchannel)
__adv_queue = Queue('adversary',
exchange=__adv_exchange,
routing_key='adversary',
durable=False)
__adv_queue = __adv_queue(__backchannel)
__adv_queue.declare()
def _register_retry_queues(self, **_: Any) -> None:
"""
Initializes a set of AMQP primitives to implement broker-based delays.
Declares an exchange/queue pair for each delay stage defined by
`amqp_events.defaults:AMQP_EVENTS_MAX_RETRIES`.
Each exchange has a single bound queue; when task needs to be delayed,
it's re-published to new exchange preserving initial routing_key.
A queue for each exchange has `message-ttl` set to a power of 2. After
message is expired, it is re-routed by broker to `dead-letter-exchange`
named `recover`. All queues defined for event handlers are also bound to
this exchange with same routing key, thus each message after a retry
appears in same incoming queue.
`events` -> `demo.my_event_queue` -> Celery worker (retry)
| (publishes new message on retry)
V
`demo:retry.N` -> `demo:retry.N`
| (message ttl expires )
V
`recover` -> (routing key) -> `demo.my_event_queue` - > Celery worker
"""
channel = self.broker_connection().default_channel
for retry in range(defaults.AMQP_EVENTS_MAX_RETRIES):
name = self.get_retry_exchange_name(retry)
retry_exchange = Exchange(name=name, type=EXCHANGE_TYPE_FANOUT)
retry_queue = Queue(
name=name,
exchange=retry_exchange,
queue_arguments={
X_MESSAGE_TTL: 2**retry * 1000, # ms
X_DEAD_LETTER_EXCHANGE: self.recover_exchange_name,
})
retry_queue.declare(channel=channel)
retry_queue.maybe_bind(channel=channel)
def get_consumers(self, Consumer, channel):
assert self.mq_config
# Declaring ourselves rather than use auto-declare.
log.debug("Declaring %s exchange", self.mq_config.exchange)
self.exchange(channel).declare()
queues = []
for queue_name, routing_keys in self.mq_config.queues.items():
queue = Queue(name=queue_name,
exchange=self.exchange,
channel=channel,
durable=True)
log.debug("Declaring queue %s", queue_name)
queue.declare()
for routing_key in routing_keys:
log.debug("Binding queue %s to %s", queue_name, routing_key)
queue.bind_to(exchange=self.exchange, routing_key=routing_key)
queues.append(queue)
consumer = Consumer(queues=queues,
callbacks=[self._callback],
auto_declare=False)
consumer.qos(prefetch_count=1, apply_global=True)
return [consumer]
def send_geo_location_to_queue(address_id, geo_location):
with Connection(settings.BROKER_URL) as connection:
connection.connect()
channel = connection.channel()
exchange = Exchange(ADDRESS_EXCHANGE, type='direct')
producer = Producer(
channel=channel,
routing_key=ADDRESS_CUSTOMER_PRODUCER_ROUTING_KEY,
exchange=exchange,
)
queue = Queue(
name=ADDRESS_CUSTOMER_PRODUCER_QUEUE,
routing_key=ADDRESS_CUSTOMER_PRODUCER_ROUTING_KEY,
exchange=exchange,
)
geo_location_from_address = geo_location
geo_location_from_address.update({'id': address_id})
queue.maybe_bind(connection)
queue.declare()
producer.publish(geo_location_from_address)
connection.close()
def connect(self):
"""Connect to broker and possibly ensure exchange/queue/routing declared.
In case of errors, this will retry connecting several times
(controlled by the `CONN_*` class variables), and gracefully
recover if possible.
If it never succeeds, the exception raised by the final
attempt is re-raised.
"""
self.logger.debug('Connecting to broker at: {}'.format(self.url))
self._connection = Connection(self.url)
# Kombu interprets interval_max incorrectly; work around that.
interval_max = self.CONN_INTERVAL_MAX - self.CONN_INTERVAL_STEP
self._connection.ensure_connection(
errback=self.conn_errback,
max_retries=self.CONN_MAX_RETRIES,
interval_start=self.CONN_INTERVAL_START,
interval_step=self.CONN_INTERVAL_STEP,
interval_max=interval_max)
exchange = Exchange(name=self.exchange_name, type=self.exchange_type)
channel = self._connection.channel()
self._producer = Producer(channel=channel, exchange=exchange, routing_key=self.routing_key)
if self.queue_name:
# Bind/declare queue.
queue = Queue(name=self.queue_name, exchange=exchange, routing_key=self.routing_key)
queue = queue(channel) # Bind queue
self.logger.debug('Declaring queue {}, on exchange {} at {}'.format(
self.queue_name, self.exchange_name, self.url))
queue.declare()
def test_declare_but_no_exchange(self):
q = Queue('a')
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
def test_declare_but_no_exchange(self):
q = Queue('a')
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn('exchange_declare', chan)
self.assertIn('queue_declare', chan)
self.assertIn('queue_bind', chan)
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn('exchange_declare', chan)
self.assertIn('queue_declare', chan)
self.assertIn('queue_bind', chan)
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
assert b.is_bound
b.declare()
assert 'exchange_declare' in chan
assert 'queue_declare' in chan
assert 'queue_bind' in chan
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
assert b.is_bound
b.declare()
assert 'exchange_declare' in chan
assert 'queue_declare' in chan
assert 'queue_bind' in chan
def __assure_named_queue(self,
connection,
exchange,
queue_name,
routing_key_extension='*'):
routing_key = '{}.{}'.format(queue_name, routing_key_extension)
queue = Queue(queue_name, exchange, routing_key, connection)
queue.declare()
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
self.assertFalse(q.queue_declare.called)
self.assertFalse(q.queue_bind.called)
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_not_called()
q.queue_bind.assert_not_called()
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_not_called()
q.queue_bind.assert_not_called()
def _declare_queue(glance_api_cfg, routing_key, conn, exchange):
queue = Queue(name=routing_key,
routing_key=routing_key,
exchange=exchange,
channel=conn.channel(),
durable=False)
queue.declare()
return queue
def _declare_queue(glance_api_cfg, routing_key, conn, exchange):
queue = Queue(name=routing_key,
routing_key=routing_key,
exchange=exchange,
channel=conn.channel(),
durable=False)
queue.declare()
return queue
def add_binding(queue_name, routing_key, exchange_name=None):
exchange_name = exchange_name or world.config['bus']['exchange_name']
exchange = Exchange(exchange_name, type=world.config['bus']['exchange_type'])
with Connection(world.config['bus_url']) as conn:
queue = Queue(queue_name, exchange=exchange, routing_key=routing_key,
channel=conn.channel())
queue.declare()
queue.purge()
_queues[queue_name] = queue
def listen_events(self, routing_key, exchange=BUS_EXCHANGE_XIVO):
with Connection(self._url) as conn:
queue = Queue(BUS_QUEUE_NAME,
exchange=exchange,
routing_key=routing_key,
channel=conn.channel())
queue.declare()
queue.purge()
self.bus_queue = queue
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
self.assertFalse(q.queue_declare.called)
self.assertFalse(q.queue_bind.called)
def listen_events(cls, routing_key, exchange=BUS_EXCHANGE_NAME):
exchange = Exchange(exchange, type=BUS_EXCHANGE_TYPE)
with Connection(BUS_URL) as conn:
queue = Queue(BUS_QUEUE_NAME,
exchange=exchange,
routing_key=routing_key,
channel=conn.channel())
queue.declare()
queue.purge()
cls.bus_queue = queue
def connect(self):
self.close_channel()
try:
self.channel = self.connection.channel()
queue = Queue(name=self.queueName, durable=False)
queue.declare(channel=self.channel)
self.producer = Producer(channel=self.channel, routing_key='HOST')
except Exception as e:
logger.exception(e)
def main():
rabbit_url = "amqp://*****:*****@10.5.9.177:5672//"
conn = Connection(rabbit_url)
channel = conn.channel()
exchange = Exchange("test", type="direct")
producer = Producer(exchange=exchange, channel=channel, routing_key="vcc")
queue = Queue(name="chungpht", exchange=exchange, routing_key="vcc")
queue.maybe_bind(conn)
queue.declare()
producer.publish("Hello from other side 222")
def init_ack():
'''初始化返回exchange和queue'''
connection = Connection(CONF.rabbit.connection)
channel = connection.channel()
exchange = Exchange(CONF.region_id + '_ack', 'direct', channel)
exchange.declare()
queue = Queue(CONF.region_id + '_ack',
exchange=exchange,
routing_key=CONF.region_id + '_ack',
channel=channel)
queue.declare()
def accumulator(self, routing_key, exchange=None):
exchange = exchange or self._default_exchange
queue_name = 'test-{}'.format(str(uuid.uuid4()))
with Connection(self._url) as conn:
queue = Queue(name=queue_name,
exchange=exchange,
routing_key=routing_key,
channel=conn.channel())
queue.declare()
queue.purge()
accumulator = BusMessageAccumulator(self._url, queue)
return accumulator
def start_consuming(
self,
callback: Callable,
queue_name: str,
prefetch_count: int = 1,
no_ack: bool = False,
expires: int = None,
callback_ready: Callable = None,
):
if self._logger is not None:
self._logger.debug("Start consuming queue: %s" % queue_name)
self._consuming = True
while self._consuming:
revived_connection = self._connection.clone()
revived_connection.ensure_connection()
channel = revived_connection.channel()
channel.basic_qos(0, prefetch_count, True)
queues = []
queue_obj = Queue(
channel=channel,
name=queue_name,
no_ack=no_ack,
durable=False,
expires=expires,
queue_arguments={"x-max-priority": 255},
)
queue_obj.declare()
queues.append(queue_obj)
consumer = Consumer(
revived_connection,
queues,
callbacks=[callback],
accept=["json"],
auto_declare=False,
prefetch_count=prefetch_count,
)
consumer.revive(channel)
consumer.consume()
while self._consuming:
callback_ready is not None and callback_ready()
try:
revived_connection.drain_events(timeout=2)
except socket.timeout:
revived_connection.heartbeat_check()
except self._connection.connection_errors + (
AMQPError,
ConnectionForced,
ConnectionError,
): # pragma: no cover
if self._logger is not None:
self._logger.exception("Connection error", stack_info=True)
break
def declare_dead_queue():
channel = connection.channel()
dead_exchange = Exchange(name=config.rabbitmq_dead_exchange(),
type='direct',
channel=channel)
dead_queue = Queue(name=config.rabbitmq_dead_queue(),
routing_key=config.rabbitmq_routing_key(),
exchange=dead_exchange,
channel=channel)
dead_queue.declare()
return dead_exchange
class JobConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/jobs".format(config.username)
def get_consumers(self, Consumer, channel):
return [
Consumer(**c) for c in self.consumers
]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
)
self.consumers.append(dict(queues=self.queue,
callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def on_message(self, body, message):
store_pulse_jobs.apply_async(
args=[body,
message.delivery_info["exchange"],
message.delivery_info["routing_key"]],
routing_key='store_pulse_jobs'
)
message.ack()
def close(self):
self.connection.release()
def producer(msg=None):
print("------- in producer")
rabbit_url = "amqp://localhost:5672/"
conn = Connection(rabbit_url)
channel = conn.channel()
exchange = Exchange("scrapy", type="direct")
producer = Producer(exchange=exchange,
channel=channel,
routing_key="quotes")
queue = Queue(name="quotation", exchange=exchange, routing_key="quotes")
queue.maybe_bind(conn)
queue.declare()
producer.publish(msg)
print("published ->")
class Cloner(object):
def __init__(self, source_connection, target_connection,
source_exchange_name, target_queue_name,
intermediate_queue_name=None):
self.source_connection = source_connection
self.target_connection = target_connection
self.source_exchange_name = source_exchange_name
self.target_queue_name = target_queue_name
self.intermediate_queue_name = (
intermediate_queue_name or source_exchange_name + '_cloner'
)
def __str__(self):
return (
'{} @ {} > {} @ {}'.format(
self.source_exchange_name, self.source_connection,
self.target_queue_name, self.target_connection,
)
)
def _setup_queues(self):
src_channel = self.source_connection.channel()
self.intermediate_queue = Queue(
name=self.intermediate_queue_name,
exchange=Exchange(self.source_exchange_name),
routing_key=self.source_exchange_name,
channel=src_channel,
auto_delete=False,
)
self.intermediate_queue.declare()
self.target_queue = self.target_connection.SimpleQueue(
self.target_queue_name
)
def copy_message(self, body, msg):
logger.debug('{} - Copying message {}'.format(self, body))
self.target_queue.put(body)
msg.ack()
def clone(self):
self._setup_queues()
src_channel = self.source_connection.channel()
Consumer(
src_channel,
self.intermediate_queue,
callbacks=[self.copy_message],
).consume()
def test_basic_get(self):
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
chan2 = self.connection.channel()
queue = Queue("amqplib_basic_get", self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
for i in range(50):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def pre_declare_queues(self, queues):
"""Pre-declare any queues that will be used in tests.
:queues: list of names of queues
"""
declared_queues = []
for queue_name in queues:
q = Queue(queue_name,
self._exchange,
channel=self._connection,
durable=self._exchange.durable,
routing_key=queue_name)
q.declare()
declared_queues.append(q)
self.queues = declared_queues
def test_basic_get(self):
if not self.verify_alive():
return
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
chan2 = self.connection.channel()
queue = Queue(self.P("basic_get"), self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
for i in range(self.event_loop_max):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def test_basic_get(self):
if not self.verify_alive():
return
chan1 = self.connection.channel()
producer = chan1.Producer(self.exchange)
chan2 = self.connection.channel()
queue = Queue(self.P('basic_get'), self.exchange, 'basic_get')
queue = queue(chan2)
queue.declare()
producer.publish({'basic.get': 'this'}, routing_key='basic_get')
chan1.close()
for i in range(self.event_loop_max):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {'basic.get': 'this'})
self.purge([queue.name])
chan2.close()
def test_task_fail(self):
"""What happens when a task fails.
Expected behaviour is that it does not ack and is left in the queue.
"""
from kombu import Exchange
e = Exchange('', type='direct')
# declare queue
consumer_queue = Queue('test.task.fail',
e,
channel=self._connection,
routing_key='test.task.fail')
client_queue = Queue('',
e,
durable=False,
channel=self._connection)
consumer_queue.declare()
client_queue.declare()
self.queues.append(consumer_queue)
self.queues.append(client_queue)
q = Qurator(task_exchange=e)
@q.task(queue_name='test.task.fail')
def fail(data):
raise Exception('YOU FAIL!')
client = RpcClient(exchange=e)
client.task('fail', {'x': 1}, server_routing_key='test.task.fail')
curr_queues = q.queues['fail']
curr_callbacks = q.callbacks['fail']
def still_around(body, message):
self.assertFalse(message.acknowledged)
message.ack()
curr_callbacks.append(still_around)
with Consumer(self._connection, curr_queues, callbacks=curr_callbacks):
self._connection.drain_events(timeout=1)
def send(self):
try:
# Connection
conn = Connection(self.broker)
# Channel
channel = conn.channel()
# Exchange
task_exchange = Exchange(self._exchange_name,
type=self._queue_type)
# Queues
if self._queue_name:
queue = Queue(name=self._queue_name, channel=channel,
exchange=task_exchange,
routing_key=self._routing_key)
queue.declare()
# Producer
producer = Producer(exchange=task_exchange, channel=channel,
routing_key=self._routing_key)
# Send message
for message in self._msgs:
serialized_message = json.dumps(message, ensure_ascii=False)
producer.publish(serialized_message)
conn.close()
except Exception, e:
self.log.error(
u'QueueManagerError - Error on sending objects from queue.')
self.log.debug(e)
raise Exception(
'QueueManagerError - Error on sending objects to queue.')
def test_task_succeed(self):
"""What happens when a task succeeds.
Expect it to be acked from the queue.
"""
from kombu import Exchange
e = Exchange('', type='direct')
# declare queue
consumer_queue = Queue('test.task.succeed',
e,
channel=self._connection,
routing_key='test.task.succeed')
client_queue = Queue('',
e,
durable=False,
channel=self._connection)
consumer_queue.declare()
client_queue.declare()
self.queues.append(client_queue)
q = Qurator(task_exchange=e, queue='test.task.succeed')
@q.task
def succeed(data):
return None
client = RpcClient(exchange=e)
client.task('succeed',
{'x': 1},
server_routing_key='test.task.succeed')
curr_queues = q.queues['succeed']
@q.rpc
def still_around(body, message):
print("Message: {!r}".format(message))
self._connection.drain_events(timeout=1)
def declare_queues():
channel = connection.channel()
almanach_exchange = Exchange(name=config.rabbitmq_retry_return_exchange(),
type='direct',
channel=channel)
retry_exchange = Exchange(name=config.rabbitmq_retry_exchange(),
type='direct',
channel=channel)
retry_queue = Queue(name=config.rabbitmq_retry_queue(),
exchange=retry_exchange,
routing_key=config.rabbitmq_routing_key(),
queue_arguments=self._get_queue_arguments(),
channel=channel)
almanach_queue = Queue(name=config.rabbitmq_queue(),
exchange=almanach_exchange,
durable=False,
routing_key=config.rabbitmq_routing_key(),
channel=channel)
retry_queue.declare()
almanach_queue.declare()
return retry_exchange
def _send_command(self, payload, server_routing_key, properties=None,
declare_queue=True):
if properties is None:
properties = {}
self.reply = None
logger.debug("Using connection: {!r}".format(CONN_DICT))
logger.info("Declaring queue %s." % self._client_queue)
queue = Queue(self._client_queue,
channel=self._conn,
durable=self._exchange.durable,
exchange=self._exchange,
routing_key=self._client_queue)
if declare_queue:
queue.declare()
self._queue = queue
with producers[self._conn].acquire(block=True) as producer:
producer.publish(payload,
serializer='json',
exchange=self._exchange,
declare=[self._exchange],
routing_key=server_routing_key,
**properties)
logger.info("Published {!r} to exchange "
"{!r}".format(payload, self._exchange))
def declare_exchange(self, name, type='direct', queues=None, **options):
"""Create or update exchange
:param name: name of exchange
:type name: str
:param type: type of exchange - direct, fanout, topic, match
:type type: str
:param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...]
:type queues: list, None or tuple
:param options: additional options for Exchange creation
"""
if queues is None:
queues = []
with connections[self.connection].acquire() as conn:
exchange = Exchange(name, type=type, channel=conn, **options)
exchange.declare()
self.exchanges[name] = exchange
for q_name, routing_key in queues:
queue = Queue(name=q_name, channel=conn)
queue.declare()
queue.bind_to(exchange=name, routing_key=routing_key)
self.logger.debug('Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name,
routing_key if routing_key else q_name, name)
class Publisher(KombuConfReader):
def __init__(self, config):
self._log = log.getLogger()
KombuConfReader.__init__(self, config)
self.connection = Connection(self.broker_url)
try:
self._init_amqp()
except Exception as exc:
self._log.error('Publisher fail in init connection: %s' % exc)
raise
def _init_amqp(self):
"""Init AMQP objects after connection"""
self.producer = self.connection.Producer()
self.exchange = Exchange(
self.exchange_name,
channel=self.connection.channel(),
type=self.exchange_type,
durable=self.exchange_is_durable)
self.queue = Queue(
self.queue_name,
self.exchange,
channel=self.connection.channel(),
durable=self.queue_is_durable,
routing_key=self.routing_key,
queue_arguments=self.queue_args)
# We declare object to broker. this way only we can
# ensure to publish to an existing queue and routing_key
# AMQP work this way, not a library principle
self.exchange.declare()
self.queue.declare()
def switch_connection(self):
"""Switch AMQP connection from url to backup_url and vice versa"""
self._log.warn('Switching AMQP connection from %s' %
self.connection.as_uri())
if (self.connection.hostname in self.broker_url
and self.broker_backup_url):
self.connection = Connection(self.broker_backup_url)
elif self.connection.hostname in self.broker_backup_url:
self.connection = Connection(self.broker_url)
else:
raise URLError('Invalid current URI to switch connection : %s' %
self.connection.as_uri())
self._init_amqp()
def _publish(self, msg):
"""Publish message ensuring connection is available"""
publish = self.connection.ensure(
self.producer, self.producer.publish, max_retries=3)
publish(msg, exchange=self.exchange,
routing_key=self.routing_key,
serializer=self.serializer,
compression=self.compression)
return True
def publish(self, msg):
"""
must return True/False if message is well publish or not
"""
try:
return self._publish(msg)
except Exception as exc:
try:
self.switch_connection()
return self._publish(msg)
except Exception as exc:
self._log.error('Publish fail when switching connection: %s' %
exc)
return False
def __assure_named_queue(self, connection, exchange, queue_name, routing_key_extension='*'):
routing_key = '{}.{}'.format(queue_name, routing_key_extension)
queue = Queue(queue_name, exchange, routing_key, connection)
queue.declare()
task_exchange.declare()
# 在这里进行了exchange和queue的绑定，并且指定了这个queue的routing_key
task_queue = Queue('piap', task_exchange, channel=channel,
routing_key='suo_piao.#', durable=False)
task_queue2 = Queue('piap2.*abc.#', task_exchange, channel=channel,
routing_key='suo_piao.#', durable=False)
task_queue3 = Queue('piap3', task_exchange, channel=channel,
routing_key='suo_piao.abc.#', durable=False)
task_queue4 = Queue('piap4', task_exchange, channel=channel,
routing_key='abc.#', durable=False)
task_queues = []
for x in xrange(1,10):
tmpQueue = Queue('testFlood'+str(x), task_exchange, channel=channel,
routing_key='abc.*.'+str(x), durable=False)
tmpQueue.declare()
task_queues.append(tmpQueue)
task_queue.declare()
task_queue2.declare()
task_queue3.declare()
task_queue4.declare()
message = Message(channel, body=
{'state_strings': [u'pending'], 'stepid': 1, 'complete_at': None,
'name': u'git', 'buildid': 1, 'results': None, 'number': 0, 'urls': [],
'complete': False})
# produce
producer = Producer(channel, exchange=task_exchange, auto_declare=False, serializer="json")
producer.publish(message.body, routing_key='abc.suo_piao')
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection, queue_suffix):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/{}".format(config.username, queue_suffix)
def get_consumers(self, Consumer, channel):
return [
Consumer(**c) for c in self.consumers
]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
)
self.consumers.append(dict(queues=self.queue,
callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error("Unable to fetch existing bindings for {}".format(
self.queue_name))
logger.error("Data ingestion may proceed, "
"but no bindings will be pruned")
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: {}".format(binding_str))
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(
settings.PULSE_GUARDIAN_URL, queue_name))
import boto
from kombu import BrokerConnection, Exchange, Queue, Consumer
connection = BrokerConnection()
connection.connect()
channel = connection.channel()
exchange = Exchange(name="android", type="fanout", channel=channel, durable=True)
exchange.declare()
channel = connection.channel()
queue = Queue(name="filr", exchange=exchange, durable=True, auto_delete=False, channel=channel, routing_key="filr")
queue.declare()
def fetch(b, m):
print b, m
consumer = Consumer(channel=connection.channel(), queues=queue, auto_declare=False, callbacks=[fetch])
consumer.consume(no_ack=False)
while True:
connection.drain_events()
pass
# execfile('.private-settings')
# sdb = boto.connect_sdb(key_id, sec_key)
# domain = sdb.create_domain('android')
# item = domain.new_item('kral_step1')
