def declare_exchanges(self):
self._exchanges = {}
for name, exchange in self.exchanges.items():
ex = exchange(self.channel)
maybe_declare(ex)
self._exchanges[name] = ex
logger.debug("bound exchange {}".format(ex))
def on_task_call(self, producer, task_id):
# Called every time a task is sent when using this backend.
# We declare the queue we receive replies on in advance of sending
# the message, but we skip this if running in the prefork pool
# (task_join_will_block), as we know the queue is already declared.
if not task_join_will_block():
maybe_declare(self.binding(producer.channel), retry=True)
def declare_queues(self):
self._queues = {}
for name, queue in self.queues.items():
q = queue(self.channel)
maybe_declare(q)
self._queues[name] = q
logger.debug("bound queue {}".format(q))
def event_dispatcher(container_service_name, nameko_config, **kwargs):
""" Yield a function that dispatches events claiming to originate from
a service called `container_service_name`.
Enables services not hosted by nameko to dispatch events into a nameko
cluster.
"""
conn = Connection(nameko_config[AMQP_URI_CONFIG_KEY])
exchange = get_event_exchange(container_service_name)
retry = kwargs.pop('retry', True)
retry_policy = kwargs.pop('retry_policy', DEFAULT_RETRY_POLICY)
with connections[conn].acquire(block=True) as connection:
maybe_declare(exchange, connection)
with producers[conn].acquire(block=True) as producer:
def dispatch(evt):
msg = evt.data
routing_key = evt.type
producer.publish(
msg, exchange=exchange, routing_key=routing_key,
retry=retry, retry_policy=retry_policy, **kwargs)
yield dispatch
def test_on_message_is_sending_to_reply_queue(self, conn):
ret_result = 'foooo'
class Foo(A):
class state:
def bar(self, my_bar):
return ret_result
a = Foo(conn)
ticket = uuid()
delivery_tag = uuid()
body, message = get_encoded_test_message('bar', {'my_bar': 'bar_arg'},
A.__class__.__name__,
reply_to=ticket,
delivery_tag=delivery_tag)
# Set up a reply queue to read from
# reply_q and reply_exchange should be set the sender
a.reply_exchange = a.reply_exchange.bind(a.connection.default_channel)
maybe_declare(a.reply_exchange)
reply_q = a.get_reply_queue(ticket)
reply_q(a.connection.default_channel).declare()
a._on_message(body, message)
a_con = Consumer(conn.channel(), reply_q)
self.assertNextMsgDataEqual(a_con, {'ok': ret_result})
def add_binding(self, source, routing_key = '',
inbox_type = ACT_TYPE.DIRECT):
source_exchange = Exchange(**source)
binder = self.actor.get_binder(inbox_type)
#@TODO: It is correct to declare the destination?
maybe_declare(source_exchange, self.actor.connection.default_channel)
binder(exchange=source_exchange, routing_key=routing_key)
def CallServer(self, method, args = None):
try:
LOG.debug(_("strBroker : %s "), self._strBroker)
connection = BrokerConnection(self._strBroker)
# create the response channel
respQueueName = self._respQueueName + str(uuid())
respconnection = BrokerConnection(self._strBroker)
respQueue = respconnection.SimpleQueue(respQueueName,
queue_opts = {'durable': False, 'auto_delete': True},
exchange_opts = {'delivery_mode' : 1,
'auto_delete' : True,
'durable' : False})
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
payload = {"RespQueue": respQueueName, "Source": self._strBroker, 'Method': method, 'args': args}
producer.publish(payload, exchange = self._exchange, serializer="json", routing_key = self._routing_key)
# wait for the response
resp_message = respQueue.get(block=True, timeout=1)
resp_message.ack()
respQueue.close()
#respQueue.delete()
except:
LOG.debug(_("Exception caught : %s"), sys.exc_info()[0])
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
if type(resp_message.payload['Result']).__name__ in ('list', 'tuple'):
nElems = len(resp_message.payload['Result'])
if resp_message.payload['Result'][ nElems - 1 ] == -128:
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
elif type(resp_message.payload['Result']).__name__ == 'int':
if resp_message.payload['Result'] == -128:
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
else:
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
return resp_message.payload['Result']
def on_task_call(self, producer, task_id):
# Called every time a task is sent when using this backend.
# We declare the queue we receive replies on in advance of sending
# the message, but we skip this if running in the prefork pool
# (task_join_will_block), as we know the queue is already declared.
if not task_join_will_block():
maybe_declare(self.binding(producer.channel), retry=True)
def _bind_queues_in_for_all_regions(self, routing_key, connection):
for region in REGIONS:
maybe_declare(
Queue(exchange=orders_exchange,
routing_key='{}_{}'.format(region, routing_key),
name='fed.{}_{}'.format(region, routing_key)),
connection)
def _send(self, connection, request):
"""
:param request: JSON serializable dict
"""
dbutils.exit_if_in_transaction(log)
log.debug("send %s" % request["request_id"])
request["response_routing_key"] = self._response_routing_key
def errback(exc, _):
log.info(
"RabbitMQ rpc got a temporary error. May retry. Error: %r",
exc,
exc_info=1)
retry_policy = {"max_retries": 10, "errback": errback}
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True,
**retry_policy)
producer.publish(
request,
serializer="json",
routing_key=self._request_routing_key,
delivery_mode=TRANSIENT_DELIVERY_MODE,
retry=True,
retry_policy=retry_policy,
)
def republish(self, backoff_exc, message, target_queue):
expiration = backoff_exc.next(message, self.exchange.name)
queue = self.make_queue(expiration)
# republish to appropriate backoff queue
conn = Connection(self.container.config[AMQP_URI_CONFIG_KEY])
with producers[conn].acquire(block=True) as producer:
properties = message.properties.copy()
headers = properties.pop('application_headers')
headers['backoff'] = expiration
expiration_seconds = float(expiration) / 1000
# force redeclaration; the publisher will skip declaration if
# the entity has previously been declared by the same connection
maybe_declare(queue, conn, retry=True, **DEFAULT_RETRY_POLICY)
producer.publish(message.body,
headers=headers,
exchange=self.exchange,
routing_key=target_queue,
expiration=expiration_seconds,
retry=True,
retry_policy=DEFAULT_RETRY_POLICY,
declare=[self.exchange, queue],
**properties)
def setup_incoming(hostname):
connection = kombu.Connection(hostname=hostname)
connection.connect()
exchange = kombu.Exchange(type="topic", name="new.bankruptcy")
channel = connection.channel()
exchange.maybe_bind(channel)
maybe_declare(exchange, channel)
queue = kombu.Queue(name='simple', exchange=exchange, routing_key='#')
queue.maybe_bind(channel)
try:
queue.declare()
except AccessRefused:
logging.error("Access Refused")
logging.debug("queue name, exchange, binding_key: %s, %s, %s", queue.name, queue.exchange, queue.routing_key)
consumer = kombu.Consumer(channel, queues=queue, callbacks=[message_received], accept=['json'])
consumer.consume()
logging.debug('channel_id: %s', consumer.channel.channel_id)
logging.debug('queue(s): %s', consumer.queues)
return connection, consumer
def emit(connection, exchange, routing_key, message):
with producers[connection].acquire(block=True) as producer:
maybe_declare(exchange, producer.channel)
producer.publish(message,
serialize='json',
compression='bzip2',
exchange=exchange,
routing_key=routing_key)
def send_notification(notification):
# print 'recd note'
with BrokerConnection(settings.AMQP_URL) as connection:
with producers[connection].acquire(block=True) as producer:
maybe_declare(notifications_exchange, producer.channel)
producer.publish(
notification, exchange="notifications", routing_key="notifications", declare=[notifications_queue]
)
def external_shutdown():
global _producer_connection
from kombu.common import maybe_declare
from kombu.pools import producers
with producers[_producer_connection].acquire(block=True) as producer:
maybe_declare(_exchange, producer.channel)
producer.publish(
{"shutdown": True}, serializer="json", routing_key=ROUTING_KEY)
def send_as_task(connection, fun, args=(), kwargs={}, priority='mid'):
payload = {'fun': fun, 'args': args, 'kwargs': kwargs}
routing_key = priority_to_routing_key[priority]
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
producer.publish(
payload, serializer='pickle', compression='bzip2',
routing_key=routing_key)
def test_with_retry(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = True
maybe_declare(entity, channel, retry=True)
self.assertTrue(channel.connection.client.ensure.call_count)
def wake(source_id, reimport=False, force_restart=False, **kwargs):
global _producer_connection
from kombu.common import maybe_declare
from kombu.pools import producers
with producers[_producer_connection].acquire(block=True) as producer:
maybe_declare(_exchange, producer.channel)
kwargs.update(dict(source=source_id, reimport=reimport, force_restart=force_restart))
producer.publish(kwargs, serializer="json", routing_key=ROUTING_KEY)
def test_binds_entities(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = False
maybe_declare(entity, channel)
entity.bind.assert_called_with(channel)
def _add_binding(self,
source,
routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
source_exchange = Exchange(**source)
binder = self.get_binder(inbox_type)
maybe_declare(source_exchange, self.connection.default_channel)
binder(exchange=source_exchange, routing_key=routing_key)
def prod(source_id):
global _producer_connection
from kombu.common import maybe_declare
from kombu.pools import producers
with producers[_producer_connection].acquire(block=True) as producer:
maybe_declare(_exchange, producer.channel)
producer.publish(
source_id, serializer="json", routing_key=ROUTING_KEY)
def notifications_send(payload, created=None):
with producers[connection].acquire(block=True) as producer:
maybe_declare(queue_notifications_events, producer.channel)
payload['created'] = created
producer.publish(payload,
exchange='geonode',
serializer='json',
routing_key='notifications')
def test_binds_entities(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = False
maybe_declare(entity, channel)
entity.bind.assert_called_with(channel)
def test_with_retry(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = True
maybe_declare(entity, channel, retry=True)
self.assertTrue(channel.connection.client.ensure.call_count)
def send_email_producer(layer_uuid, user_id):
with producers[connection].acquire(block=True) as producer:
maybe_declare(queue_email_events, producer.channel)
payload = {"layer_uuid": layer_uuid, "user_id": user_id}
producer.publish(payload,
exchange='geonode',
serializer='json',
routing_key='email')
def geoserver_upload_layer(payload):
with producers[connection].acquire(block=True) as producer:
maybe_declare(queue_geoserver_events, producer.channel)
producer.publish(
payload,
exchange='geonode',
serializer='json',
routing_key='geonode.geoserver'
)
def prepare(self):
exchange = self.exchange
queue = self.queue
with self.get_connection() as conn:
if queue is not None:
maybe_declare(queue, conn)
elif exchange is not None:
maybe_declare(exchange, conn)
def send_notification(notification):
with BrokerConnection(settings.AMPQ_URL) as connection:
with producers[connection].acquire(block=True) as producer:
maybe_declare(notifications_exchange, producer.channel)
producer.publish(
notification,
exchange=notifications_exchange,
routing_key='notifications'
)
def publish(self, body, routing_key=None, delivery_mode=None,
mandatory=False, immediate=False, priority=0,
content_type=None, content_encoding=None, serializer=None,
headers=None, compression=None, exchange=None, retry=False,
retry_policy=None, declare=[], **properties):
headers = {} if headers is None else headers
retry_policy = {} if retry_policy is None else retry_policy
routing_key = self.routing_key if routing_key is None else routing_key
compression = self.compression if compression is None else compression
exchange = exchange or self.exchange
logger.info('Producer publish')
callback = properties.pop('callback', None)
task_id = body['id']
if callback and not callable(callback):
raise ValueError('callback should be callable')
body, content_type, content_encoding = self._prepare(
body, serializer, content_type, content_encoding,
compression, headers)
self.serializer = self.app.backend.serializer
serialization.registry.enable(serializer)
(self.content_type,
self.content_encoding,
self.encoder) = serialization.registry._encoders[self.serializer]
conn = self.conn_pool.connection()
# auto create/bind exchange/queue for the first call and caches
for entity in declare:
common.maybe_declare(entity, self.channel, retry=True)
publish = conn.publish
result = publish(body, priority=priority, content_type=content_type,
content_encoding=content_encoding, headers=headers,
properties=properties, routing_key=routing_key,
mandatory=mandatory, immediate=immediate,
exchange=exchange, declare=declare)
if callback:
async_result = self.result_cls(task_id=task_id,
result=result,
producer=self)
if conn.confirm_delivery:
conn.confirm_delivery_handler.add_callback(lambda result:
callback(async_result))
else:
logger.info("Run callback passed from send_task")
callback(async_result)
return result
def viewing_layer(user, owner, layer_id):
with producers[connection].acquire(block=True) as producer:
maybe_declare(queue_layer_viewers, producer.channel)
payload = {"viewer": user, "owner_layer": owner, "layer_id": layer_id}
producer.publish(payload,
exchange='geonode',
serializer='json',
routing_key='geonode.viewer')
def send_as_task(connection, fun, args=(), kwargs={}, priority="mid"):
payload = {"fun": fun, "args": args, "kwargs": kwargs}
routing_key = priority_to_routing_key[priority]
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
producer.publish(payload, serializer="pickle",
compression="bzip2",
routing_key=routing_key)
def prepare(self):
exchange = self.exchange
queue = self.queue
with self.get_connection() as conn:
if queue is not None:
maybe_declare(queue, conn)
elif exchange is not None:
maybe_declare(exchange, conn)
def run(self):
try:
result = {
"result": self.rpc._local_call(self.body["method"], *self.body["args"], **self.body["kwargs"]),
"request_id": self.body["request_id"],
"exception": None,
}
except Exception as e:
import sys
import traceback
exc_info = sys.exc_info()
backtrace = "\n".join(traceback.format_exception(*(exc_info or sys.exc_info())))
# Utility to generate human readable errors
def translate_error(err):
from socket import error as socket_error
if type(err) == socket_error:
return "Cannot reach server"
return str(err)
result = {
"request_id": self.body["request_id"],
"result": None,
"exception": translate_error(e),
"exception_type": type(e).__name__,
"traceback": backtrace,
}
log.error("RunOneRpc: exception calling %s: %s" % (self.body["method"], backtrace))
finally:
django.db.connection.close()
with self._response_conn_pool[_amqp_connection()].acquire(block=True) as connection:
def errback(exc, _):
log.info("RabbitMQ rpc got a temporary error. May retry. Error: %r", exc, exc_info=1)
retry_policy = {"max_retries": 10, "errback": errback}
connection.ensure_connection(**retry_policy)
with Producer(connection) as producer:
maybe_declare(_amqp_exchange(), producer.channel, True, **retry_policy)
producer.publish(
result,
serializer="json",
routing_key=self.body["response_routing_key"],
delivery_mode=TRANSIENT_DELIVERY_MODE,
retry=True,
retry_policy=retry_policy,
immedate=True,
mandatory=True,
)
def send_as_task(connection, fun, args=(), kwargs={}, priority='mid'):
payload = {'fun': fun, 'args': args, 'kwargs': kwargs}
routing_key = priority_to_routing_key[priority]
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
producer.publish(payload,
serializer='pickle',
compression='bzip2',
routing_key=routing_key)
def send_as_task(connection, fun, args, kwargs, priority="mid"):
payload = {"fun": fun, "args": args, "kwargs": kwargs}
routing_key = priority_to_routing_key[priority]
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
producer.publish(payload,
serializer="pickle",
compression="zlib",
routing_key=routing_key)
def geoserver_upload_dataset(payload):
with producers[connection].acquire(
block=True, timeout=broker_socket_timeout) as producer:
maybe_declare(queue_geoserver_events, producer.channel)
producer.publish(payload,
exchange='geonode',
serializer=task_serializer,
routing_key='geonode.geoserver',
timeout=broker_socket_timeout)
producer.release()
def notifications_send(payload, created=None):
with producers[connection].acquire(block=True) as producer:
maybe_declare(queue_notifications_events, producer.channel)
payload['created'] = created
producer.publish(
payload,
exchange='geonode',
serializer='json',
routing_key='notifications'
)
def send_message(self, **kwargs):
connection = BrokerConnection('amqp://%(mq_user)s:%(mq_password)s@'
'%(mq_host)s:%(mq_port)s//' % kwargs)
message = {'From': __file__, 'Date': str(datetime.datetime.now())}
with producers[connection].acquire(block=True) as producer:
maybe_declare(entropy_exchange, producer.channel)
producer.publish(message,
exchange=entropy_exchange,
routing_key=PASS_KEY,
serializer='json')
def test_binds_entities(self):
channel = Mock()
channel.connection.client.declared_entities = set()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = False
entity.bind.return_value = entity
entity.bind.return_value.channel = channel
maybe_declare(entity, channel)
entity.bind.assert_called_with(channel)
def test_with_retry(self):
channel = Mock()
client = channel.connection.client = Mock()
client.declared_entities = set()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = True
entity.channel = channel
maybe_declare(entity, channel, retry=True)
assert channel.connection.client.ensure.call_count
def notifications_send(payload, created=None):
with producers[connection].acquire(
block=True, timeout=broker_socket_timeout) as producer:
maybe_declare(queue_notifications_events, producer.channel)
payload['created'] = created
producer.publish(payload,
exchange='geonode',
serializer=task_serializer,
routing_key='notifications',
timeout=broker_socket_timeout)
producer.release()
def test_uncacheable(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = False
entity.is_bound = True
maybe_declare(entity, channel)
self.assertEqual(entity.declare.call_count, 1)
maybe_declare(entity, channel)
self.assertEqual(entity.declare.call_count, 2)
def geoserver_upload_layer(payload):
with producers[connection].acquire(block=True, timeout=broker_socket_timeout) as producer:
maybe_declare(queue_geoserver_events, producer.channel)
producer.publish(
payload,
exchange='geonode',
serializer=task_serializer,
routing_key='geonode.geoserver',
timeout=broker_socket_timeout
)
producer.release()
def _get_connection(self):
if hasattr(self, '_connection'):
return self._connection
self._connection = BrokerConnection(**self.config)
with producers[self.connection].acquire(block=False) as producer:
for queue in task_queues:
maybe_declare(queue, producer.channel)
return self._connection
def test_binds_entities(self):
channel = Mock()
channel.connection.client.declared_entities = set()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = False
entity.bind.return_value = entity
entity.bind.return_value.channel = channel
maybe_declare(entity, channel)
entity.bind.assert_called_with(channel)
def test_with_retry(self):
channel = Mock()
client = channel.connection.client = Mock()
client.declared_entities = set()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = True
entity.channel = channel
maybe_declare(entity, channel, retry=True)
assert channel.connection.client.ensure.call_count
def test_uncacheable(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = False
entity.is_bound = True
maybe_declare(entity, channel)
self.assertEqual(entity.declare.call_count, 1)
maybe_declare(entity, channel)
self.assertEqual(entity.declare.call_count, 2)
def send_email_producer(dataset_uuid, user_id):
with producers[connection].acquire(
block=True, timeout=broker_socket_timeout) as producer:
maybe_declare(queue_email_events, producer.channel)
payload = {"dataset_uuid": dataset_uuid, "user_id": user_id}
producer.publish(payload,
exchange='geonode',
serializer=task_serializer,
routing_key='email',
timeout=broker_socket_timeout)
producer.release()
def viewing_layer(user, owner, layer_id):
with producers[connection].acquire(
block=True, timeout=broker_socket_timeout) as producer:
maybe_declare(queue_layer_viewers, producer.channel)
payload = {"viewer": user, "owner_layer": owner, "layer_id": layer_id}
producer.publish(payload,
exchange='geonode',
serializer=task_serializer,
routing_key='geonode.viewer',
timeout=broker_socket_timeout)
producer.release()
def get_binder(self, type):
if type == ACT_TYPE.DIRECT:
entity = self.type_to_queue[type]()
binder = entity.queue_bind
else:
entity = self.type_to_exchange[type]()
binder = entity.exchange_bind
#@TODO: Declare probably should not happened here
entity.maybe_bind(self.connection.default_channel)
maybe_declare(entity, entity.channel)
return binder
def setup(self):
exchange = self.exchange
queue = self.queue
verify_amqp_uri(self.amqp_uri)
with get_connection(self.amqp_uri) as conn:
if queue is not None:
maybe_declare(queue, conn)
elif exchange is not None:
maybe_declare(exchange, conn)
def test_cacheable(self):
channel = Mock()
entity = Mock()
entity.can_cache_declaration = True
entity.is_bound = True
maybe_declare(entity, channel)
self.assertEqual(entity.declare.call_count, 1)
self.assertIn(entity, declared_entities[channel.connection.client])
maybe_declare(entity, channel)
self.assertEqual(entity.declare.call_count, 1)
def get_binder(self, type):
if type == ACTOR_TYPE.DIRECT:
entity = self.type_to_queue[type]()
elif type in self.types:
entity = self.type_to_exchange[type]()
else:
raise ValueError('Unsupported type: {0}'.format(type))
binder = entity.bind_to
# @TODO: Declare probably should not happened here
entity.maybe_bind(self.connection.default_channel)
maybe_declare(entity, entity.channel)
return binder
def notifications_send(payload, created=None):
with producers[connection].acquire(block=True, timeout=broker_socket_timeout) as producer:
maybe_declare(queue_notifications_events, producer.channel)
payload['created'] = created
producer.publish(
payload,
exchange='geonode',
serializer=task_serializer,
routing_key='notifications',
timeout=broker_socket_timeout
)
producer.release()
def get_binder(self, type):
if type == ACTOR_TYPE.DIRECT:
entity = self.type_to_queue[type]()
elif type in self.types:
entity = self.type_to_exchange[type]()
else:
raise ValueError('Unsupported type: {0}'.format(type))
binder = entity.bind_to
# @TODO: Declare probably should not happened here
entity.maybe_bind(self.connection.default_channel)
maybe_declare(entity, entity.channel)
return binder
def test_binds_entities(self):
# Given: A mock Channel and mock entity
channel = self._get_mock_channel()
# Given: A mock Entity that is not bound
entity = self._get_mock_entity()
assert not entity.is_bound, "Expected entity unbound to begin test."
# When: calling maybe_declare with default of no retry policy
maybe_declare(entity, channel)
# Then: the entity is now bound because it called to bind it
assert entity.is_bound is True, "Expected entity is now marked bound."
def test_with_retry(self):
# Given: A mock Channel and mock entity
channel = self._get_mock_channel()
# Given: A mock Entity that is already bound
entity = self._get_mock_entity(is_bound=True,
can_cache_declaration=True)
entity.channel = channel
assert entity.is_bound, "Expected entity is bound to begin this test."
# When calling maybe_declare with retry enabled (default policy)
maybe_declare(entity, channel, retry=True)
# Then: the connection client used ensure to ensure the retry policy
assert channel.connection.client.ensure.call_count
