def publish(self, exchange_name, exchange_key, message, mheaders, mexp=0):
ebo=2
connected=True
while True:
if 'v03.' in exchange_key:
ct='application/json'
else:
ct='text/plain'
try:
if self.hc.use_amqp:
self.logger.debug("publish AMQP is used")
if mexp:
expms = '%s' % mexp
msg = amqp.Message(message, content_type=ct, application_headers=mheaders,
expiration=expms)
else:
msg = amqp.Message(message, content_type=ct, application_headers=mheaders)
self.channel.basic_publish(msg, exchange_name, exchange_key)
self.channel.tx_commit()
elif self.hc.use_amqplib:
self.logger.debug("publish AMQPLIB is used")
if mexp:
expms = '%s' % mexp
msg = amqplib_0_8.Message(message, content_type=ct, application_headers=mheaders,
expiration=expms)
else:
msg = amqplib_0_8.Message(message, content_type=ct, application_headers=mheaders)
self.channel.basic_publish(msg, exchange_name, exchange_key)
self.channel.tx_commit()
elif self.hc.use_pika:
self.logger.debug("publish PIKA is used")
if mexp:
expms = '%s' % mexp
properties = pika.BasicProperties(content_type=ct, delivery_mode=1, headers=mheaders,
expiration=expms)
else:
properties = pika.BasicProperties(content_type=ct, delivery_mode=1, headers=mheaders)
self.channel.basic_publish(exchange_name, exchange_key, message, properties, True)
else:
self.logger.debug("Couldn't choose an AMQP client library, setting it back to default amqp")
self.hc.use_amqp = True
raise ConnectionError("No AMQP client library is set")
return True
except Exception as err:
if ebo < 65:
ebo = ebo * 2
self.logger.error("sr_amqp/publish: Sleeping %d seconds ... and reconnecting" % ebo)
self.logger.debug('Exception details: ', exc_info=True)
time.sleep(ebo)
connected=False
if not connected:
self.hc.reconnect()
def amqp_send_msg(self, msg='', routing_key=''):
"""Publish AMQP message"""
if not isinstance(msg, amqp.Message):
if isinstance(msg, basestring):
msg = amqp.Message(msg)
else:
msg = amqp.Message(json.dumps(msg),
content_type='application/json')
log.debug("%s: Sending AMQP msg with routing key '%s' and body %r.",
self.lbl, routing_key, msg.body)
self.chan.basic_publish(msg, self.exchange, routing_key)
def post(self, payload):
if self.is_exchange:
message = amqp.Message(payload)
while True:
try:
self.channel.basic_publish(msg=message,
exchange=self.queue_name)
break
except Exception as e:
logger.error('Queue post to Exchange: %s' % e)
self.channel = self.queue_connection.re_channel(
self.channel)
else:
while True:
try:
if not self.simple_queue:
self.simple_queue = self.queue_connection._connection.SimpleQueue(
self.queue_name)
self.simple_queue.put(payload)
break
except Exception as e:
logger.error('Queue put: %s' % e)
self.queue_connection.reconnect()
if self.simple_queue:
self.simple_queue.close()
def _send_reply(
self,
reply_to: str,
channel: amqp.Channel,
result: Any,
exc_info: ExcInfoType,
) -> None:
logger.debug("Sending reply result=%r", result)
reply = {'result': result}
if exc_info:
reply['exception'] = self._exc_info_dict(exc_info)
try:
body = json.dumps(reply)
except Exception as e:
logger.error('Cannot serialize result as JSON: %s', e)
exc_info = sys.exc_info()
reply = {
'result': None,
'exception': self._exc_info_dict(exc_info)
}
body = json.dumps(reply)
msg = amqp.Message(body=body)
channel.basic_publish(msg, exchange="", routing_key=reply_to)
def call(self, md):
self.corr_id = str(uuid.uuid4())
message = amqp.Message(body=md, correlation_id=self.corr_id, reply_to=self.callback_queue)
self.channel.basic_publish(message, exchange='', routing_key=queue_name)
self.connection.drain_events()
return self.response
def _send_to_queue(self, args, kwargs, host=None, local=False):
"""
Sends this task to queue.
:param args: Arguments that will be passed to task on execution.
:param kwargs: Keyword arguments that will be passed to task
on execution.
:param host: Send this task to specific host. ``host`` will be
appended to the queue name.
:param local: Send this task to this host. Hostname of current host
will be appended to the queue name.
:return: :const:`None`
"""
logger.debug("Task.send_to_queue args=%r, kwargs=%r", args, kwargs)
queue = get_queue_name(self.queue,
host=host,
local=local or self.local)
description = self._get_description(args, kwargs, queue)
self._send_signal(signals.task_presend,
args=args,
kwargs=kwargs,
description=description)
body = json.dumps(description)
msg = amqp.Message(body=body)
with self.kuyruk.channel() as ch:
ch.queue_declare(queue=queue, durable=True, auto_delete=False)
ch.basic_publish(msg, exchange="", routing_key=queue)
self._send_signal(signals.task_postsend,
args=args,
kwargs=kwargs,
description=description)
def submit_job(app_id, app_key, task_id, mimetype, text_col, dedupe):
"""Submit a job to the queue for the Celery worker. Create the required JSON message and post it to RabbitMQ."""
# These are the args that the Python function in the adjunct processor will use.
kwargs = {
"app_id": app_id,
"app_key": app_key,
"task_id": task_id,
"format": mimetype,
"text_col": text_col,
"dedupe": dedupe,
"s3_endpoint": S3ENDPOINT,
"bucket": BUCKET,
"redis_port": REDISPORT,
"redis_host": REDISHOST
}
# Recreate a celery message manually so that we don't need to import celery_tasks.py which has heavy dependencies.
job = {
"id": task_id,
# "task": "synapsify_adjunct.celery_tasks.synapsify_master",
"task": "dc2_master",
"kwargs": kwargs
}
# Connect to RabbitMQ and post.
conn = amqp.Connection(host=RMQHOST,
port=RMQPORT,
userid=RMQUSERNAME,
password=RMQPASSWORD,
virtual_host=RMQVHOST,
insist=False)
cha = conn.channel()
msg = amqp.Message(json.dumps(job))
msg.properties["content_type"] = "application/json"
cha.basic_publish(routing_key=RMQEXCHANGE, msg=msg)
cha.close()
conn.close()
def publish_messages():
with amqp.Connection('localhost',
userid="iotile",
password="******",
confirm_publish=True) as c:
ch = c.channel()
with open("/opt/src/messages.txt", "r") as input_messages:
line = input_messages.readline()
while len(line) > 0:
signal, site, machine, value = line.split(",")
body = {"value": float(value)}
jbody = json.dumps(body)
message = amqp.Message(body=jbody,
application_headers={
"timestamp_in_ms":
int(time.time() * 1000)
})
ch.basic_publish(
message,
exchange="amq.topic",
routing_key=f'raw.arch.{site}.{machine}.{signal}')
time.sleep(0.1)
line = input_messages.readline()
def amqp_consume(self):
"""Connect to Hub Server and set up and start AMQP consumer"""
# define callback queue
self.queue = self.chan.queue_declare(exclusive=True).queue
self.chan.queue_bind(self.queue, self.exchange, self.queue)
self.chan.basic_consume(self.queue,
callback=self.amqp_handle_msg,
no_ack=True)
log.debug("%s: Initialized amqp connection, channel, queue.", self.lbl)
# send rpc request
self.worker_id = None
self.correlation_id = uuid.uuid4().hex
reply_to = self.queue
routing_key = '%s.worker.%s' % (self.key, self.worker_type)
msg = amqp.Message(json.dumps(self.worker_kwargs),
correlation_id=self.correlation_id,
reply_to=reply_to,
content_type='application/json')
self.amqp_send_msg(msg, routing_key)
log.info("%s: sent RPC request, will wait for response.", self.lbl)
# wait for rpc response
try:
while not self.worker_id:
log.debug("%s: Waiting for RPC response.", self.lbl)
self.chan.wait()
except BaseException as exc:
log.error(
"%s: Amqp consumer received %r while waiting for RPC "
"response. Stopping.", self.lbl, exc)
log.info("%s: Finished waiting for RPC response.", self.lbl)
super(HubClient, self).amqp_consume()
def _send(self, command, payload=None):
# send rpc request
if self.correlation_id:
raise Exception("Can't send second request while already waiting.")
self.response = None
self.correlation_id = uuid.uuid4().hex
routing_key = '%s.%s' % (self.key, command)
msg = amqp.Message(json.dumps(payload),
correlation_id=self.correlation_id,
reply_to=self.queue,
content_type='application/json')
log.debug("Sending AMQP msg with routing key '%s' and body %r.",
routing_key, msg.body)
self.chan.basic_publish(msg, self.exchange, routing_key)
log.info("Sent RPC request, will wait for response.")
# wait for rpc response
try:
while self.correlation_id:
log.debug("Waiting for RPC response.")
self.chan.wait()
except BaseException as exc:
log.error(
"Amqp consumer received %r while waiting for RPC "
"response. Stopping.", exc)
log.info("Finished waiting for RPC response.")
response = self.response
self.response = None
return response
def Publish(self, object):
channel = self.Connect_AMQP_UserPass()
channel.basic_publish(amqp.Message(body=json.dumps(object.EntityJSON)),
exchange=self.args.exchange
or object.DocumentType.encode("utf-8"),
routing_key=self.args.about
or object.ResourceID.encode("utf-8"))
def send_rpc(self, dest, method, params, block=True, timeout=None):
if len(self._connections) == 0:
raise RuntimeError('Not connected')
id_ = str(uuid.uuid1())
body = {
'reply_to': self.rpc_out_queue.queue,
'method': method,
'id': id_,
'params': params
}
LOG.debug('sending %s to %s', body, dest + ':rpc')
msg = amqp.Message(body=json.dumps(body),
reply_to=self.rpc_out_queue.queue)
self.active_rpcs[id_] = gevent.event.AsyncResult()
self.rpc_out_channel.basic_publish(msg, routing_key=dest + ':rpc')
try:
result = self.active_rpcs[id_].get(block=block, timeout=timeout)
except gevent.timeout.Timeout:
self.active_rpcs.pop(id_)
raise
return result
def __call__(self, report):
# Don't mess with the passed in report.
report = dict(report)
if not self.inherit_id or not report.get('id'):
# Discard any existing id.
original_id = report.pop('id', None)
# Hash it, to make an ID
oops_id = "OOPS-%s" % md5(dumps(report)).hexdigest()
# Store the id in what we send on the wire, so that the recipient
# has it.
report['id'] = oops_id
message = amqp.Message(dumps(report))
# We don't want to drop OOPS on the floor if rabbit is restarted.
message.properties["delivery_mode"] = 2
channel = self.get_channel()
if channel is None:
return []
try:
channel.basic_publish(
message, self.exchange_name, routing_key=self.routing_key)
except amqplib_error_types as e:
self.channels.channel = None
if is_amqplib_connection_error(e):
# Could not connect / interrupted connection
return []
# Unknown error mode : don't hide it.
raise
return [report['id']]
def test_publish_get(self):
self.channel.queue_declare(queue='py-amqp-unittest',
durable=False,
exclusive=True)
self.channel.basic_publish(amqp.Message('Unittest'),
routing_key='py-amqp-unittest')
msg = self.channel.basic_get(queue='py-amqp-unittest', )
assert msg.body_size == 8
assert msg.body == 'Unittest'
assert msg.frame_method == amqp.spec.Basic.GetOk
assert msg.delivery_tag == 1
assert msg.ready is True
assert msg.delivery_info == {
'delivery_tag': 1,
'redelivered': False,
'exchange': '',
'routing_key': 'py-amqp-unittest',
'message_count': 0
}
assert msg.properties == {'content_encoding': 'utf-8'}
self.channel.basic_ack(msg.delivery_tag)
msg = self.channel.basic_get(queue='py-amqp-unittest', )
assert msg is None
def _build_message_body(self, message):
"""
Build the AMQP message based on the input data
and the client's config.
:param ProviderEventsUpdateMessage message: Data for message body
:returns: Message with configuration
:rtype: amqp.Message
"""
message_str = json.dumps(message, default=self._serialize_models)
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
txt = json.dumps(json.loads(message_str),
indent=4,
separators=(',', ': '))
__log__.debug("Message: \n" + txt)
return amqp.Message(
body=message_str,
content_type=self.__amqp_config__["content_type"],
content_encoding=self.__amqp_config__["content_encoding"],
application_headers={
"__TypeId__":
"com.sios.stc.model.messaging." + type(message).__name__
})
def send_tasks_to_queue(self, subtasks):
if self.config.EAGER:
for subtask in subtasks:
subtask.task.apply(*subtask.args, **subtask.kwargs)
return
declared_queues = set()
with self.channel() as ch:
for subtask in subtasks:
queue = subtask.task._queue_for_host(subtask.host)
if queue not in declared_queues:
ch.queue_declare(queue=queue,
durable=True,
auto_delete=False)
declared_queues.add(queue)
description = subtask.task._get_description(
subtask.args, subtask.kwargs)
subtask.task._send_signal(signals.task_presend,
args=subtask.args,
kwargs=subtask.kwargs,
description=description)
body = json.dumps(description)
msg = amqp.Message(body=body)
ch.basic_publish(msg, exchange="", routing_key=queue)
subtask.task._send_signal(signals.task_postsend,
args=subtask.args,
kwargs=subtask.kwargs,
description=description)
def publish(self, msg, topic, **kwargs):
exchange_name = topic.split('.')[0]
routing_key = topic
content_type = 'text/plain'
if isinstance(msg, (list, dict)):
msg = json.dumps(msg)
content_type = 'application/json'
message = amqp.Message(msg,
content_type=content_type,
delivery_mode=1,
**kwargs)
num = len(self.pub_conncluster.all_connection_pools())
for i in range(num):
try:
connection = self.pub_conncluster.get_connection()
if not connection:
continue
except Exception as e:
logger.error(e)
continue
try:
return connection.publish(message, exchange_name, routing_key)
except socket.error as e:
logger.error(e)
self.pub_conncluster.connection_error(connection)
except ConnectionError as e:
logger.error(e)
self.pub_conncluster.connection_error(connection)
except Exception as e:
logger.error(e)
connection.disconnect()
finally:
self.pub_conncluster.release(connection)
logger.error('PUB FAIL', {'topic': topic, 'msg': msg})
def apply_async(self,
args,
kwargs=None,
correlation_id=None,
routing_key=None,
send_result=False,
result_queue=None):
if correlation_id is None:
correlation_id = str(uuid4())
job = {'task': self.name, 'args': args, 'kwargs': kwargs}
job_raw = json.dumps(job)
reply_to = result_queue or 'result_{0}'.format(correlation_id)
result = AsyncResult(reply_to)
msg = amqp.Message(job_raw,
content_type='application/json',
reply_to=reply_to,
correlation_id=correlation_id)
if routing_key is None:
routing_key = self.routing_key
with AmqpConnection() as channel:
if send_result and result_queue is None:
channel.queue_declare(queue=reply_to, auto_delete=False)
channel.basic_publish(msg,
exchange=self.exchange,
routing_key=routing_key)
logger.info('Task queued: name=%s, id=%s', self.name, correlation_id)
logger.debug('Task details: args=%s, kwargs=%s', repr(args),
repr(kwargs))
if send_result:
return result
def log(self, severity, msg):
index_data = json.dumps({
'index': {
'_index': settings.ES['index'],
'_type': self.log_name,
}
})
msg = json.dumps({
self.log_name: {
'severity': severity.upper(),
'msg': msg,
'timestamp': datetime.datetime.now(tz=UTC).isoformat(),
},
})
# TODO: this should be some generic method so that ES/RabbitMQ can
# be replaced later with something else
bulk_msg = '\n'.join([index_data, msg, '']) # trailing newline added
print bulk_msg
self.amqp_channel.basic_publish(
amqp.Message(bulk_msg),
exchange=self.amqp_exchange_name)
def send(self, message):
msg = amqp.Message(
message.json(),
content_type='application/json',
)
self._channel.basic_publish(msg, routing_key=self._queue)
def publish(self, message_data):
encoded = json.dumps({'data': message_data})
message = amqp.Message(encoded)
message.properties['delivery_mode'] = self.delivery_mode
self.channel.basic_publish(message,
exchange=self.exchange,
routing_key=self.routing_key)
return message
def main():
parser = OptionParser(
usage='usage: %prog [options] message\nexample: %prog hello world', )
parser.add_option(
'--host',
dest='host',
help='AMQP server to connect to (default: %default)',
default='localhost',
)
parser.add_option(
'-u',
'--userid',
dest='userid',
help='userid to authenticate as (default: %default)',
default='guest',
)
parser.add_option(
'-p',
'--password',
dest='password',
help='password to authenticate with (default: %default)',
default='guest',
)
parser.add_option(
'--ssl',
dest='ssl',
action='store_true',
help='Enable SSL (default: not enabled)',
default=False,
)
options, args = parser.parse_args()
if not args:
parser.print_help()
sys.exit(1)
msg_body = ' '.join(args)
conn = amqp.Connection(options.host,
userid=options.userid,
password=options.password,
ssl=options.ssl)
ch = conn.channel()
ch.exchange_declare('myfan', 'fanout')
msg = amqp.Message(msg_body,
content_type='text/plain',
application_headers={
'foo': 7,
'bar': 'baz'
})
ch.basic_publish(msg, 'myfan')
ch.close()
conn.close()
def prepare_message(self, body, priority=None,
content_type=None, content_encoding=None,
headers=None, properties=None):
"""Encapsulate data into a AMQP message."""
return amqp.Message(body, priority=priority,
content_type=content_type,
content_encoding=content_encoding,
application_headers=headers,
**properties)
def emiter_loop(*, url):
conn = yield from make_rabbitmq_connection(url=url)
chan = yield from make_rabbitmq_channel(conn)
for x in range(10000):
print("Sleep...")
yield from asyncio.sleep(1)
print("Publish: {}".format(x))
chan.basic_publish(amqp.Message("{}".format(x)), "events")
def test_invalid_json(self):
"""Message is dropped when JSON is not valid"""
with run_worker() as worker:
worker.expect('Consumer started')
with new_instance().channel() as ch:
msg = amqp.Message(body='foo')
ch.basic_publish(msg, exchange='', routing_key='kuyruk')
worker.expect('Cannot decode message')
self.assertEqual(len_queue('kuyruk'), 0)
def mq_send(self,**kwargs):
""
msg = kwargs.pop('msg', None)
exchange_name = kwargs.pop('exchange_name', None)
routing_key = kwargs.pop('routing_key', None)
message = amqp.Message(str(msg))
self.chan.basic_publish(message,exchange=exchange_name,routing_key=routing_key)
def call(self, args):
self.corr_id = str(uuid.uuid1())
message = amqp.Message(body=args,
correlation_id=self.corr_id,
reply_to=self.callback_queue)
self.channel.basic_publish(message,
exchange='',
routing_key=MarkdownRpcClient.QUEUE_NAME,
timeout=TIMEOUT)
self.connection.drain_events(timeout=TIMEOUT)
return self.response
def put_nowait(self, obj):
if self.lazy_limit and self.qsize_diff < self.qsize_diff_limit:
pass
elif self.full():
raise BaseQueue.Full
else:
self.qsize_diff = 0
with self.lock:
self.qsize_diff += 1
msg = amqp.Message(umsgpack.packb(obj))
return self.channel.basic_publish(msg, exchange="", routing_key=self.name)
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
self.channel.basic_publish(amqp.Message(str(n),
correlation_id=self.corr_id,
reply_to=self.callback_queue),
exchange='',
routing_key='rpc_queue')
while self.response is None:
self.channel.wait()
return int(self.response)
def test_invalid_task_path(self):
"""Message is dropped when task cannot be imported"""
with run_worker() as worker:
worker.expect('Consumer started')
with new_instance().channel() as ch:
desc = {'module': 'kuyruk', 'function': 'foo'}
body = json.dumps(desc)
msg = amqp.Message(body=body)
ch.basic_publish(msg, exchange='', routing_key='kuyruk')
worker.expect('Cannot import task')
self.assertEqual(len_queue('kuyruk'), 0)
