def __publish(self,
message,
routing_key,
echange,
delivery_mode_v=1,
encoding=DateFormatEncoder,
reply_to=None,
correlation_id=None,
publish_watch=True):
properties = BasicProperties(delivery_mode=delivery_mode_v)
if reply_to is not None:
properties.reply_to = reply_to
if correlation_id is not None:
properties.correlation_id = correlation_id
message_utf8 = TransmetteurMessageMilleGrilles.__json_helper.dict_vers_json(
message, encoding)
self.__logger.debug("publish message: %s" % message_utf8)
with self.__channel_lock:
# Utiliser pubdog pour la connexion publishing par defaut
self.__channel.basic_publish(exchange=echange,
routing_key=routing_key,
body=message_utf8,
properties=properties,
mandatory=True)
if publish_watch:
self.publish_watch()
def send_message_to(
self,
exchange_name: str,
ch: BlockingChannel,
routing_key: str,
properties: BasicProperties,
body: bytes,
):
if properties.headers:
redelivery_count = properties.headers.get("redelivery_count", 0)
properties.headers["redelivery_count"] = redelivery_count + 1
else:
properties.headers = {"redelivery_count": 1}
self.printer.print_send_message_to(exchange_name, routing_key,
properties.headers)
ch.basic_publish(
exchange=exchange_name,
routing_key=routing_key,
body=body,
properties=properties,
)
return properties.headers
def send_message(self, message, tx_id, questionnaire_id, case_id=None):
"""
Sends a message to rabbit mq and returns a true or false depending on if it was successful
:param message: The message to send to the rabbit mq queue
:param tx_id: Transaction ID used to trace a transaction through the whole system.
:param questionnaire_id: Questionnaire ID used to identify the questionnaire.
:param case_id: ID used to identify a single instance of a survey collection for a respondent
:return: a boolean value indicating if it was successful
"""
message_as_string = str(message)
logger.info("sending message", category="rabbitmq")
logger.info("message payload",
message=message_as_string,
category="rabbitmq")
connection = None
try:
connection = self._connect()
channel = connection.channel()
channel.queue_declare(queue=self.queue, durable=True)
properties = BasicProperties(headers={}, delivery_mode=2)
properties.headers["tx_id"] = tx_id
properties.headers["questionnaire_id"] = questionnaire_id
if case_id:
properties.headers["case_id"] = case_id
published = channel.basic_publish(
exchange="",
routing_key=self.queue,
body=message_as_string,
mandatory=True,
properties=properties,
)
if published:
logger.info("sent message", category="rabbitmq")
else:
logger.error("unable to send message", category="rabbitmq")
return published
except AMQPError as e:
logger.error("unable to send message",
exc_info=e,
category="rabbitmq")
return False
finally:
if connection:
self._disconnect(connection)
def publish(self, status):
self.channel.basic_publish(exchange="",
routing_key=self.exchange,
body=status,
properties=BasicProperties(
content_type="text/plain",
delivery_mode=1))
def handle_delivery(channel, method_frame, body):
global COUNT
j = json.loads(body)
# pprint.pprint(j)
if 'queue_name' in j:
queue_name = str(j["queue_name"])
elif 'job' in j and 'job_info' in j['job'] and 'job_queue' in j['job'][
'job_info']:
queue_name = str(j['job']['job_info']['job_queue'])
else:
queue_name = "<unknown>"
# ask for an action
if queue_name == 'product_processed':
channel2 = CONNECTION.channel()
channel2.queue_declare(queue=queue_name, durable=True)
channel2.basic_publish(
exchange='',
routing_key=queue_name,
body=j['body'],
properties=BasicProperties(
delivery_mode=2, # make message persistent
))
# Acknowledge the message
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
# quit if no more
COUNT -= 1
if COUNT == 0:
return True
return False
def process_converted(converted_body):
"""Callback for body after convert"""
def rmq_consume_error(problem):
"""Something went wrong with channel.basic_publish"""
#pylint: disable=unused-argument
self.running = False
# Pylint Rational:
# Fixme: we should probably do something with the problem.
#pylint: disable=no-member
reactor.stop()
def on_consumed(argument=None):
"""Called when channel.basic_publish completes"""
# Pylint Rational:
# Callback that should work with one or zero arguments.
# Argument ignored.
#pylint: disable=unused-argument
self.hysteresis_queue.get(self._process_body)
if isinstance(converted_body, (bytes, str)):
props = BasicProperties(delivery_mode=2)
self.publish_deferred = self.channel.basic_publish(
properties=props,
exchange=self.exchange,
routing_key=self.routing_key,
body=converted_body)
self.publish_deferred.addCallbacks(on_consumed,
rmq_consume_error)
else:
raise TypeError(
"converter should produce string or bytes, not",
type(converted_body))
def send_message(self, queue_name, msg):
self.channel.basic_publish(exchange=EXCHANGE_CONFIG['exchange_name'],
routing_key=EXCHANGE_CONFIG['routing_key'],
body=msg,
properties=BasicProperties(
delivery_mode=2, ))
self.connection.close()
def call(self, msg):
assert self._connection, "Connect Before Rpc Call!"
self._rabbit_channel.queue_declare(queue=self._routing_key)
result = self._rabbit_channel.queue_declare(queue="", exclusive=True)
self._callback_queue = result.method.queue
self._corr_id = None
self._rabbit_channel.basic_consume(
queue=self._callback_queue,
on_message_callback=self._on_response,
auto_ack=True,
)
self._rsp = None
self._corr_id = str(uuid.uuid4())
self._rabbit_channel.basic_publish(
exchange="",
routing_key=str(self._routing_key),
properties=BasicProperties(
reply_to=self._callback_queue, correlation_id=self._corr_id
),
body=str(msg),
)
while self._rsp is None:
self._connection.poll_event()
self._rabbit_channel.queue_delete(self._callback_queue)
return self._rsp
def _get_message_properties(
self,
headers: Optional[Dict[Text, Text]] = None) -> "BasicProperties":
"""Create RabbitMQ message `BasicProperties`.
The `app_id` property is set to the value of `self.rasa_environment` if
present, and the message delivery mode is set to 2 (persistent). In
addition, the `headers` property is set if supplied.
Args:
headers: Message headers to add to the message properties of the
published message (key-value dictionary). The headers can be retrieved in
the consumer from the `headers` attribute of the message's
`BasicProperties`.
Returns:
`pika.spec.BasicProperties` with the `RASA_ENVIRONMENT` environment variable
as the properties' `app_id` value, `delivery_mode`=2 and `headers` as the
properties' headers.
"""
from pika.spec import BasicProperties
# make message persistent
kwargs = {"delivery_mode": 2}
if self.rasa_environment:
kwargs["app_id"] = self.rasa_environment
if headers:
kwargs["headers"] = headers
return BasicProperties(**kwargs)
def on_message(channel, method_frame, header_frame, body):
message_dict = json.loads(body)
url = message_dict["url"]
job_id = message_dict["id"]
level = message_dict["level"] if "level" in message_dict else 0
found_files, found_urls = crawl(url)
if level < MAX_CRAWL:
level = level + 1
update_job(database, job_id, found_files, len(found_urls))
for found_url in found_urls:
messagedata = json.dumps({
"id": job_id,
"level": level,
"url": found_url
})
pub_channel.basic_publish(exchange=EXCHANGE,
routing_key=EXCHANGE,
body=messagedata,
properties=BasicProperties(
content_type="application/json",
delivery_mode=2))
else:
update_job(database, job_id, found_files, 0)
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
print str(datetime.datetime.utcnow()) + "Done with message " + str(
method_frame.delivery_tag)
def _process_message(self,
unused_channel,
basic_deliver,
properties,
body,
handler=None):
try:
result = yield handler(self.logger, body)
self.logger.info("Message has been processed successfully")
if properties is not None and properties.reply_to is not None:
self.logger.info(
f"Sending result back to "
f"queue: {properties.reply_to}, correlation id: {properties.correlation_id}"
)
publish_channel = list(self._publish_channels.values())[0]
yield publish_channel.publish(properties=BasicProperties(
correlation_id=properties.correlation_id),
body=str(result),
mandatory=False,
reply_to=properties.reply_to)
self.logger.info(
f"Sent result back to caller. "
f"Queue: {properties.reply_to}, correlation id: {properties.correlation_id}"
)
except Exception as e:
self.logger.exception("Failed to handle received message.")
raise Exception("Failed to handle received message.")
finally:
unused_channel.basic_ack(basic_deliver.delivery_tag)
def publish(self, message, **kwargs):
"""Publish a message.
:param message: The message to publish
:param kwargs: Additional message properties
:Keyword Arguments:
* *routing_key* --
Routing key to use when publishing
* *headers* --
Headers to be included as part of the message properties
* *expiration* --
Expiration to be included as part of the message properties
:return: None
"""
self._work_queue.put(('basic_publish', {
'exchange':
self._exchange,
'routing_key':
kwargs['routing_key'],
'body':
message,
'properties':
BasicProperties(app_id='beer-garden',
content_type='text/plain',
headers=kwargs.pop('headers', None),
expiration=kwargs.pop('expiration', None))
}))
def publish(self, message):
self._message_number_out += 1
amqp_message_update_meta(message, self.get_meta())
amqp_msg = amqp_message_encode(message)
log.debug("Publish message #%s, AMQP message: %s" %
(self._message_number_out, amqp_msg))
properties = BasicProperties(
app_id=self.app_id,
content_type='application/json',
content_encoding='utf-8',
delivery_mode=2, # persistent
)
try:
yield self._channel.basic_publish(
self.exchange_name,
self.queue_out_routing_key,
amqp_msg,
properties=properties,
)
except ChannelClosed:
self.retry_channel()
self._cached_messages.append(message)
except AMQPError:
self.retry_connect()
self._cached_messages.append(message)
def mark_link(self, id, error=None, http_code="0", url="NULL"):
status = http_code if not http_code == "" else "0"
self.channel.basic_publish(exchange='',
routing_key=self.config.get(
'QUEUES', 'business_saver'),
properties=BasicProperties(delivery_mode=2),
body=json.JSONEncoder().encode({
'parse_status':
status,
'id':
id,
'url':
url,
'phone':
"NULL",
'fax':
"NULL",
'website':
"NULL",
'address_region':
"NULL",
'postal_code':
"NULL",
'street_address':
"NULL",
'category':
"NULL",
}))
self.logger.info('_' * 70)
self.logger.info('[{queue}] Pushed {url}'.format(
url=url, queue=self.config.get('QUEUES', 'business_saver')))
self.logger.info('_' * 70)
def send_message(self, message, queue):
"""
Sends a message to rabbit mq and returns a true or false depending on if it was successful
:param message: The message to send to the rabbit mq queue
:param queue: the name of the queue
:return: a boolean value indicating if it was successful
"""
message_as_string = str(message)
logger.info("sending message to rabbit mq", message=message_as_string)
try:
self._connect()
channel = self.connection.channel()
channel.queue_declare(queue=queue, durable=True)
published = channel.basic_publish(exchange='',
routing_key=queue,
body=message_as_string,
mandatory=True,
properties=BasicProperties(
delivery_mode=2, ))
if published:
logger.info("sent message to rabbit mq")
else:
logger.error("unable to send message to rabbit mq",
message=message_as_string)
return published
except AMQPError as e:
logger.error("unable to send message to rabbit mq",
exc_info=e,
message=message_as_string)
return False
finally:
self._disconnect()
def post():
data = request.get_json()
if 'urls' not in data or len(data['urls']) < 1:
return jsonify({"error": "Payload doesn't include valid data"}), 400
job = {
"id": str(uuid.uuid4()),
"files": [],
"total": len(data['urls']),
"completed": 0
}
database.tasks.insert(job)
for url in data['urls']:
print url
messagedata = json.dumps({"id": job["id"], "level": 0, "url": url})
channel.basic_publish(exchange=EXCHANGE,
routing_key=EXCHANGE,
body=messagedata,
properties=BasicProperties(
content_type="application/json",
delivery_mode=2))
print "Published message for url " + url
return jsonify({"id": job["id"]})
def main():
connection = BlockingConnection(
parameters=URLParameters("amqp://*****:*****@localhost:5672/%2F")
)
props = BasicProperties(
content_type="application/json",
content_encoding="utf8",
delivery_mode=2,
)
ch = connection.channel()
ch.exchange_declare(exchange="worker.mm", exchange_type="topic")
ch.queue_declare(queue="splitter_action_request_q")
ch.queue_bind(
queue="splitter_action_request_q",
exchange="worker.mm",
routing_key="splitter.*.request",
)
ch.basic_publish(
exchange="worker.mm",
routing_key="splitter.split.request",
properties=props,
body=json.dumps(
{
"cache_dir": "/home/xy/misc/mtm/cache/youtube/好葉怎樣練習冥想完整教學 動畫講解-NLJcwbpkiJ0",
"action.type": "split",
"partial_duration": 2 * 60,
}
),
)
def _process_message(self, unused_channel, basic_deliver, properties,
body: ReqPayload):
handler = self.get_handler(body)
try:
result = handler(*body['arg'], **body['kwarg'])
result = self.serialize(result)
self.logger.info(
f"[ {properties.correlation_id} ] Message has been processed successfully"
)
if properties is not None and properties.reply_to is not None:
self.logger.info(
f"[ {properties.reply_to} ][ {properties.correlation_id} ] sending result"
)
publish_channel = list(self.rcv_channels.values())[0]
yield publish_channel.publish(properties=BasicProperties(
correlation_id=properties.correlation_id),
body=result,
mandatory=False,
reply_to=properties.reply_to)
except Exception:
self.logger.exception("Failed to handle received message.")
raise Exception("Failed to handle received message.")
finally:
unused_channel.basic_ack(basic_deliver.delivery_tag)
def publish(self, message, **kwargs):
"""Publish a message.
:param message: The message to publish
:param kwargs: Additional message properties
:Keyword Arguments:
* *routing_key* --
Routing key to use when publishing
* *headers* --
Headers to be included as part of the message properties
* *expiration* --
Expiration to be included as part of the message properties
:return: None
"""
self._work_queue.put((
"basic_publish",
{
"exchange":
self._exchange,
"routing_key":
kwargs["routing_key"],
"body":
message,
"properties":
BasicProperties(
app_id="beer-garden",
content_type="text/plain",
headers=kwargs.pop("headers", None),
expiration=kwargs.pop("expiration", None),
),
},
))
def publish(self, body, exchange, properties=None, mandatory=True):
"""
Publish a message. Creates a brand new channel in the first time, then uses the existing channel onwards.
:param body: message
:param exchange: The exchange to publish to
:param properties: RabbitMQ message properties
:param mandatory: RabbitMQ publish mandatory param
"""
body = self.serialize(body)
self.logger.info("Trying to publish message")
if properties is None:
properties = BasicProperties(delivery_mode=2)
publish_channel = self.pub_channels.get(exchange)
if publish_channel is None:
self.logger.error("There is not publisher for the given exchange")
try:
yield publish_channel.publish(body,
properties=properties,
mandatory=mandatory)
except Exception:
self.logger.exception(f"Failed to publish message")
raise Exception("Failed to publish message")
def publish(self, exchange, routing_key, body, durable=False, ttl=0):
'''
Envia mensagem para servidor.
'''
channel = self.out_channels_exchange.get(exchange, None)
if channel is None:
channel = self.connection.channel()
channel.exchange_declare(exchange=exchange,
type='topic',
durable=durable)
self.out_channels_exchange[exchange] = channel #mantem cache
params = {}
if durable:
params['delivery_mode'] = 2 # make message persistent
if ttl:
params['expiration'] = str(ttl)
if params:
properties = BasicProperties(**params)
else:
properties = None
channel.basic_publish(exchange=exchange,
routing_key=routing_key,
body=body,
properties=properties)
def add_snapshot(user_id, snapshot_id):
snapshot_path = Path("users") / str(
user_id) / "snapshots" / str(snapshot_id)
Path(snapshot_path).mkdir(parents=True, exist_ok=True)
snap_file = request.files["file"].filename
with open(snap_file, "rb") as f:
snap = parse_from(f.read())
snap_serial_dic = snapshot_to_dict(self.parsers,
str(snapshot_path),
snap)
if not self.setup_publisher(
snap_serial_dic,
BasicProperties(headers={
"snapshot_id": snapshot_id,
"user_id": user_id
},
message_id="snap_" + str(snapshot_id) +
"_" + str(user_id))):
data = {
"error":
"no publisher and no message queue url were supplied."
}
# headers = {"Content-Type": "application/json"}
return data
return ""
def rpc(self, body, receive_queue, publish_exchange, timeout, ttl):
receive_channel = self.rcv_channels.get(receive_queue)
if receive_channel is None:
self.logger.error("There is not receiver for the given queue")
self.logger.info(
f"Preparing to rpc call. Publish exchange: {publish_exchange}; Receive queue: {receive_queue}"
)
yield receive_channel.consume(self._rpc_callback_process)
correlation_id = str(uuid.uuid1())
self.logger.info(
f"Starting rpc calling correlation id: {correlation_id}")
if correlation_id in self._rpc_corr_id_dict:
self.logger.warning(
f"Correlation id exists before calling. {correlation_id}")
del self._rpc_corr_id_dict[correlation_id]
self._rpc_corr_id_dict[correlation_id] = Future()
properties = BasicProperties(correlation_id=correlation_id,
reply_to=receive_queue,
expiration=str(ttl * 1000))
yield self.publish(body,
publish_exchange,
properties=properties,
mandatory=True)
self.logger.info(f"RPC message has been sent. {correlation_id}")
result = yield self._wait_result(correlation_id, timeout)
if correlation_id in self._rpc_corr_id_dict:
del self._rpc_corr_id_dict[correlation_id]
self.logger.info(f"RPC message gets response. {correlation_id}")
return result
def post_message_to_queue(case_id, qid, line_number):
properties = BasicProperties(content_type='application/json',
delivery_mode=PERSISTENT_DELIVERY_MODE)
message = {
"event": {
"type": "QUESTIONNAIRE_LINKED",
"source": "RM",
"channel": "RM",
"dateTime": datetime.utcnow().isoformat() + 'Z',
"transactionId": str(uuid.uuid4())
},
"payload": {
"uac": {
"questionnaireId": qid,
"caseId": case_id,
}
}
}
with RabbitContext() as rabbit:
rabbit.channel.basic_publish(exchange='events',
routing_key='event.questionnaire.update',
body=json.dumps(message),
properties=properties,
mandatory=True)
print(
f"Success: Line {line_number}: Case ID {case_id} and QID {qid} have PASSED and have been LINKED"
)
def send_msg_example(queue: str, message: str):
"""Пример отправки сообщения в очередь
:param queue: название очереди
:param message: тело сообщения
"""
# подключаемся к серверу
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
# проверяем, что очередь сущетсвует, или создаем новую
# method = channel.queue_declare('') создаст временную очередь со случайным именем
channel.queue_declare(
queue=queue, # название
durable=True, # объявить устойчивой
)
# необязательно: создаём обменник и связываем с очередью
# channel.exchange_declare('logs', exchange_type='fanout')
# channel.queue_bind(method.method.queue, 'logs')
# с типом fanout при отправке сообщения routing_key можно не указывать
# отправляем сообщение
channel.basic_publish(
exchange='', # точка обмена
routing_key=queue, # имя очереди
body=message, # сообщение
properties=BasicProperties(
delivery_mode=2, # объявить устойчивым
)
)
connection.close()
def on_queue_declared(frame):
print "demo_send: Queue Declared"
for x in xrange(0, 2):
# Construct request message with json format
msg = {}
msg['cust_no'] = "6223000000000000%04i" % (x % 10)
msg['trans_date'] = "201205%02i" % (x % 7)
msg['trans_time'] = "0919%02i" % (x % 60)
msg['flow_no'] = "%015i" % x
msg['trans_type'] = "%02i" % (x % 2)
msg['trans_amt'] = x * 9.9
msg['trans_loc'] = "trans location:%02i" % x
jsonmsg = json.dumps(msg, indent=2)
# Create properties with when we sent the message, the app_id
# user we connected with, a content type and non persisted messages
properties = BasicProperties(timestamp=time.time(),
app_id=__file__,
user_id='guest',
content_type="text/plain",
delivery_mode=1)
# Send the message
channel.basic_publish(exchange='',
routing_key="krqueue",
body=jsonmsg,
properties=properties)
print "demo_send:" + jsonmsg
# Close our connection
print "Closing client"
connection.close()
def send(self, queue_name, body):
msg('Publishing %s body to "%s" on %s' % (len(body), queue_name, self))
properties = BasicProperties(delivery_mode=1)
@defer.inlineCallbacks
def on_declare(queue):
msg("Queue %s declared" % queue_name)
def on_publish_failed(result):
channel, response, props, body = result
msg("Publish failed %s" % response)
self.channel.add_on_return_callback(on_publish_failed)
yield self.channel.basic_publish(exchange='',
body=body,
routing_key=queue_name,
properties=properties,
mandatory=True)
def on_declare_fail(error):
msg('Can not declare queue %s' % error)
d = self.channel.queue_declare(queue=queue_name,
auto_delete=False,
exclusive=False)
d.addCallbacks(on_declare, on_declare_fail)
return d
def _sync_request(self, timeout: int = 3) -> bool:
self._sync_response = None
self.sync_id = str(uuid4())
self._channel.basic_publish(exchange=self.exchange_name,
routing_key=bdc.SYNC_REQ,
properties=BasicProperties(
reply_to=bdc.SYNC_RES,
message_id=self.user_id,
correlation_id=self.sync_id,
),
body=str())
time_sent = time()
# while no response and no timeout
while self._sync_response is None and (time() - time_sent) < timeout:
self._connection.process_data_events()
# if timeout (i.e. no response), assume there are no other clients
if self._sync_response is None:
return False
self._put_current_state(json.loads(self._sync_response))
return True
def publish_impl(self,
exchange,
routing_key,
body,
properties,
immediate=False,
mandatory=False,
durable_msg=False):
"""
Publishes a message on an exchange.
"""
log.debug("AMQPTransport.publish(%s): ex %s key %s",
self._client.channel_number, exchange, routing_key)
if durable_msg:
delivery_mode = 2
else:
delivery_mode = None
props = BasicProperties(headers=properties,
delivery_mode=delivery_mode)
self._client.basic_publish(
exchange=exchange, # todo
routing_key=routing_key, # todo
body=body,
properties=props,
immediate=immediate, # todo
mandatory=mandatory) # todo
def _process_message(self,
unused_channel,
basic_deliver,
properties,
body,
exchange,
handler=None,
return_callback=None,
close_callback=None):
try:
result = yield handler(body)
self.logger.info("message has been processed successfully")
if properties is not None \
and properties.reply_to is not None:
self.logger.info(
"sending result back to queue: %s, correlation id: %s",
properties.reply_to, properties.correlation_id)
yield self.publish(
exchange=exchange,
routing_key=properties.reply_to,
properties=BasicProperties(
correlation_id=properties.correlation_id),
body=str(result),
mandatory=False,
return_callback=return_callback,
close_callback=close_callback)
self.logger.info(
"sent result back to caller. queue : %s, correlation id: %s",
properties.reply_to, properties.correlation_id)
except Exception:
import traceback
self.logger.error(traceback.format_exc())
raise RabbitMQReceiveError("failed to handle received message.")
finally:
unused_channel.basic_ack(basic_deliver.delivery_tag)
