def publish(self, exchange_name, exchange_key, message, mheaders, mexp=0): ebo=2 connected=True while True: if 'v03.' in exchange_key: ct='application/json' else: ct='text/plain' try: if self.hc.use_amqp: self.logger.debug("publish AMQP is used") if mexp: expms = '%s' % mexp msg = amqp.Message(message, content_type=ct, application_headers=mheaders, expiration=expms) else: msg = amqp.Message(message, content_type=ct, application_headers=mheaders) self.channel.basic_publish(msg, exchange_name, exchange_key) self.channel.tx_commit() elif self.hc.use_amqplib: self.logger.debug("publish AMQPLIB is used") if mexp: expms = '%s' % mexp msg = amqplib_0_8.Message(message, content_type=ct, application_headers=mheaders, expiration=expms) else: msg = amqplib_0_8.Message(message, content_type=ct, application_headers=mheaders) self.channel.basic_publish(msg, exchange_name, exchange_key) self.channel.tx_commit() elif self.hc.use_pika: self.logger.debug("publish PIKA is used") if mexp: expms = '%s' % mexp properties = pika.BasicProperties(content_type=ct, delivery_mode=1, headers=mheaders, expiration=expms) else: properties = pika.BasicProperties(content_type=ct, delivery_mode=1, headers=mheaders) self.channel.basic_publish(exchange_name, exchange_key, message, properties, True) else: self.logger.debug("Couldn't choose an AMQP client library, setting it back to default amqp") self.hc.use_amqp = True raise ConnectionError("No AMQP client library is set") return True except Exception as err: if ebo < 65: ebo = ebo * 2 self.logger.error("sr_amqp/publish: Sleeping %d seconds ... and reconnecting" % ebo) self.logger.debug('Exception details: ', exc_info=True) time.sleep(ebo) connected=False if not connected: self.hc.reconnect()
def amqp_send_msg(self, msg='', routing_key=''): """Publish AMQP message""" if not isinstance(msg, amqp.Message): if isinstance(msg, basestring): msg = amqp.Message(msg) else: msg = amqp.Message(json.dumps(msg), content_type='application/json') log.debug("%s: Sending AMQP msg with routing key '%s' and body %r.", self.lbl, routing_key, msg.body) self.chan.basic_publish(msg, self.exchange, routing_key)
def post(self, payload): if self.is_exchange: message = amqp.Message(payload) while True: try: self.channel.basic_publish(msg=message, exchange=self.queue_name) break except Exception as e: logger.error('Queue post to Exchange: %s' % e) self.channel = self.queue_connection.re_channel( self.channel) else: while True: try: if not self.simple_queue: self.simple_queue = self.queue_connection._connection.SimpleQueue( self.queue_name) self.simple_queue.put(payload) break except Exception as e: logger.error('Queue put: %s' % e) self.queue_connection.reconnect() if self.simple_queue: self.simple_queue.close()
def _send_reply( self, reply_to: str, channel: amqp.Channel, result: Any, exc_info: ExcInfoType, ) -> None: logger.debug("Sending reply result=%r", result) reply = {'result': result} if exc_info: reply['exception'] = self._exc_info_dict(exc_info) try: body = json.dumps(reply) except Exception as e: logger.error('Cannot serialize result as JSON: %s', e) exc_info = sys.exc_info() reply = { 'result': None, 'exception': self._exc_info_dict(exc_info) } body = json.dumps(reply) msg = amqp.Message(body=body) channel.basic_publish(msg, exchange="", routing_key=reply_to)
def call(self, md): self.corr_id = str(uuid.uuid4()) message = amqp.Message(body=md, correlation_id=self.corr_id, reply_to=self.callback_queue) self.channel.basic_publish(message, exchange='', routing_key=queue_name) self.connection.drain_events() return self.response
def _send_to_queue(self, args, kwargs, host=None, local=False): """ Sends this task to queue. :param args: Arguments that will be passed to task on execution. :param kwargs: Keyword arguments that will be passed to task on execution. :param host: Send this task to specific host. ``host`` will be appended to the queue name. :param local: Send this task to this host. Hostname of current host will be appended to the queue name. :return: :const:`None` """ logger.debug("Task.send_to_queue args=%r, kwargs=%r", args, kwargs) queue = get_queue_name(self.queue, host=host, local=local or self.local) description = self._get_description(args, kwargs, queue) self._send_signal(signals.task_presend, args=args, kwargs=kwargs, description=description) body = json.dumps(description) msg = amqp.Message(body=body) with self.kuyruk.channel() as ch: ch.queue_declare(queue=queue, durable=True, auto_delete=False) ch.basic_publish(msg, exchange="", routing_key=queue) self._send_signal(signals.task_postsend, args=args, kwargs=kwargs, description=description)
def submit_job(app_id, app_key, task_id, mimetype, text_col, dedupe): """Submit a job to the queue for the Celery worker. Create the required JSON message and post it to RabbitMQ.""" # These are the args that the Python function in the adjunct processor will use. kwargs = { "app_id": app_id, "app_key": app_key, "task_id": task_id, "format": mimetype, "text_col": text_col, "dedupe": dedupe, "s3_endpoint": S3ENDPOINT, "bucket": BUCKET, "redis_port": REDISPORT, "redis_host": REDISHOST } # Recreate a celery message manually so that we don't need to import celery_tasks.py which has heavy dependencies. job = { "id": task_id, # "task": "synapsify_adjunct.celery_tasks.synapsify_master", "task": "dc2_master", "kwargs": kwargs } # Connect to RabbitMQ and post. conn = amqp.Connection(host=RMQHOST, port=RMQPORT, userid=RMQUSERNAME, password=RMQPASSWORD, virtual_host=RMQVHOST, insist=False) cha = conn.channel() msg = amqp.Message(json.dumps(job)) msg.properties["content_type"] = "application/json" cha.basic_publish(routing_key=RMQEXCHANGE, msg=msg) cha.close() conn.close()
def publish_messages(): with amqp.Connection('localhost', userid="iotile", password="******", confirm_publish=True) as c: ch = c.channel() with open("/opt/src/messages.txt", "r") as input_messages: line = input_messages.readline() while len(line) > 0: signal, site, machine, value = line.split(",") body = {"value": float(value)} jbody = json.dumps(body) message = amqp.Message(body=jbody, application_headers={ "timestamp_in_ms": int(time.time() * 1000) }) ch.basic_publish( message, exchange="amq.topic", routing_key=f'raw.arch.{site}.{machine}.{signal}') time.sleep(0.1) line = input_messages.readline()
def amqp_consume(self): """Connect to Hub Server and set up and start AMQP consumer""" # define callback queue self.queue = self.chan.queue_declare(exclusive=True).queue self.chan.queue_bind(self.queue, self.exchange, self.queue) self.chan.basic_consume(self.queue, callback=self.amqp_handle_msg, no_ack=True) log.debug("%s: Initialized amqp connection, channel, queue.", self.lbl) # send rpc request self.worker_id = None self.correlation_id = uuid.uuid4().hex reply_to = self.queue routing_key = '%s.worker.%s' % (self.key, self.worker_type) msg = amqp.Message(json.dumps(self.worker_kwargs), correlation_id=self.correlation_id, reply_to=reply_to, content_type='application/json') self.amqp_send_msg(msg, routing_key) log.info("%s: sent RPC request, will wait for response.", self.lbl) # wait for rpc response try: while not self.worker_id: log.debug("%s: Waiting for RPC response.", self.lbl) self.chan.wait() except BaseException as exc: log.error( "%s: Amqp consumer received %r while waiting for RPC " "response. Stopping.", self.lbl, exc) log.info("%s: Finished waiting for RPC response.", self.lbl) super(HubClient, self).amqp_consume()
def _send(self, command, payload=None): # send rpc request if self.correlation_id: raise Exception("Can't send second request while already waiting.") self.response = None self.correlation_id = uuid.uuid4().hex routing_key = '%s.%s' % (self.key, command) msg = amqp.Message(json.dumps(payload), correlation_id=self.correlation_id, reply_to=self.queue, content_type='application/json') log.debug("Sending AMQP msg with routing key '%s' and body %r.", routing_key, msg.body) self.chan.basic_publish(msg, self.exchange, routing_key) log.info("Sent RPC request, will wait for response.") # wait for rpc response try: while self.correlation_id: log.debug("Waiting for RPC response.") self.chan.wait() except BaseException as exc: log.error( "Amqp consumer received %r while waiting for RPC " "response. Stopping.", exc) log.info("Finished waiting for RPC response.") response = self.response self.response = None return response
def Publish(self, object): channel = self.Connect_AMQP_UserPass() channel.basic_publish(amqp.Message(body=json.dumps(object.EntityJSON)), exchange=self.args.exchange or object.DocumentType.encode("utf-8"), routing_key=self.args.about or object.ResourceID.encode("utf-8"))
def send_rpc(self, dest, method, params, block=True, timeout=None): if len(self._connections) == 0: raise RuntimeError('Not connected') id_ = str(uuid.uuid1()) body = { 'reply_to': self.rpc_out_queue.queue, 'method': method, 'id': id_, 'params': params } LOG.debug('sending %s to %s', body, dest + ':rpc') msg = amqp.Message(body=json.dumps(body), reply_to=self.rpc_out_queue.queue) self.active_rpcs[id_] = gevent.event.AsyncResult() self.rpc_out_channel.basic_publish(msg, routing_key=dest + ':rpc') try: result = self.active_rpcs[id_].get(block=block, timeout=timeout) except gevent.timeout.Timeout: self.active_rpcs.pop(id_) raise return result
def __call__(self, report): # Don't mess with the passed in report. report = dict(report) if not self.inherit_id or not report.get('id'): # Discard any existing id. original_id = report.pop('id', None) # Hash it, to make an ID oops_id = "OOPS-%s" % md5(dumps(report)).hexdigest() # Store the id in what we send on the wire, so that the recipient # has it. report['id'] = oops_id message = amqp.Message(dumps(report)) # We don't want to drop OOPS on the floor if rabbit is restarted. message.properties["delivery_mode"] = 2 channel = self.get_channel() if channel is None: return [] try: channel.basic_publish( message, self.exchange_name, routing_key=self.routing_key) except amqplib_error_types as e: self.channels.channel = None if is_amqplib_connection_error(e): # Could not connect / interrupted connection return [] # Unknown error mode : don't hide it. raise return [report['id']]
def test_publish_get(self): self.channel.queue_declare(queue='py-amqp-unittest', durable=False, exclusive=True) self.channel.basic_publish(amqp.Message('Unittest'), routing_key='py-amqp-unittest') msg = self.channel.basic_get(queue='py-amqp-unittest', ) assert msg.body_size == 8 assert msg.body == 'Unittest' assert msg.frame_method == amqp.spec.Basic.GetOk assert msg.delivery_tag == 1 assert msg.ready is True assert msg.delivery_info == { 'delivery_tag': 1, 'redelivered': False, 'exchange': '', 'routing_key': 'py-amqp-unittest', 'message_count': 0 } assert msg.properties == {'content_encoding': 'utf-8'} self.channel.basic_ack(msg.delivery_tag) msg = self.channel.basic_get(queue='py-amqp-unittest', ) assert msg is None
def _build_message_body(self, message): """ Build the AMQP message based on the input data and the client's config. :param ProviderEventsUpdateMessage message: Data for message body :returns: Message with configuration :rtype: amqp.Message """ message_str = json.dumps(message, default=self._serialize_models) if logging.getLogger().getEffectiveLevel() == logging.DEBUG: txt = json.dumps(json.loads(message_str), indent=4, separators=(',', ': ')) __log__.debug("Message: \n" + txt) return amqp.Message( body=message_str, content_type=self.__amqp_config__["content_type"], content_encoding=self.__amqp_config__["content_encoding"], application_headers={ "__TypeId__": "com.sios.stc.model.messaging." + type(message).__name__ })
def send_tasks_to_queue(self, subtasks): if self.config.EAGER: for subtask in subtasks: subtask.task.apply(*subtask.args, **subtask.kwargs) return declared_queues = set() with self.channel() as ch: for subtask in subtasks: queue = subtask.task._queue_for_host(subtask.host) if queue not in declared_queues: ch.queue_declare(queue=queue, durable=True, auto_delete=False) declared_queues.add(queue) description = subtask.task._get_description( subtask.args, subtask.kwargs) subtask.task._send_signal(signals.task_presend, args=subtask.args, kwargs=subtask.kwargs, description=description) body = json.dumps(description) msg = amqp.Message(body=body) ch.basic_publish(msg, exchange="", routing_key=queue) subtask.task._send_signal(signals.task_postsend, args=subtask.args, kwargs=subtask.kwargs, description=description)
def publish(self, msg, topic, **kwargs): exchange_name = topic.split('.')[0] routing_key = topic content_type = 'text/plain' if isinstance(msg, (list, dict)): msg = json.dumps(msg) content_type = 'application/json' message = amqp.Message(msg, content_type=content_type, delivery_mode=1, **kwargs) num = len(self.pub_conncluster.all_connection_pools()) for i in range(num): try: connection = self.pub_conncluster.get_connection() if not connection: continue except Exception as e: logger.error(e) continue try: return connection.publish(message, exchange_name, routing_key) except socket.error as e: logger.error(e) self.pub_conncluster.connection_error(connection) except ConnectionError as e: logger.error(e) self.pub_conncluster.connection_error(connection) except Exception as e: logger.error(e) connection.disconnect() finally: self.pub_conncluster.release(connection) logger.error('PUB FAIL', {'topic': topic, 'msg': msg})
def apply_async(self, args, kwargs=None, correlation_id=None, routing_key=None, send_result=False, result_queue=None): if correlation_id is None: correlation_id = str(uuid4()) job = {'task': self.name, 'args': args, 'kwargs': kwargs} job_raw = json.dumps(job) reply_to = result_queue or 'result_{0}'.format(correlation_id) result = AsyncResult(reply_to) msg = amqp.Message(job_raw, content_type='application/json', reply_to=reply_to, correlation_id=correlation_id) if routing_key is None: routing_key = self.routing_key with AmqpConnection() as channel: if send_result and result_queue is None: channel.queue_declare(queue=reply_to, auto_delete=False) channel.basic_publish(msg, exchange=self.exchange, routing_key=routing_key) logger.info('Task queued: name=%s, id=%s', self.name, correlation_id) logger.debug('Task details: args=%s, kwargs=%s', repr(args), repr(kwargs)) if send_result: return result
def log(self, severity, msg): index_data = json.dumps({ 'index': { '_index': settings.ES['index'], '_type': self.log_name, } }) msg = json.dumps({ self.log_name: { 'severity': severity.upper(), 'msg': msg, 'timestamp': datetime.datetime.now(tz=UTC).isoformat(), }, }) # TODO: this should be some generic method so that ES/RabbitMQ can # be replaced later with something else bulk_msg = '\n'.join([index_data, msg, '']) # trailing newline added print bulk_msg self.amqp_channel.basic_publish( amqp.Message(bulk_msg), exchange=self.amqp_exchange_name)
def send(self, message): msg = amqp.Message( message.json(), content_type='application/json', ) self._channel.basic_publish(msg, routing_key=self._queue)
def publish(self, message_data): encoded = json.dumps({'data': message_data}) message = amqp.Message(encoded) message.properties['delivery_mode'] = self.delivery_mode self.channel.basic_publish(message, exchange=self.exchange, routing_key=self.routing_key) return message
def main(): parser = OptionParser( usage='usage: %prog [options] message\nexample: %prog hello world', ) parser.add_option( '--host', dest='host', help='AMQP server to connect to (default: %default)', default='localhost', ) parser.add_option( '-u', '--userid', dest='userid', help='userid to authenticate as (default: %default)', default='guest', ) parser.add_option( '-p', '--password', dest='password', help='password to authenticate with (default: %default)', default='guest', ) parser.add_option( '--ssl', dest='ssl', action='store_true', help='Enable SSL (default: not enabled)', default=False, ) options, args = parser.parse_args() if not args: parser.print_help() sys.exit(1) msg_body = ' '.join(args) conn = amqp.Connection(options.host, userid=options.userid, password=options.password, ssl=options.ssl) ch = conn.channel() ch.exchange_declare('myfan', 'fanout') msg = amqp.Message(msg_body, content_type='text/plain', application_headers={ 'foo': 7, 'bar': 'baz' }) ch.basic_publish(msg, 'myfan') ch.close() conn.close()
def prepare_message(self, body, priority=None, content_type=None, content_encoding=None, headers=None, properties=None): """Encapsulate data into a AMQP message.""" return amqp.Message(body, priority=priority, content_type=content_type, content_encoding=content_encoding, application_headers=headers, **properties)
def emiter_loop(*, url): conn = yield from make_rabbitmq_connection(url=url) chan = yield from make_rabbitmq_channel(conn) for x in range(10000): print("Sleep...") yield from asyncio.sleep(1) print("Publish: {}".format(x)) chan.basic_publish(amqp.Message("{}".format(x)), "events")
def test_invalid_json(self): """Message is dropped when JSON is not valid""" with run_worker() as worker: worker.expect('Consumer started') with new_instance().channel() as ch: msg = amqp.Message(body='foo') ch.basic_publish(msg, exchange='', routing_key='kuyruk') worker.expect('Cannot decode message') self.assertEqual(len_queue('kuyruk'), 0)
def mq_send(self,**kwargs): "" msg = kwargs.pop('msg', None) exchange_name = kwargs.pop('exchange_name', None) routing_key = kwargs.pop('routing_key', None) message = amqp.Message(str(msg)) self.chan.basic_publish(message,exchange=exchange_name,routing_key=routing_key)
def call(self, args): self.corr_id = str(uuid.uuid1()) message = amqp.Message(body=args, correlation_id=self.corr_id, reply_to=self.callback_queue) self.channel.basic_publish(message, exchange='', routing_key=MarkdownRpcClient.QUEUE_NAME, timeout=TIMEOUT) self.connection.drain_events(timeout=TIMEOUT) return self.response
def put_nowait(self, obj): if self.lazy_limit and self.qsize_diff < self.qsize_diff_limit: pass elif self.full(): raise BaseQueue.Full else: self.qsize_diff = 0 with self.lock: self.qsize_diff += 1 msg = amqp.Message(umsgpack.packb(obj)) return self.channel.basic_publish(msg, exchange="", routing_key=self.name)
def call(self, n): self.response = None self.corr_id = str(uuid.uuid4()) self.channel.basic_publish(amqp.Message(str(n), correlation_id=self.corr_id, reply_to=self.callback_queue), exchange='', routing_key='rpc_queue') while self.response is None: self.channel.wait() return int(self.response)
def test_invalid_task_path(self): """Message is dropped when task cannot be imported""" with run_worker() as worker: worker.expect('Consumer started') with new_instance().channel() as ch: desc = {'module': 'kuyruk', 'function': 'foo'} body = json.dumps(desc) msg = amqp.Message(body=body) ch.basic_publish(msg, exchange='', routing_key='kuyruk') worker.expect('Cannot import task') self.assertEqual(len_queue('kuyruk'), 0)