async def _rpc_handle(self, message: aio_pika.IncomingMessage) -> None: """ Handles JSON-RPC request. :param message: incoming message """ try: reply_to = message.reply_to response_text = await self._dispatcher.dispatch(message.body, context=message) if response_text is not None: if reply_to is None: logger.warning("property 'reply_to' is missing") else: async with self._connection.channel() as channel: await channel.default_exchange.publish( aio_pika.Message( body=response_text.encode(), reply_to=reply_to, correlation_id=message.correlation_id, content_type=pjrpc.common.DEFAULT_CONTENT_TYPE, ), routing_key=reply_to, ) message.ack() except Exception as e: logger.exception("jsonrpc request handling error: %s", e)
async def handle_message(self, message: IncomingMessage) -> None: message.ack() correlation_id = message.correlation_id future = self._reply_futures.pop(correlation_id, None) if future is not None: try: try: accept = self.accept = prepare_accept_content( self.cluster_proxy.accept) body = loads(message.body, content_type=message.content_type, content_encoding=message.content_encoding, accept=accept) except Exception as e: future.set_exception(e) else: future.set_result(body) except asyncio.InvalidStateError as e: # for catching the errors after the future obj was cancelled by the asyncio.wait_for timeout. sys.stdout.write("{}, correlation id: {}".format( e, correlation_id)) else: sys.stdout.write( "Unknown correlation id: {}".format(correlation_id))
async def _process_message(self, message: IncomingMessage, registered_obj: RegisteredCoroOrGen, **callable_kwargs): logger.debug(f"Processing message {message.delivery_tag}") try: if inspect.iscoroutinefunction(registered_obj.coro_or_gen): # it returns return_val = await registered_obj.coro_or_gen(**callable_kwargs ) if return_val is not None: await self._handle_return_value(message, return_val) else: # it yields async for v in registered_obj.coro_or_gen(**callable_kwargs): if v is not None: await self._handle_return_value(message, v) # At this point, we're processed the message and sent along new ones successfully. We can ack the original message # TODO consider doing all of this transactionally logger.debug(f"Acking message {message.delivery_tag}") message.ack() except asyncio.CancelledError: # We were told to cancel. nack with requeue so someone else will pick up the work message.nack(requeue=True) logger.info("Cancellation requested.") raise except Exception as e_: await self._handle_message_exception(message, e_)
async def consumer_func(message: aio_pika.IncomingMessage): json_body = json.loads(message.body.decode('utf-8')) task = loop.run_in_executor(executor, check, json_body['ip'], json_body['port']) res, _ = await asyncio.wait([task]) if list(res)[0].result(): message.delivery_mode = 2 await pika_channel.default_exchange.publish(message, routing_key=os.environ['RABBIT_MOTD_QUEUE']) message.ack()
def on_message(message: IncomingMessage): try: data_dict = json.loads(message.body.decode()) # check if the packet={'key':[rg]<String>,'path_param':[_SHA1_attr/d]<String>,'data':[adapter packet]<JSON>} from the queue is empty if data_dict is None or not bool(data_dict): print('> data_dict is empty. data_dict---- ' + str(data_dict)) return 'failed' else: # print('> redis_key: ' + data_dict['key'] + '\nredis_path_param: ' + data_dict['path_param'] + '\nadapter_data_packet: ' + data_dict['data']) # this needs to be tested exp_val = data_dict['data']['id'] + '_' + data_dict['data'][ 'observationDateTime'] except json.decoder.JSONDecodeError as json_error: print('> JsonDecodeError!!!!' + str(json_error)) return 'failed' try: # check if redis already has existing data? chck_if_rg_exists = redis_client.jsonget(data_dict['key'], Path.rootPath()) # (nil) - never seen packet -> None, Insert # {}, !exists # data, upsert # redis already has previous data if chck_if_rg_exists is not None: print('> ' + str(data_dict['key']) + 'exists.') print('> Upserting ' + str(data_dict['data']) + ' at .' + data_dict['path_param']) print('> Upsertion still in progress...') redis_client.jsonset( data_dict['key'], '.' + data_dict['key'] + '.' + data_dict['path_param'], data_dict['data']) print('> Upsertion successful!') message.ack() # First time the ingestor receives a packet belonging to RG else: print('> RG=' + data_dict['key'] + ' is not present in Redis. Inserting RG with {} at root.') # create first entry in the redis server # origin = {rg: {SHA : {}}} origin = {data_dict['key']: {data_dict['path_param']: {}}} redis_client.jsonset(data_dict['key'], Path.rootPath(), origin) print('> Insertion still in progress...') # insert data now # JSON.GET resource-group-key/redis-key .path_param (SHA1...) # JSON.GET resource-group .resource-group.SHA1.... {adapter_data} redis_client.jsonset( data_dict['key'], '.' + data_dict['key'] + '.' + data_dict['path_param'], data_dict['data']) print('> Insertion successful!') message.ack() except redis.exceptions.ResponseError as r_error: print('> Response Error from Redis!!!! ' + str(r_error)) return 'failed'
async def _on_message(self, message: aio_pika.IncomingMessage): logging.debug(f'Received message {str(message.body)}.') if getattr(self, 'custom_consume_function', None): if not await self.custom_consume_function(**self.custom_consume_function_kwargs): self.logger.error('Failed process message.') message.nack() return self.logger.info('Message has been successfully processed.') message.ack()
def __call__(self, message: IncomingMessage): value = int(message.body.decode()) if value == 3 and self.rejected < 3: print('[blue]Nacking[/blue] 3') message.nack() self.rejected += 1 elif value == 5: print('[red]Rejecting[/red] 5') message.reject() else: print(f'[green]Processed[/green] {value}') message.ack()
async def on_message(message: IncomingMessage): print(message.body) body = loads(message.body) event = body['event'] #print('# {} [received]'.format(event)) print('# {} [received] sleep 1 sec..'.format(event)) await asyncio.sleep(1) # Represents async I/O operations del body['event'] await dispatch(event, body) # manual ack here # not with message.process() context, which auto ack even when error message.ack()
async def orderValidation(self, msg: IncomingMessage): startTime = time.time() self.app['EVENT_PROGRESS'].labels('ORDER').inc() data = msg.body.decode() now = datetime.now() now = now.strftime('%Y-%m-%d %H:%M:%S') data = data.split('|') account = data[0] action = data[1] order = data[2].split('.') order = { 'orderId': order[3], 'timestamp': now, 'account': account, 'action': action, 'stock': order[0], 'price': int(order[1]), 'vol': int(order[2]), 'cumVol': 0, 'status': '0', 'selected': False } params = { 'account': account if action == 'BUY' else f"{account}.{order['stock']}", 'amount': order['price'] * order['vol'] if action == 'BUY' else order['vol'] } # Check account availability to account services API async with self.webservice.request('POST', 'http://localhost:8001/order', json=params) as resp: valid = await resp.text() if valid == 'OK': self.app['EVENT_COUNTER'].labels('ORDER', 'SUCCESS').inc() await self.orderCollection.insert_one(order) order['eventLatency'] = self.app['EVENT_LATENCY'] order['eventProgress'] = self.app['EVENT_PROGRESS'] order['orderCounter'] = self.app['ORDER_COUNTER'] order['eventTime'] = startTime self.messages.on_next( (msg, order)) # Send valid order to order observable else: self.app['EVENT_COUNTER'].labels('ORDER', 'FAIL').inc() self.app['EVENT_PROGRESS'].labels('ORDER').dec() order['status'] = 'R' await self.orderCollection.insert_one(order) msg.ack() latency = time.time() - startTime self.app['EVENT_LATENCY'].labels('ORDER', 'FAIL').observe(latency)
async def on_message(message: IncomingMessage): print('> *****Inside key_generator*****') global data_redis_q, rg_dict data_redis_q = {} # data_dict is actual adapter packet data_dict = json.loads(message.body.decode()) # default value is for all aqm/flood type sensors default = '_d' # extract resource-group from the data packet res_id = data_dict['id'] rg = res_id.split('/')[3] # generate the SHA1 of the id sha_id = hashlib.sha1(res_id.encode()) # print("> RG from data---- " + rg) # print("> rg_dict---- " + str(rg_dict)) # print("> SHA1[res_id]---- " + sha_id.hexdigest()) # Check if _rg is present in rg_dict{} if rg in rg_dict.keys(): print('> RG is present.') # encode SHA1 of resource-id attribute = rg_dict[rg] path_param = '_' + sha_id.hexdigest() + '_' + data_dict[attribute] else: print('> RG is not present.') path_param = '_' + sha_id.hexdigest() + '_' + default # generate a dict = { 'key' : <resource-group-name>, 'path_param': <_SHA1(id)_attr/d>, 'data': adapter packet } data_redis_q['key'] = rg.replace('-', '_') data_redis_q['path_param'] = path_param data_redis_q['data'] = data_dict # print('> (on_message) routing_key---- '+routing_key) # publish the data into redis-ingestion-queue print('> bool(data_redis_q):---- ' + str(bool(data_redis_q))) if data_redis_q is not None and bool(data_redis_q): await latest_exchange.publish(message=Message( (json.dumps(data_redis_q)).encode()), routing_key=routing_key) print('> Message published.') # message.ack() else: # do nothing pass message.ack()
async def handle_msg(msg: IncomingMessage): print(f"dcmq: got message with routing key {msg.routing_key}") ds = datasetFromBinary(msg.body) uri = "" if "uri" in msg.headers: uri = msg.headers["uri"] if ds != None: try: await dcmhandler(channel, ds, uri, msg.routing_key, *additional_args) msg.ack() except Exception as e: msg.reject(requeue=True) raise(e)
def __init__( self, message: apika.IncomingMessage, log: Optional[logging.Logger] = None, ): self.command_id: str | None = None self.sender: str | None = None self.body = {} self.message = message self._log = log self.is_valid = True # Acknowledges receipt of message message.ack() self.info: Dict[Any, Any] = message.info() self.headers = self.info["headers"] for key in self.headers: if isinstance(self.headers[key], bytes): self.headers[key] = self.headers[key].decode() self.message_code = self.headers.get("message_code", None) if self.message_code is None: self.is_valid = False if self._log: self._log.warning( f"received message without message_code: {message}") return self.sender = self.headers.get("sender", None) if self.sender is None and self._log: self._log.warning(f"received message without sender: {message}") self.command_id = message.correlation_id command_id_header = self.headers.get("command_id", None) if command_id_header and command_id_header != self.command_id: if self._log: self._log.error(f"mismatch between message " f"correlation_id={self.command_id} " f"and header command_id={command_id_header} " f"in message {message}") self.is_valid = False return self.body = json.loads(self.message.body.decode())
async def on_rabbitmq_message(self, message: aio_pika.IncomingMessage): self.concurrent_requests += 1 if self.concurrent_requests >= self.max_concurrent_requests: self.logger.warning('TOO MANY CONCURRENT REQUESTS... requeuing') message.reject(requeue=True) self.concurrent_requests -= 1 return if not self.status.is_active(): message.reject(requeue=True) self.concurrent_requests -= 1 return callback_url = self.get_callback_for_message(message) if not callback_url: self.logger.error('WTF? unknown routing key: {}'.format( message.routing_key)) message.reject(requeue=True) self.concurrent_requests -= 1 return try: msg = json.dumps({ 'headers': message.headers, 'content_encoding': message.content_encoding, 'message_id': message.message_id, 'type': message.type, 'routing_key': message.routing_key, 'body': message.body.decode(message.content_encoding or 'utf8') }) await self.kiss_api.send_msg(msg, callback_url) message.ack() except KissApiException as e: message.reject(requeue=True) raise e except KissOfflineException as e: message.reject(requeue=True) self.root_service.loop.create_task( self.take_a_break( 60, 'Kiss is Offline. Stopping {} for 1 min. Details: {}'. format(self, e))) finally: self.concurrent_requests -= 1
async def callback(self, message: IncomingMessage): """ Process all messages from 'core' queue by self.API object :param channel: :param body: :param envelope: :param properties: :return: """ with message.process(): try: logging.debug(" [x] Received %r" % message.body) await self.api.process(message.body.decode("utf-8")) message.ack() except Exception as e: logging.error("Broker callback error") logging.error(e)
async def received_job(message: IncomingMessage) -> None: """ AMQP job hook """ log.info("Received new job", extra=message.info()) try: job = _validate_message(message) except ValidationError as e: log.error(f"{e}. Rejecting without requeuing", extra=message.info()) message.reject(requeue=False) return result = await _process_job(job) message.ack() log.info("Job completed", extra={ "job_id": result.job.id, "url": result.job.url, "status": result.status, "size": result.size })
async def on_message(self, subscriber: AbstractSubscriber, message: IncomingMessage): try: context_data = { Headers.message_id: message.message_id or str(uuid.uuid4()), Headers.routing_key: message.routing_key, } context_data.update(**message.headers or {}) context = Context(**context_data) if message.type == 'call': result = await self.rpc.on_call_message(message.routing_key, message, context, subscriber, self.serializer) else: logger.info('sbus <~~~ %s: %s', message.routing_key, message.body[:256]) result = await subscriber.on_message(message.body, message.routing_key, context, serializer=self.serializer) message.ack() return result except Exception as error: await self._handle_exception(message, error)
async def process_incoming_message(self, message: aio_pika.IncomingMessage): request_payload = message.body request = None if self.request_class: request = self.request_class() request.ParseFromString(request_payload) else: request = eval(request_payload.decode()) result = await self.handler(request) if self.response_class: result = self.generate_response(result) else: result = str(result).encode() await self.publisher.default_exchange.publish( aio_pika.Message(body=result, correlation_id=message.correlation_id), routing_key=message.reply_to, ) message.ack()
async def _handle_msg( self, action: BindConsumer, pika_msg: PikaIncomingMessage, msg: AmqpMsg, ): self.log.info('received msg {} in queue {} ' 'from exchange {} topic {}'.format( pika_msg.delivery_tag, action.queue, msg.exchange, msg.topic, )) self.log.debug('msg received: {}'.format(str(msg))) result = True try: result = await action.callback(msg) self.log.debug('msg processed') except Exception as e: self.log.error('an error occurred when processing message') result = False if self._consumer_error_handlers: for handler in self._consumer_error_handlers: handler(e) else: traceback.print_exc() if not action.auto_ack and result: self.log.debug('sending ack for message {}'.format( pika_msg.delivery_tag)) pika_msg.ack() elif not action.auto_ack: self.log.debug('sending nack for message {}'.format( pika_msg.delivery_tag)) pika_msg.nack(requeue=action.nack_requeue)
async def message_consume(self, exchange: aio_pika.Exchange, message: aio_pika.IncomingMessage): await self.work(message) message.ack()
def on_response(self, message: IncomingMessage): message.ack() if message.correlation_id in self.futures: future = self.futures.pop(message.correlation_id) future.set_result(message.body)
async def on_message(self, message: IncomingMessage): message.ack() routing_key = message.routing_key body = message.body.decode() await self.listener(routing_key, json.loads(body))
async def consumer(message: IncomingMessage): entry = json.loads(message.body.decode('utf-8')) await db.save(entry) message.ack()
async def on_message(message: IncomingMessage): print(" [x] Received message") event = json.loads(message.body.decode('utf-8')) print("Message body is:", message.body) await bot.check_message(event) message.ack()
async def on_message(message: aio_pika.IncomingMessage): # TODO process messages print(message.body) message.ack() return message