async def _on_fail(self, message: IncomingMessage, error: Exception): try: if message.reply_to: response = self.serializer.serialize( Response( status=getattr(error, 'status', 500), body=ErrorResponseBody(error=str(error)).dict() ) ) await self.channel.default_exchange.publish( Message( response, delivery_mode=message.delivery_mode, correlation_id=message.correlation_id, timestamp=int(time.time()), ), message.reply_to, mandatory=False ) finally: logger.exception( 'Invalid message (%s) with routing_key %s, ' "don't retry it! Correlation id: %s", message.body, message.routing_key, message.headers.get(Headers.correlation_id) if message.headers else None, ) message.nack(requeue=False)
async def handle_message(self, message: IncomingMessage) -> None: message.ack() correlation_id = message.correlation_id future = self._reply_futures.pop(correlation_id, None) if future is not None: try: try: accept = self.accept = prepare_accept_content( self.cluster_proxy.accept) body = loads(message.body, content_type=message.content_type, content_encoding=message.content_encoding, accept=accept) except Exception as e: future.set_exception(e) else: future.set_result(body) except asyncio.InvalidStateError as e: # for catching the errors after the future obj was cancelled by the asyncio.wait_for timeout. sys.stdout.write("{}, correlation id: {}".format( e, correlation_id)) else: sys.stdout.write( "Unknown correlation id: {}".format(correlation_id))
async def _process_message(self, message: IncomingMessage, registered_obj: RegisteredCoroOrGen, **callable_kwargs): logger.debug(f"Processing message {message.delivery_tag}") try: if inspect.iscoroutinefunction(registered_obj.coro_or_gen): # it returns return_val = await registered_obj.coro_or_gen(**callable_kwargs ) if return_val is not None: await self._handle_return_value(message, return_val) else: # it yields async for v in registered_obj.coro_or_gen(**callable_kwargs): if v is not None: await self._handle_return_value(message, v) # At this point, we're processed the message and sent along new ones successfully. We can ack the original message # TODO consider doing all of this transactionally logger.debug(f"Acking message {message.delivery_tag}") message.ack() except asyncio.CancelledError: # We were told to cancel. nack with requeue so someone else will pick up the work message.nack(requeue=True) logger.info("Cancellation requested.") raise except Exception as e_: await self._handle_message_exception(message, e_)
async def on_message(self, message: IncomingMessage): if message.headers['target'] == self.name and self.ifPropertyAllowed( message.headers['property']) and self.runChecks( message.headers): self.exec(message.headers, message) else: message.nack()
async def _rpc_handle(self, message: aio_pika.IncomingMessage) -> None: """ Handles JSON-RPC request. :param message: incoming message """ try: reply_to = message.reply_to response_text = await self._dispatcher.dispatch(message.body, context=message) if response_text is not None: if reply_to is None: logger.warning("property 'reply_to' is missing") else: async with self._connection.channel() as channel: await channel.default_exchange.publish( aio_pika.Message( body=response_text.encode(), reply_to=reply_to, correlation_id=message.correlation_id, content_type=pjrpc.common.DEFAULT_CONTENT_TYPE, ), routing_key=reply_to, ) message.ack() except Exception as e: logger.exception("jsonrpc request handling error: %s", e)
async def consumer_func(message: aio_pika.IncomingMessage): json_body = json.loads(message.body.decode('utf-8')) task = loop.run_in_executor(executor, check, json_body['ip'], json_body['port']) res, _ = await asyncio.wait([task]) if list(res)[0].result(): message.delivery_mode = 2 await pika_channel.default_exchange.publish(message, routing_key=os.environ['RABBIT_MOTD_QUEUE']) message.ack()
async def route_message(self, message:aio_pika.IncomingMessage): handler = self.routes.get(message.routing_key) if not handler: message.reject() return await handler(message)
async def _on_message(self, message: aio_pika.IncomingMessage): logging.debug(f'Received message {str(message.body)}.') if getattr(self, 'custom_consume_function', None): if not await self.custom_consume_function(**self.custom_consume_function_kwargs): self.logger.error('Failed process message.') message.nack() return self.logger.info('Message has been successfully processed.') message.ack()
def on_message(message: IncomingMessage): try: data_dict = json.loads(message.body.decode()) # check if the packet={'key':[rg]<String>,'path_param':[_SHA1_attr/d]<String>,'data':[adapter packet]<JSON>} from the queue is empty if data_dict is None or not bool(data_dict): print('> data_dict is empty. data_dict---- ' + str(data_dict)) return 'failed' else: # print('> redis_key: ' + data_dict['key'] + '\nredis_path_param: ' + data_dict['path_param'] + '\nadapter_data_packet: ' + data_dict['data']) # this needs to be tested exp_val = data_dict['data']['id'] + '_' + data_dict['data'][ 'observationDateTime'] except json.decoder.JSONDecodeError as json_error: print('> JsonDecodeError!!!!' + str(json_error)) return 'failed' try: # check if redis already has existing data? chck_if_rg_exists = redis_client.jsonget(data_dict['key'], Path.rootPath()) # (nil) - never seen packet -> None, Insert # {}, !exists # data, upsert # redis already has previous data if chck_if_rg_exists is not None: print('> ' + str(data_dict['key']) + 'exists.') print('> Upserting ' + str(data_dict['data']) + ' at .' + data_dict['path_param']) print('> Upsertion still in progress...') redis_client.jsonset( data_dict['key'], '.' + data_dict['key'] + '.' + data_dict['path_param'], data_dict['data']) print('> Upsertion successful!') message.ack() # First time the ingestor receives a packet belonging to RG else: print('> RG=' + data_dict['key'] + ' is not present in Redis. Inserting RG with {} at root.') # create first entry in the redis server # origin = {rg: {SHA : {}}} origin = {data_dict['key']: {data_dict['path_param']: {}}} redis_client.jsonset(data_dict['key'], Path.rootPath(), origin) print('> Insertion still in progress...') # insert data now # JSON.GET resource-group-key/redis-key .path_param (SHA1...) # JSON.GET resource-group .resource-group.SHA1.... {adapter_data} redis_client.jsonset( data_dict['key'], '.' + data_dict['key'] + '.' + data_dict['path_param'], data_dict['data']) print('> Insertion successful!') message.ack() except redis.exceptions.ResponseError as r_error: print('> Response Error from Redis!!!! ' + str(r_error)) return 'failed'
async def process_message(self, message: aio_pika.IncomingMessage) -> None: if self.closing: await message.nack(requeue=True) return task_message = TaskMessage.parse_raw(message.body, content_type=message.content_type) task = self.registry.get_task(task_message.name) if not task: self.logger.error( "task_not_found", extra={ "task_name": task_message.name, }, ) message.reject(requeue=False) return async with message.process(reject_on_redelivered=True, requeue=True): logging_extra = { "name": task_message.name, "uid": task_message.uid } self.logger.info( "task_processing_start", extra=logging_extra, ) try: await self.processing_lock.acquire() await asyncio.wait_for(task.callback(**task_message.kwargs), timeout=task_message.timeout) except RejectMessage as e: await message.reject(requeue=e.requeue) self.logger.error("task_rejected", extra=dict(logging_extra, requeue=e.requeue)) except NackMessage as e: await message.nack(requeue=e.requeue) self.logger.error("task_nack", extra=dict(logging_extra, requeue=e.requeue)) except asyncio.TimeoutError: self.logger.error( "task_timeout", extra=dict(logging_extra, timeout=task_message.timeout), ) raise except Exception: self.logger.exception("error_while_processing_task", extra=logging_extra) raise finally: self.processing_lock.release()
async def on_message(message: IncomingMessage): print(message.body) body = loads(message.body) event = body['event'] #print('# {} [received]'.format(event)) print('# {} [received] sleep 1 sec..'.format(event)) await asyncio.sleep(1) # Represents async I/O operations del body['event'] await dispatch(event, body) # manual ack here # not with message.process() context, which auto ack even when error message.ack()
async def orderValidation(self, msg: IncomingMessage): startTime = time.time() self.app['EVENT_PROGRESS'].labels('ORDER').inc() data = msg.body.decode() now = datetime.now() now = now.strftime('%Y-%m-%d %H:%M:%S') data = data.split('|') account = data[0] action = data[1] order = data[2].split('.') order = { 'orderId': order[3], 'timestamp': now, 'account': account, 'action': action, 'stock': order[0], 'price': int(order[1]), 'vol': int(order[2]), 'cumVol': 0, 'status': '0', 'selected': False } params = { 'account': account if action == 'BUY' else f"{account}.{order['stock']}", 'amount': order['price'] * order['vol'] if action == 'BUY' else order['vol'] } # Check account availability to account services API async with self.webservice.request('POST', 'http://localhost:8001/order', json=params) as resp: valid = await resp.text() if valid == 'OK': self.app['EVENT_COUNTER'].labels('ORDER', 'SUCCESS').inc() await self.orderCollection.insert_one(order) order['eventLatency'] = self.app['EVENT_LATENCY'] order['eventProgress'] = self.app['EVENT_PROGRESS'] order['orderCounter'] = self.app['ORDER_COUNTER'] order['eventTime'] = startTime self.messages.on_next( (msg, order)) # Send valid order to order observable else: self.app['EVENT_COUNTER'].labels('ORDER', 'FAIL').inc() self.app['EVENT_PROGRESS'].labels('ORDER').dec() order['status'] = 'R' await self.orderCollection.insert_one(order) msg.ack() latency = time.time() - startTime self.app['EVENT_LATENCY'].labels('ORDER', 'FAIL').observe(latency)
async def on_message(message: IncomingMessage): print('> *****Inside key_generator*****') global data_redis_q, rg_dict data_redis_q = {} # data_dict is actual adapter packet data_dict = json.loads(message.body.decode()) # default value is for all aqm/flood type sensors default = '_d' # extract resource-group from the data packet res_id = data_dict['id'] rg = res_id.split('/')[3] # generate the SHA1 of the id sha_id = hashlib.sha1(res_id.encode()) # print("> RG from data---- " + rg) # print("> rg_dict---- " + str(rg_dict)) # print("> SHA1[res_id]---- " + sha_id.hexdigest()) # Check if _rg is present in rg_dict{} if rg in rg_dict.keys(): print('> RG is present.') # encode SHA1 of resource-id attribute = rg_dict[rg] path_param = '_' + sha_id.hexdigest() + '_' + data_dict[attribute] else: print('> RG is not present.') path_param = '_' + sha_id.hexdigest() + '_' + default # generate a dict = { 'key' : <resource-group-name>, 'path_param': <_SHA1(id)_attr/d>, 'data': adapter packet } data_redis_q['key'] = rg.replace('-', '_') data_redis_q['path_param'] = path_param data_redis_q['data'] = data_dict # print('> (on_message) routing_key---- '+routing_key) # publish the data into redis-ingestion-queue print('> bool(data_redis_q):---- ' + str(bool(data_redis_q))) if data_redis_q is not None and bool(data_redis_q): await latest_exchange.publish(message=Message( (json.dumps(data_redis_q)).encode()), routing_key=routing_key) print('> Message published.') # message.ack() else: # do nothing pass message.ack()
async def handle_msg(msg: IncomingMessage): print(f"dcmq: got message with routing key {msg.routing_key}") ds = datasetFromBinary(msg.body) uri = "" if "uri" in msg.headers: uri = msg.headers["uri"] if ds != None: try: await dcmhandler(channel, ds, uri, msg.routing_key, *additional_args) msg.ack() except Exception as e: msg.reject(requeue=True) raise(e)
def __init__( self, message: apika.IncomingMessage, log: Optional[logging.Logger] = None, ): self.command_id: str | None = None self.sender: str | None = None self.body = {} self.message = message self._log = log self.is_valid = True # Acknowledges receipt of message message.ack() self.info: Dict[Any, Any] = message.info() self.headers = self.info["headers"] for key in self.headers: if isinstance(self.headers[key], bytes): self.headers[key] = self.headers[key].decode() self.message_code = self.headers.get("message_code", None) if self.message_code is None: self.is_valid = False if self._log: self._log.warning( f"received message without message_code: {message}") return self.sender = self.headers.get("sender", None) if self.sender is None and self._log: self._log.warning(f"received message without sender: {message}") self.command_id = message.correlation_id command_id_header = self.headers.get("command_id", None) if command_id_header and command_id_header != self.command_id: if self._log: self._log.error(f"mismatch between message " f"correlation_id={self.command_id} " f"and header command_id={command_id_header} " f"in message {message}") self.is_valid = False return self.body = json.loads(self.message.body.decode())
async def _process_task(self, message: aio_pika.IncomingMessage): async with message.process(): name, params = self._deserialize_task_params(message.body) spec: TaskSpec = TaskRegistry.tasks[name] logger.info("Received task '{}'{}".format( params.task_id, f" (try #{params.retries})" if params.retries > 0 else "", )) result: TaskResult = await self._run_task(spec, params) if result.status == TaskStatus.RETRIED: params.retries += 1 if params.retries <= 5: await self.submit_task(name, **params.asdict()) if result.status != TaskStatus.RETRIED: logger.info(f"Storing result for task '{result.task_id}'") await retry( self._results_backend.store, args=(result.task_id, result), max_wait_time=60.0, retry_callback=self._log_results_backend_retry, ) logger.info("Finished task '{}' with status '{}'{}".format( result.task_id, result.status, f" (try #{params.retries})" if params.retries > 0 else "", ))
async def on_message_common(message: IncomingMessage): with message.process(): json_data = message.body.decode('utf-8') data = ujson.loads(json_data) logger.info(f'Receiver #: {json_data}') command = data.get('command') body = data.get('body') if command == ROBOT_ADD_FRIEND_COMMAND: await robot_add_friend_callback(body) if command == ROBOT_JOIN_GROUP_COMMAND: await robot_join_group_callback(body) if command == OPEN_GROUP_SUCCESS_COMMAND: await bind_group_success_callback(body) if command == ROBOT_JOIN_FAILED_COMMAND: await join_group_failed_callback(body) if command == ROBOT_KICKED_COMMAND: await robot_kicked_callback(body) if command == ROBOT_BLOCKED_COMMAND: await robot_blocked(body) if command == USER_JOIN_GROUP_COMMAND: await user_join_group_callback(body) if command == UNBIND_GROUP_SUCCESS_COMMAND: await unbind_group_callback(body) if command == GROUP_INFO_COMMAND: await group_info_callback(body)
async def handler(message: aio_pika.IncomingMessage): """ Обработка задач. :param message: :return: """ async with message.process(): identifier = UUID(message.body.decode()) task = await context.task_provider.get(identifier) if task is None: return await context.logger.info(f"Start calculating task {task.id}.") start_time = datetime.now() task.start_time = start_time task.status = entities.TaskStatusEnum.running await context.task_provider.update(task) work() end_time = datetime.now() task.execution_time = end_time - start_time task.status = entities.TaskStatusEnum.completed await context.task_provider.update(task) await context.logger.info(f"Task is done.")
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage): with message.process(): pieces = message.routing_key.split(".") #print(" [x] %r:%r" % ( # message.routing_key, # message.body #)) try: if pieces[0] == "c2": query = await db_model.c2profile_query() profile = await db_objects.get(query, name=pieces[2]) if profile.last_heartbeat < datetime.datetime.utcnow() + datetime.timedelta(seconds=-30) or not profile.container_running: profile.running = False # container just started, clearly the inner service isn't running #print("setting running to false") profile.container_running = True profile.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(profile) elif pieces[0] == "pt": if pieces[2] != 'external': query = await db_model.payloadtype_query() payload_type = await db_objects.get(query, ptype=pieces[2]) payload_type.container_running = True payload_type.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(payload_type) # now send updated PT code to everybody transform_code = open("./app/api/transforms/transforms.py", 'rb').read() await send_pt_rabbitmq_message("*", "load_transform_code", base64.b64encode(transform_code).decode('utf-8'), "") except Exception as e: logger.exception("Exception in rabbit_heartbeat_callback: {}, {}".format(pieces, str(e)))
async def event_callback(msg: IncomingMessage): '''handles incoming events from the other services''' with msg.process(): body = json.loads(msg.body.decode()) guild_id = body['guild_id'] # private = body['private'] await sio.emit('log_pool', body, room=f'guild_{guild_id}')
async def process_message(message: aio_pika.IncomingMessage): async with message.process(): print(json.loads(message.body), flush=True) await connection_manager.emit('new-post', json.loads(message.body), room=connection.sid) await asyncio.sleep(1)
async def process_rabbit_message(self, message: aio_pika.IncomingMessage): async with message.process(): messageStr = str(message.body.decode()) messageStr = messageStr.replace("'", '"') messageJson = ast.literal_eval(messageStr) for k, v in messageJson.items(): await self.sendCmdsToClient(k, v)
async def message_process(self, message: aio_pika.IncomingMessage, func=print): 'Message decode and feedback' with message.process(requeue=True): message_data = json.loads(message.body.decode()) func(message_data)
async def on_message(channel: Channel, routing_key: str, message: IncomingMessage): async with message.process(ignore_processed=True): # print(" [x] Received message %r" % message) msg_json = loads(message.body.decode('utf-8')) LOGGER.info(" Message body is: %r" % msg_json) bot = WaBot(msg_json) try: async def send_message(chat_id, text): msg = {'message': text, 'chat_id': chat_id} _msg = Message(body=dumps(msg).encode(), delivery_mode=DeliveryMode.PERSISTENT) await channel.default_exchange.publish(message=_msg, routing_key=routing_key) LOGGER.info(f'Sent message: {chat_id} -> {text}') chatId = msg_json['chatId'] reply = bot.processing() await send_message(chatId, reply) await message.ack() except Exception as e: await message.reject() raise e
async def on_message(exchange: Exchange, message: IncomingMessage): with message.process(): payload = json.loads(message.body.decode()) try: if payload['type'] == 'signup': return_data = await signup(payload['data']) elif payload['type'] == 'login': return_data = await login(payload['data']) elif payload['type'] == 'validate': return_data = await validate(payload['data']) except ValueError as err: response = json.dumps({'status': err, 'data': {}}).encode() else: if isinstance(return_data, str): response = json.dumps({ 'status': return_data, 'data': {} }).encode() else: response = json.dumps({ 'status': 'ok', 'data': return_data }).encode() await exchange.publish(Message(body=response, content_type='application/json', correlation_id=message.correlation_id), routing_key=message.reply_to)
async def get_homework_step_info(exchange: Exchange, message: IncomingMessage): with message.process(): try: data = json.loads(message.body.decode()) if not isinstance(data.get('homework_id', None), int): response = { 'status': 400, 'description': 'invalid homework_id' } from .Step import views homework_id = data['homework_id'] response = await views.get_homework_step_info(homework_id) if response is None: response = { 'status': 404, 'description': 'homework_step not found' } else: response = {'status': 200, 'data': response} except Exception as e: response = {'status': 500, 'description': str(e)} finally: response = json.dumps(response).encode() await exchange.publish(Message( body=response, correlation_id=message.correlation_id), routing_key=message.reply_to)
async def process(self, message: IncomingMessage): async with message.process(): call_id, path = message.body.decode().splitlines() call = await Call.get_or_none(id=call_id) if not call: logger.warning(f'Bad call_id: {call_id}. Call not found') return if not Path(path).exists(): logger.warning(f'Converted record file not found. File: {path}') return async with async_open(path, 'rb') as f: data = await f.read() link = await upload_record( self.s3client, data, call.create_record_filename(), ) await CallRecord.create( call_id=call_id, file_name=link, attempts_count=1, )
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage): with message.process(): pieces = message.routing_key.split(".") #print(" [x] %r:%r" % ( # message.routing_key, # message.body #)) try: if pieces[0] == "c2": query = await db_model.c2profile_query() profile = await db_objects.get(query, name=pieces[2]) profile.container_running = True profile.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(profile) elif pieces[0] == "pt": query = await db_model.payloadtype_query() payload_type = await db_objects.get(query, ptype=pieces[2]) payload_type.container_running = True payload_type.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(payload_type) # now send updated PT code to everybody transform_code = open("./app/api/transforms/utils.py", 'rb').read() await send_pt_rabbitmq_message( "*", "load_transform_code", base64.b64encode(transform_code).decode('utf-8')) except Exception as e: print("Exception in rabbit_heartbeat_callback: {}, {}".format( pieces, str(e)))
async def on_message(self, message: IncomingMessage): """ Handle incoming messages Well defined types (RmqMessageTypes) are sent to system handlers, all others are enqueued to behaviour mailbox for user handling. """ # If context processor will catch an exception, the message will be returned to the queue. async with message.process(): self.log.debug(f"Received (info/body:") self.log.debug(f" {message.info()}") self.log.debug(f" {message.body.decode()}") self.traces.append(TraceStoreMessage.from_msg(message), category="incoming") if message.type in (RmqMessageTypes.CONTROL.name, RmqMessageTypes.RPC.name): handler = self.handlers.get(handler=message.type) if issubclass(handler, SystemHandler): handler_instance = handler(core=self) return await handler_instance.handle(message) else: return await handler(self, message) for behaviour in self.behaviours: await behaviour.enqueue(message) self.log.debug(f"Message enqueued to: {behaviour} --> {message.body}") self.traces.append( TraceStoreMessage.from_msg(message), category=str(behaviour) )
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage): with message.process(): pieces = message.routing_key.split(".") #print(" [x] %r:%r" % ( # message.routing_key, # message.body #)) try: if pieces[0] == "c2": query = await db_model.c2profile_query() try: profile = await db_objects.get(query, name=pieces[2], deleted=False) except Exception as e: print("Sending sync message to {}".format(pieces[2])) await send_c2_rabbitmq_message(pieces[2], "sync_classes", "", "") return if profile.last_heartbeat < datetime.datetime.utcnow() + datetime.timedelta(seconds=-30) or not profile.container_running: profile.running = False # container just started, clearly the inner service isn't running #print("setting running to false") profile.container_running = True profile.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(profile) elif pieces[0] == "pt": query = await db_model.payloadtype_query() try: payload_type = await db_objects.get(query, ptype=pieces[2], deleted=False) payload_type.container_running = True payload_type.last_heartbeat = datetime.datetime.utcnow() await db_objects.update(payload_type) except Exception as e: await send_pt_rabbitmq_message(pieces[2], "sync_classes", "", "") except Exception as e: logger.exception("Exception in rabbit_heartbeat_callback: {}, {}".format(pieces, str(e)))