async def on_message(exchange: Exchange, message: IncomingMessage):
with message.process():
payload = json.loads(message.body.decode())
try:
if payload['type'] == 'signup':
return_data = await signup(payload['data'])
elif payload['type'] == 'login':
return_data = await login(payload['data'])
elif payload['type'] == 'validate':
return_data = await validate(payload['data'])
except ValueError as err:
response = json.dumps({'status': err, 'data': {}}).encode()
else:
if isinstance(return_data, str):
response = json.dumps({
'status': return_data,
'data': {}
}).encode()
else:
response = json.dumps({
'status': 'ok',
'data': return_data
}).encode()
await exchange.publish(Message(body=response,
content_type='application/json',
correlation_id=message.correlation_id),
routing_key=message.reply_to)
async def process_rabbit_message(self, message: aio_pika.IncomingMessage):
async with message.process():
messageStr = str(message.body.decode())
messageStr = messageStr.replace("'", '"')
messageJson = ast.literal_eval(messageStr)
for k, v in messageJson.items():
await self.sendCmdsToClient(k, v)
async def on_message(self, message: IncomingMessage):
""" Handle incoming messages
Well defined types (RmqMessageTypes) are sent to system handlers,
all others are enqueued to behaviour mailbox for user handling.
"""
# If context processor will catch an exception, the message will be returned to the queue.
async with message.process():
self.log.debug(f"Received (info/body:")
self.log.debug(f" {message.info()}")
self.log.debug(f" {message.body.decode()}")
self.traces.append(TraceStoreMessage.from_msg(message), category="incoming")
if message.type in (RmqMessageTypes.CONTROL.name, RmqMessageTypes.RPC.name):
handler = self.handlers.get(handler=message.type)
if issubclass(handler, SystemHandler):
handler_instance = handler(core=self)
return await handler_instance.handle(message)
else:
return await handler(self, message)
for behaviour in self.behaviours:
await behaviour.enqueue(message)
self.log.debug(f"Message enqueued to: {behaviour} --> {message.body}")
self.traces.append(
TraceStoreMessage.from_msg(message), category=str(behaviour)
)
async def on_message_common(message: IncomingMessage):
with message.process():
json_data = message.body.decode('utf-8')
data = ujson.loads(json_data)
logger.info(f'Receiver #: {json_data}')
command = data.get('command')
body = data.get('body')
if command == ROBOT_ADD_FRIEND_COMMAND:
await robot_add_friend_callback(body)
if command == ROBOT_JOIN_GROUP_COMMAND:
await robot_join_group_callback(body)
if command == OPEN_GROUP_SUCCESS_COMMAND:
await bind_group_success_callback(body)
if command == ROBOT_JOIN_FAILED_COMMAND:
await join_group_failed_callback(body)
if command == ROBOT_KICKED_COMMAND:
await robot_kicked_callback(body)
if command == ROBOT_BLOCKED_COMMAND:
await robot_blocked(body)
if command == USER_JOIN_GROUP_COMMAND:
await user_join_group_callback(body)
if command == UNBIND_GROUP_SUCCESS_COMMAND:
await unbind_group_callback(body)
if command == GROUP_INFO_COMMAND:
await group_info_callback(body)
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage):
with message.process():
pieces = message.routing_key.split(".")
#print(" [x] %r:%r" % (
# message.routing_key,
# message.body
#))
try:
if pieces[0] == "c2":
query = await db_model.c2profile_query()
try:
profile = await db_objects.get(query, name=pieces[2], deleted=False)
except Exception as e:
print("Sending sync message to {}".format(pieces[2]))
await send_c2_rabbitmq_message(pieces[2], "sync_classes", "", "")
return
if profile.last_heartbeat < datetime.datetime.utcnow() + datetime.timedelta(seconds=-30) or not profile.container_running:
profile.running = False # container just started, clearly the inner service isn't running
#print("setting running to false")
profile.container_running = True
profile.last_heartbeat = datetime.datetime.utcnow()
await db_objects.update(profile)
elif pieces[0] == "pt":
query = await db_model.payloadtype_query()
try:
payload_type = await db_objects.get(query, ptype=pieces[2], deleted=False)
payload_type.container_running = True
payload_type.last_heartbeat = datetime.datetime.utcnow()
await db_objects.update(payload_type)
except Exception as e:
await send_pt_rabbitmq_message(pieces[2], "sync_classes", "", "")
except Exception as e:
logger.exception("Exception in rabbit_heartbeat_callback: {}, {}".format(pieces, str(e)))
async def handle_consume(message: IncomingMessage) -> None:
async with message.process():
logger.info(f"{message.body = }")
data = schemas.TransactionInputSchemas.parse_raw(message.body)
async with SessionLocal() as session:
transaction = await commit_db(
session,
obj_callable=add_transaction,
kwargs={
"uuid": data.uuid,
"amount": data.amount,
"wallet_id": data.wallet_id,
"transaction_type": data.transaction_type,
"currency": data.currency,
},
async_callback=after_transaction_create)
if data.transaction_type == schemas.TransactionType.TRANSFER:
await commit_db(session,
obj_callable=add_transaction,
kwargs={
"uuid": uuid.uuid4(),
"amount": data.amount,
"wallet_id": data.transfer_wallet_id,
"transaction_type": data.transaction_type,
"currency": data.currency,
"transfer_transaction": transaction
},
async_callback=after_transaction_create)
async def get_homework_step_info(exchange: Exchange, message: IncomingMessage):
with message.process():
try:
data = json.loads(message.body.decode())
if not isinstance(data.get('homework_id', None), int):
response = {
'status': 400,
'description': 'invalid homework_id'
}
from .Step import views
homework_id = data['homework_id']
response = await views.get_homework_step_info(homework_id)
if response is None:
response = {
'status': 404,
'description': 'homework_step not found'
}
else:
response = {'status': 200, 'data': response}
except Exception as e:
response = {'status': 500, 'description': str(e)}
finally:
response = json.dumps(response).encode()
await exchange.publish(Message(
body=response, correlation_id=message.correlation_id),
routing_key=message.reply_to)
async def message_process(self,
message: aio_pika.IncomingMessage,
func=print):
'Message decode and feedback'
with message.process(requeue=True):
message_data = json.loads(message.body.decode())
func(message_data)
async def process(self, message: IncomingMessage):
async with message.process():
call_id, path = message.body.decode().splitlines()
call = await Call.get_or_none(id=call_id)
if not call:
logger.warning(f'Bad call_id: {call_id}. Call not found')
return
if not Path(path).exists():
logger.warning(f'Converted record file not found. File: {path}')
return
async with async_open(path, 'rb') as f:
data = await f.read()
link = await upload_record(
self.s3client,
data,
call.create_record_filename(),
)
await CallRecord.create(
call_id=call_id,
file_name=link,
attempts_count=1,
)
async def event_callback(msg: IncomingMessage):
'''handles incoming events from the other services'''
with msg.process():
body = json.loads(msg.body.decode())
guild_id = body['guild_id']
# private = body['private']
await sio.emit('log_pool', body, room=f'guild_{guild_id}')
async def process_message(message: aio_pika.IncomingMessage):
async with message.process():
print(json.loads(message.body), flush=True)
await connection_manager.emit('new-post',
json.loads(message.body),
room=connection.sid)
await asyncio.sleep(1)
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage):
with message.process():
pieces = message.routing_key.split(".")
#print(" [x] %r:%r" % (
# message.routing_key,
# message.body
#))
try:
if pieces[0] == "c2":
query = await db_model.c2profile_query()
profile = await db_objects.get(query, name=pieces[2])
profile.container_running = True
profile.last_heartbeat = datetime.datetime.utcnow()
await db_objects.update(profile)
elif pieces[0] == "pt":
query = await db_model.payloadtype_query()
payload_type = await db_objects.get(query, ptype=pieces[2])
payload_type.container_running = True
payload_type.last_heartbeat = datetime.datetime.utcnow()
await db_objects.update(payload_type)
# now send updated PT code to everybody
transform_code = open("./app/api/transforms/utils.py",
'rb').read()
await send_pt_rabbitmq_message(
"*", "load_transform_code",
base64.b64encode(transform_code).decode('utf-8'))
except Exception as e:
print("Exception in rabbit_heartbeat_callback: {}, {}".format(
pieces, str(e)))
async def rabbit_heartbeat_callback(message: aio_pika.IncomingMessage):
with message.process():
pieces = message.routing_key.split(".")
#print(" [x] %r:%r" % (
# message.routing_key,
# message.body
#))
try:
if pieces[0] == "c2":
query = await db_model.c2profile_query()
profile = await db_objects.get(query, name=pieces[2])
if profile.last_heartbeat < datetime.datetime.utcnow() + datetime.timedelta(seconds=-30) or not profile.container_running:
profile.running = False # container just started, clearly the inner service isn't running
#print("setting running to false")
profile.container_running = True
profile.last_heartbeat = datetime.datetime.utcnow()
await db_objects.update(profile)
elif pieces[0] == "pt":
if pieces[2] != 'external':
query = await db_model.payloadtype_query()
payload_type = await db_objects.get(query, ptype=pieces[2])
payload_type.container_running = True
payload_type.last_heartbeat = datetime.datetime.utcnow()
await db_objects.update(payload_type)
# now send updated PT code to everybody
transform_code = open("./app/api/transforms/transforms.py", 'rb').read()
await send_pt_rabbitmq_message("*", "load_transform_code",
base64.b64encode(transform_code).decode('utf-8'), "")
except Exception as e:
logger.exception("Exception in rabbit_heartbeat_callback: {}, {}".format(pieces, str(e)))
async def _process_task(self, message: aio_pika.IncomingMessage):
async with message.process():
name, params = self._deserialize_task_params(message.body)
spec: TaskSpec = TaskRegistry.tasks[name]
logger.info("Received task '{}'{}".format(
params.task_id,
f" (try #{params.retries})" if params.retries > 0 else "",
))
result: TaskResult = await self._run_task(spec, params)
if result.status == TaskStatus.RETRIED:
params.retries += 1
if params.retries <= 5:
await self.submit_task(name, **params.asdict())
if result.status != TaskStatus.RETRIED:
logger.info(f"Storing result for task '{result.task_id}'")
await retry(
self._results_backend.store,
args=(result.task_id, result),
max_wait_time=60.0,
retry_callback=self._log_results_backend_retry,
)
logger.info("Finished task '{}' with status '{}'{}".format(
result.task_id,
result.status,
f" (try #{params.retries})" if params.retries > 0 else "",
))
async def on_message(channel: Channel, routing_key: str,
message: IncomingMessage):
async with message.process(ignore_processed=True):
# print(" [x] Received message %r" % message)
msg_json = loads(message.body.decode('utf-8'))
LOGGER.info(" Message body is: %r" % msg_json)
bot = WaBot(msg_json)
try:
async def send_message(chat_id, text):
msg = {'message': text, 'chat_id': chat_id}
_msg = Message(body=dumps(msg).encode(),
delivery_mode=DeliveryMode.PERSISTENT)
await channel.default_exchange.publish(message=_msg,
routing_key=routing_key)
LOGGER.info(f'Sent message: {chat_id} -> {text}')
chatId = msg_json['chatId']
reply = bot.processing()
await send_message(chatId, reply)
await message.ack()
except Exception as e:
await message.reject()
raise e
async def handler(message: aio_pika.IncomingMessage):
"""
Обработка задач.
:param message:
:return:
"""
async with message.process():
identifier = UUID(message.body.decode())
task = await context.task_provider.get(identifier)
if task is None:
return
await context.logger.info(f"Start calculating task {task.id}.")
start_time = datetime.now()
task.start_time = start_time
task.status = entities.TaskStatusEnum.running
await context.task_provider.update(task)
work()
end_time = datetime.now()
task.execution_time = end_time - start_time
task.status = entities.TaskStatusEnum.completed
await context.task_provider.update(task)
await context.logger.info(f"Task is done.")
async def on_message(message: IncomingMessage):
ts = datetime.datetime.utcnow().isoformat(timespec='milliseconds') + "Z"
async with message.process():
str_to_save = "{}, Topic {}: {}".format(ts, message.routing_key, message.body.decode())
async with aiofiles.open(FILENAME, "a") as out:
await out.write(str_to_save + "\n")
await out.flush()
async def _on_rpc_response(self, message: IncomingMessage):
async with message.process(requeue=False):
logger.debug(msg='Received PRC result: {}'.format(message.body))
try:
f = self.futures.pop(message.correlation_id)
f.set_result(json.loads(message.body.decode()))
except IndexError:
pass
async def on_message(message: IncomingMessage):
with message.process():
logger.debug(f'Receiver #: {message.body.decode("utf-8")}')
data = ujson.loads(message.body.decode('utf-8'))
command = data.get('command')
body = data.get('body')
if command == GROUP_MSG_COMMAND:
await group_message_callback_view(body)
async def on_message(message: IncomingMessage):
with message.process():
if message.routing_key == 'user.homework.complete':
from .Step.models import PassedStepUser
data = json.loads(message.body.decode())
await PassedStepUser.create(step_id=data['step_id'],
user_id=data['user_id'])
async def on_ad_message(message: IncomingMessage):
with message.process():
data = ujson.loads(message.body.decode('utf-8'))
command = data.get('command')
body = data.get('body')
logger.debug(f'Receiver #: {data}')
if command == 'message_class_result':
await ad_message_monitor_send(body)
async def process_message(self, message: IncomingMessage, *args, **kwargs):
async with message.process(
ignore_processed=self.config.driver.ignore_processed):
log.info(
f'Message <IncomingMessage correlation_id: {message.correlation_id}> received'
)
await self._process_message(message)
async def database_message_receiver(message: aio_pika.IncomingMessage):
with message.process():
msg = json.loads(message.body)
if msg["type"] == "update":
asyncio.create_task(update_query(msg))
if msg["type"] == "insert":
asyncio.create_task(insert_query(msg))
await send_to_analyze(msg["body"])
async def process_message(message: aio_pika.IncomingMessage):
async with message.process(ignore_processed=True):
message_body = ujson.loads(message.body)
forwarded = await forward_to_http_endpoint(message_body=message_body)
if not forwarded:
await message.nack(requeue=False)
return
await message.ack()
async def handle_render(lock_render: Callable[[int], Awaitable[None]],
reschedule: Callable[[int], Awaitable[None]],
message: aio_pika.IncomingMessage) -> None:
with message.process():
kwargs = msgpack.unpackb(message.body, raw=False)
try:
await render_or_reschedule(lock_render, reschedule, **kwargs)
except:
logger.exception('Error during render')
async def on_message_member(message: IncomingMessage):
with message.process():
data = ujson.loads(message.body.decode('utf-8'))
command = data.get('command')
body = data.get('body')
logger.debug(
f'Receiver #: {command}: {body[0]["group_id"]}: {len(body)}')
if command == GROUP_MEMBERS_COMMAND:
await member_info_call_back(body)
async def on_message(exchange: Exchange, message: IncomingMessage):
with message.process():
# print(message.correlation_id)
n = message.body.decode()
# try to parse the request, if fails send status 400 bad request
try:
req = json.loads(n)
method = req.get('method', None)
payload = req.get('payload', None)
assert method in [
'p.get', 'p.new', 'p.update', 'p.remove',
'u.get_plist', 'u.get', 'u.auth', 'u.new', 'u.remove', 'u.update', 'u.freeze'
]
assert type(payload) == dict
response = {}
if db_response != None:
if "_id" in db_response:
db_response.pop("_id")
n = "[.] done " + n
response["status"] = "done"
response["result"] = db_response
response = json.dumps(response)
# print(response, type(response))
if response is None:
response = json.dumps({"status": 404, "message": n})
n = "[!] Not Found " + n
except (KeyError, AssertionError):
response = json.dumps({"status": 400, "message": n})
n = "[!] Invalid Incoming Request "+ n
except ConnectionError:
response = json.dumps({"status": 500, "message": n})
n = "[!] Failed to Connect Database " + n
# except:
# response = json.dumps({"status": 500, "message": n})
# n = "[!] Internal Error " + n
# return the object requested found in db back to the RPC
print(response)
await exchange.publish(
Message(
body=response.encode(),
correlation_id=message.correlation_id
),
routing_key=message.reply_to
)
# log
print(n)
await logger.emit(n, tee=False)
async def on_message(message: aio_pika.IncomingMessage):
with message.process():
data = json.loads(message.body)
log.debug(data)
if data["Channel"] == "Log":
await sio.emit("logger", data=json.dumps(data))
elif data["Channel"] == "Progress":
await sio.emit("progress", data=json.dumps(data))
asyncio.sleep(2)
async def process_message(message: aio_pika.IncomingMessage):
async with message.process():
item = json.loads(message.body)
log.debug(item['name'])
log.debug(f'Recieved message {item}')
await insert_user_task(item)
# if I not scared to lost data from queue
# loop = asyncio.get_event_loop()
# loop.create_task(insert_user_task(message))
log.debug(f'Task finished {item}')
async def on_message(self, message: IncomingMessage):
"""
on_message doesn't necessarily have to be defined as async.
"""
async with message.process():
self.log.debug(f"Received:")
self.log.debug(f" {message.info()}")
self.log.debug(f" {message.body}")
self.core.traces.append(TraceStoreMessage.from_msg(message), category="incoming")
await self.behaviour.enqueue(message)
