def client_task(func_body, target_queue: 'Queue'):
logger.info(f"@pyrogram_task_call: {prettify(func_body)}")
response = None
def callback(body, message):
nonlocal response
response = body
logger.info(f"@pyrogram_task_response: {body} ")
# connections
with Connection('amqp://localhost') as conn:
# produce
producer = conn.Producer(serializer='pickle')
producer.publish(body=func_body,
exchange=tg_exchange,
routing_key=target_queue.name,
declare=[target_queue, callback_queue],
retry=True,
correlation_id=uuid(),
reply_to=callback_queue.name,
retry_policy={
'interval_start': 0,
'interval_step': 2,
'interval_max': 30,
'max_retries': 30,
})
with Consumer(conn,
callbacks=[callback],
queues=[callback_queue],
no_ack=True):
while response is None:
conn.drain_events()
return response
def handle_predelete(sender, instance, **kwargs):
logger.info(
f"\npre_delete: {str(sender)} : <{prettify(instance.__dict__)}> : {kwargs}"
)
messages = models.Message.objects.filter(forward_from_chat=instance, )
if messages:
for message in messages:
logger.info(message)
def iter_messages(
client: 'ClientProxy',
chat_id: int,
last_message_id: int,
) -> Generator['types.Message', None, None]:
offset_id = last_message_id
last_offset_id = last_message_id
row = 0
sleep_counter = 0
sleep_time = 1
total_msgs = 0
started = time.perf_counter()
while True:
if not row % 5:
sleep_counter += 1
time.sleep(sleep_time)
else:
time.sleep(0.1)
messages = client('get_history',
chat_id=chat_id,
limit=100,
offset=-1,
offset_id=offset_id)
if not messages:
return
last_offset_id, offset_id = offset_id, messages[-1].message_id
row += 1
total_msgs += len(messages)
# logger.info((messages[0].message_id, messages[-1].message_id, len(messages)))
logger.info(
f"got {total_msgs} messages in {time.perf_counter() - started - sleep_counter * sleep_time:.3f} of chat: {chat_id}"
)
# all_messages.extend(messages)
for message in messages:
yield message
if last_offset_id == offset_id:
return
if offset_id == 1:
return
def callback(body, message):
nonlocal response
response = body
logger.info(f"@pyrogram_task_response: {body} ")
def callback(body, message):
nonlocal response
response = body
logger.info(f"@tg_function_response: {body}")
def on_task(self, body: dict, message: pyamqp.Message):
func = body['func']
args = body['args']
kwargs = body['kwargs']
logger.info(
f'on telegram consumer {self.index} Got task: {prettify(body)}')
logger.info(f"task_queue_length: {len(self.task_queues)}")
# if len(self.task_queues):
# messages = client_task(
# {
# 'func': 'iter_history',
# 'args': [],
# 'kwargs': {
# 'chat_id':'just_123_test',
# },
# },
# self.task_queues.values()[0]
# )
# logger.info(messages)
if func == 'task_init_clients':
response = self.telegram_tasks.init_clients_task(*args, **kwargs)
elif func == 'task_get_me':
response = self.telegram_tasks.get_me_task(*args, **kwargs)
elif func == 'add_telegram_channel_task':
response = self.telegram_tasks.add_telegram_channel_task(
*args, **kwargs)
elif func == 'task_iterate_dialogs':
response = self.telegram_tasks.iterate_dialogs_task(
*args, **kwargs)
elif func == 'log_chat_shared_medias_task':
response = self.telegram_tasks.log_chat_shared_medias_task(
*args, **kwargs)
elif func == 'log_chat_member_count_task':
response = self.telegram_tasks.log_chat_member_count_task(
*args, **kwargs)
elif func == 'log_message_views_task':
response = self.telegram_tasks.log_message_views_task(
*args, **kwargs)
elif func == 'iterate_chat_history_task':
response = self.telegram_tasks.iterate_chat_history_task(
*args, **kwargs)
elif func == 'log_admin_logs_task':
response = self.telegram_tasks.log_admin_logs_task(*args, **kwargs)
elif func == 'log_chat_members_task':
response = self.telegram_tasks.log_chat_members_task(
*args, **kwargs)
elif func == 'task_analyze_chat_members':
response = self.task_analyze_chat_members(*args, **kwargs)
elif func == 'upload_post':
response = self.telegram_tasks.upload_post(*args, **kwargs)
else:
response = BaseResponse().done()
self.producer.publish(
body=prettify(response, include_class_name=False),
exchange='',
routing_key=message.properties['reply_to'],
correlation_id=message.properties['correlation_id'],
serializer='json',
retry=True,
)
message.ack()
def run(self) -> None:
logger.info(f"Telegram Task Consumer {self.index} started ....")
TelegramTaskConsumer(connection=Connection('amqp://localhost'),
index=self.index,
task_queues=self.task_queues).run()
def _parse_chat_member(
*,
raw_chat_member: 'types.ChatMember',
db_chat: 'tg_models.ChatMember',
db_user: '******',
raw_admin_log_event: 'types.ChannelAdminLogEvent',
invited: bool = False,
toggle_ban: bool = False,
promoted: bool = False,
demoted: bool = False,
is_previous: bool = None,
) -> 'tg_models.ChatMember':
db_membership = tg_models.Membership.objects.get_membership_by_user_id(
db_chat=db_chat,
user_id=raw_admin_log_event.user.id,
)
invited_by = db_user if invited else None
demoted_by = db_user if demoted else None
promoted_by = db_user if promoted else None
kicked_by = db_user if toggle_ban else None
if invited_by is None and demoted_by is None and promoted_by is None and kicked_by is None:
raise ValueError(f'wrong value: at least one must be true')
if promoted:
logger.info(f"before: {db_membership}")
if db_membership:
db_chat_member = tg_models.ChatMember.objects.update_or_create_from_raw(
raw_chat_member=raw_chat_member,
db_membership=db_membership,
event_date_ts=raw_admin_log_event.date,
db_invited_by=invited_by,
db_demoted_by=demoted_by,
db_kicked_by=kicked_by,
db_promoted_by=promoted_by,
is_previous=is_previous)
db_membership.update_membership_status(
new_status=db_chat_member,
event_date_ts=raw_admin_log_event.date,
)
else:
db_chat_member = tg_models.ChatMember.objects.update_or_create_from_raw(
raw_chat_member=raw_chat_member,
db_membership=None,
event_date_ts=raw_admin_log_event.date,
db_invited_by=invited_by,
db_demoted_by=demoted_by,
db_kicked_by=kicked_by,
db_promoted_by=promoted_by,
is_previous=is_previous,
)
logger.info(f"after_usr: {db_chat_member.user}")
logger.info(f"after_chat: {db_chat}")
logger.info(f"after_date: {raw_admin_log_event.date}")
db_membership = tg_models.Membership.objects.update_or_create_membership(
db_user=db_chat_member.user,
db_chat=db_chat,
new_status=db_chat_member,
event_date_ts=raw_admin_log_event.date)
if promoted:
logger.info(f"after: {db_membership}")
db_chat_member.update_membership(db_membership=db_membership)
return db_chat_member
def update_chat_from_raw(self, *, chat_id: int,
raw_chat: types.Chat) -> bool:
logger.info(f'running the base method in {self}')
return False
def handle_presave(sender, instance, **kwargs):
logger.info(
f"\npresave: {str(sender)} : <{prettify(instance.__dict__)}> : {kwargs}"
)
def handle_postsave(sender, instance, created, update_fields, raw, **kwargs):
# def handle_postsave(sender,**kwargs):
logger.info(f"\npostsave: {str(sender)} : <{str(instance)}> : {kwargs}")
def configure_workers(sender=None, conf=None, **kwargs):
if sender.hostname == 'celery@telegram_worker':
logger.info("Telegram consumer starting...")
tg_tasks.init_consumer.delay()
tg_tasks.init_clients.apply_async(countdown=5)