async def delete_post(self, task_message: DeletePostTaskMessage) -> bool:
post = get_object_or_none(Post, pk=task_message.post_id)
if not post:
logger.error('Post Not Found Post id {}'.format(
task_message.post_id))
return False
social_uid, access_token = self._get_vk_credentials(post.user)
method = WallDeleteMethod(priority=task_message.priority,
access_token=access_token,
owner_id=format_group_id(
post.group.group_id),
post_id=post.post_id)
response_dict = await method.execute()
status = self.check_response(post,
response_dict,
fail_status=Post.FAILED_DELETE,
delete_status=Post.IS_DELETED)
if not status:
return False
post.status = Post.IS_DELETED if task_message.delete_status == Post.IS_DELETED else task_message.delete_status
post.save()
return True
async def delete_video(self, task_message: DeleteVideoTaskMessage) -> bool:
video = get_object_or_none(Video, pk=task_message.video_id)
if not video:
logger.error('Video Not Found vid: {}'.format(
task_message.video_id))
return False
social_uid, access_token = self._get_vk_credentials(video.user)
method = VideoDeleteMethod(access_token=access_token,
video_id=video.vid,
owner_id=video.owner,
priority=task_message.priority)
response_dict = await method.execute()
status = self.check_response(video,
response_dict,
fail_status=Video.DELETE_FAIL,
delete_status=Video.IS_DELETED)
if not status:
return False
video.status = Video.IS_DELETED
video.save()
return True
async def create_post(self, task_message: PostTaskMessage) -> bool:
post = get_object_or_none(Post, pk=task_message.post_id)
if not post:
logger.error('Post Not Found Post id {}'.format(
task_message.post_id))
return False
user = User.objects.get(pk=task_message.user_id)
group = Group.objects.get(pk=task_message.target_group_id)
campaign = AdvertisingCampaign.objects.get(pk=task_message.campaign_id)
description = task_message.text
target_video_upload = task_message.target_video_upload
social_uid, access_token = self._get_vk_credentials(user=user)
video = Video.objects.filter(group=group,
campaign=campaign,
user=user,
status=Video.IS_ACTIVE).first()
if not video:
video = Video(group=group,
campaign=campaign,
user=user,
status=Video.IN_PROGRESS,
factor=settings.KOFF)
video.save()
to_group = target_video_upload == TargetVideoUpload.GROUP.value
status_ok = await self._video_save(post=post,
video=video,
access_token=access_token,
campaign=campaign,
description=description,
priority=task_message.priority,
group=group,
to_group=to_group)
if not status_ok:
post.status = Post.FAILED
post.vk_response = video.vk_response
post.save()
return False
if status_ok == VideoSaveMethod.TOO_MUCH_REQUESTS_MSG:
return False
return await self._wall_post(access_token=access_token,
group=group,
video=video,
priority=task_message.priority,
description=description,
post=post,
campaign_pk=campaign.pk)
async def bulk_delete_post(self, task_message: BulkDeletePostTaskMessage) -> bool:
user = get_object_or_none(User, pk=task_message.user_id)
if not user:
logger.error('User not found {}'.format(task_message.user_id))
return False
social_uid, access_token = self._get_vk_credentials(user)
posts_queryset = Post.objects.filter(pk__in=task_message.post_ids)
posts_list = list(posts_queryset)
call_method_list = []
post_splits = split_list(posts_list, 25)
for post_split in post_splits:
for post in post_split:
args = {
VK_MESSAGE_KEY.OWNER_ID: format_group_id(post.group.group_id),
VK_MESSAGE_KEY.POST_ID: post.post_id
}
call = ExecuteMethod.construct(
ExecuteMethod.WALL_ENTITY,
ExecuteMethod.DELETE_METHOD,
json.dumps(args)
)
call_method_list.append(call)
method = ExecuteMethod(
priority=task_message.priority,
access_token=access_token,
code=ExecuteMethod.construct_code(call_method_list)
)
response_dict = await method.execute()
if not response_dict:
error = DefaultError(
error_key=ErrorCode.RESPONSE_IS_EMPTY_KEY,
text=ErrorCode.RESPONSE_IS_EMPTY_TEXT
)
for post in posts_list:
post.vk_response = error.to_json()
post.status = Post.FAILED_DELETE
post.save()
return False
if VK_MESSAGE_KEY.ERROR in response_dict:
# берем ошибку по коду ответа (только в execute)
error_dict = response_dict.get(VK_MESSAGE_KEY.ERROR)
error = DefaultError.get_error_from_response(
error_dict=error_dict,
error_key=VK_MESSAGE_KEY.ERROR_CODE
)
sentry_error = {
'error': response_dict,
'error_msg': error,
'user_id': user.pk,
'username': user.get_full_name()
}
sentry_logger.error(msg=sentry_error)
for post in posts_list:
post.vk_response = error
post.status = Post.FAILED_DELETE
post.save()
return False
response_list = response_dict.get(VK_MESSAGE_KEY.RESPONSE)
if VK_MESSAGE_KEY.EXECUTE_ERRORS in response_dict:
error_list = response_dict.get(VK_MESSAGE_KEY.EXECUTE_ERRORS)
sentry_logger.warning(msg="Execute errors while post deleting", extra={
'error': response_dict,
'user_id': user.pk,
'username': user.get_full_name()
})
false_response_posts = []
# порядок постов в сплите совпадает с порядко постов в ответе
for index, post in enumerate(post_split):
if not response_list[index]:
false_response_posts.append(post)
if false_response_posts:
for index, post in enumerate(false_response_posts):
concrete_error = error_list[index]
concrete_error_msg = DefaultError.get_error_from_response(
error_dict=concrete_error,
error_key=VK_MESSAGE_KEY.ERROR_MSG
)
# error code 7, Access denied распознаем, как удаленные
error_message = concrete_error.get(VK_MESSAGE_KEY.ERROR_MSG)
if error_message == ErrorCode.WALL_DELETE_ACCESS_DENIED_KEY:
post.status = Post.IS_DELETED
else:
post.status = Post.FAILED_DELETE
post.vk_response = concrete_error_msg
post.save()
self.log_to_sentry(post, concrete_error_msg, level='warning')
else:
for index, post in enumerate(post_split):
if response_list[index]:
post.status = Post.IS_DELETED
post.save()
call_method_list[:] = []
posts_queryset.update(status=Post.IS_DELETED)
return True
async def get_stats_post(self, task_message: GetStatsPostMessage):
posts_queryset = Post.objects.filter(pk__in=task_message.post_ids, group__members__gt=5000)
prefetch = Prefetch(
'posts',
queryset=posts_queryset,
to_attr='custom_prefetch_posts'
)
user_id_list = posts_queryset.values_list('user_id', flat=True).distinct()
users = User.objects.filter(pk__in=user_id_list).prefetch_related(prefetch)
try:
if not posts_queryset:
logger.error('Posts are empty {}'.format(task_message.post_ids))
return False
posts_queryset.update(is_stat_fetching=True)
for user in users:
posts = user.custom_prefetch_posts
social_uid, access_token = self._get_vk_credentials(user)
api_calls = [
ExecuteMethod.construct(
ExecuteMethod.STATS_ENTITY,
ExecuteMethod.POST_REACH,
json.dumps({
VK_MESSAGE_KEY.OWNER_ID: format_group_id(post.group.group_id),
VK_MESSAGE_KEY.POST_ID: post.post_id
})
) for post in posts
]
execute_parts = split_list(api_calls, 25)
posts_parts = split_list(posts, 25)
for index_part, execute_part in enumerate(execute_parts):
post_part = posts_parts[index_part]
method = ExecuteMethod(
priority=task_message.priority,
access_token=access_token,
code=ExecuteMethod.construct_code(execute_part)
)
response_dict = await method.execute()
if not response_dict:
traceback_log = traceback.format_exc()
sentry_logger.error(msg="False response on get stat Post", extra={
'trace': traceback_log,
'locals': locals()
})
continue
response_list = response_dict.get(VK_MESSAGE_KEY.RESPONSE)
if VK_MESSAGE_KEY.ERROR in response_dict:
sentry_error = {
'error': response_dict
}
sentry_logger.warning(msg=sentry_error)
continue
if VK_MESSAGE_KEY.EXECUTE_ERRORS in response_dict:
false_response_list = []
for index, post_api_call in enumerate(execute_part):
if not response_list[index]:
post = post_part[index]
false_response_list.append(post)
errors = response_dict.get(VK_MESSAGE_KEY.EXECUTE_ERRORS)
if errors and false_response_list:
for index, post in enumerate(false_response_list):
error = errors[index]
post_stats, is_created = PostStats.objects.get_or_create(post=post)
post_stats.vk_response = error
post_stats.save()
sentry_logger.warning(
msg='False response get_stat for post pk {}'.format(post.pk),
extra={
'post_id_in_vk': post.post_id,
'group': post.group.name,
'user_pk': post.user_id,
'username': post.user.get_full_name(),
'error': error,
'link': 'https://vk.com/wall-{}_{}'.format(
post.group.group_id,
post.post_id
)
}
)
SocialService.vk_check_post(SocialType.VK, post_id=post.pk)
for index, post_api_call in enumerate(execute_part):
if response_list[index]:
posts_array = response_list[index]
post = post_part[index]
post_stats, is_created = PostStats.objects.get_or_create(post=post)
u_stat, _ = UnitedPostsStat.objects.get_or_create(
user=post.user,
group=post.group,
campaign=post.campaign
)
if 'reach_subscribers' in posts_array[0] and 'reach_total' in posts_array[0]:
if post.campaign.campaign_type in [AdvertisingCampaign.VK_VIDEO_POST,
AdvertisingCampaign.VK_REPOST]:
last_views = int(
post_stats.reach.split('/')[1].replace('-', '0')) if post_stats.reach else 0
diff = posts_array[0]['reach_total'] - last_views
today = timezone.make_aware(
datetime.now(),
timezone.get_default_timezone()
).date()
views_stat, _ = ViewsStat.objects.get_or_create(
group=post.group,
campaign=post.campaign,
date__contains=today,
user=post.user
)
if diff:
views_stat.views += diff
views_stat.save()
post_stats.reach = '{}/{}'.format(
str(posts_array[0]['reach_subscribers']),
str(posts_array[0]['reach_total'])
)
post_stats.save()
try:
if len(SuspiciousUser.objects.filter(user=post.user, campaign=post.campaign)) > 0:
reach_tags = {
'type': 'reach',
'campaign_pk': post.campaign.pk,
'user_pk': post.user.pk,
'user': translit(str(post.user), 'ru', reversed=True),
'group_pk': post.group.pk,
'group_name': translit(post.group.name, 'ru', reversed=True)
}
StatsClient.event('video-seed__suspicious_users',
value=posts_array[0]['reach_total'],
tags=reach_tags)
except Exception as e:
sentry_logger.error(msg=e)
for post in posts:
post.is_stat_fetching = False
post.last_stat_fetch_date = timezone.make_aware(datetime.now(), timezone.get_default_timezone())
post.save()
except Exception as e:
traceback_log = traceback.format_exc()
sentry_logger.error(msg=e, extra={
'trace': traceback_log,
'locals': locals()
})
posts_queryset.update(is_stat_fetching=False,
last_stat_fetch_date=timezone.make_aware(datetime.now(), timezone.get_default_timezone()))
return True
async def get_stats_group(self, task_message: GetStatsGroupMessage):
campaign = AdvertisingCampaign.objects.get(pk=task_message.campaign_id)
groups = Group.objects.filter(pk__in=task_message.group_ids)
user_groups_queryset = UserGroup.objects.filter(
group_id__in=task_message.group_ids)
prefetch = Prefetch('user_groups_m2m',
queryset=user_groups_queryset,
to_attr='custom_user_groups')
users = User.objects.filter(pk__in=user_groups_queryset.values_list('user_id', flat=True).distinct()) \
.prefetch_related(prefetch)
if not groups:
logger.error('Groups is empty {}'.format(task_message.group_ids))
return False
try:
date_to = datetime.now().strftime('%Y-%m-%d')
for user in users:
token = self._get_vk_credentials(user)[1]
user_groups = user.custom_user_groups
execute_calls = []
for user_group in user_groups:
execute_calls.append(
ExecuteMethod.construct(
ExecuteMethod.STATS_ENTITY,
ExecuteMethod.GET_METHOD,
json.dumps({
VK_MESSAGE_KEY.GROUPD_ID:
user_group.group.group_id,
VK_MESSAGE_KEY.DATE_FROM:
campaign.date_created.strftime('%Y-%m-%d'),
VK_MESSAGE_KEY.DATE_TO:
date_to
})))
user_group.group.is_stat_fetching = True
user_group.group.save()
execute_parts = split_list(execute_calls, 25)
group_parts = split_list(user_groups, 25)
for index_part, execute_part in enumerate(execute_parts):
group_part = group_parts[index_part]
method = ExecuteMethod(
priority=task_message.priority,
access_token=token,
code=ExecuteMethod.construct_code(execute_part))
response_dict = await method.execute()
if not response_dict:
traceback_log = traceback.format_exc()
sentry_logger.error(
msg="False response on get stat Group",
extra={
'trace': traceback_log,
'locals': locals()
})
continue
response_list = response_dict.get(VK_MESSAGE_KEY.RESPONSE)
if VK_MESSAGE_KEY.ERROR in response_dict:
sentry_error = {'error': response_dict}
sentry_logger.warning(msg=sentry_error)
continue
for index, group_api_call in enumerate(execute_part):
if response_list[index]:
posts = Post.objects.all().filter(
group=group_part[index].group,
campaign__pk=task_message.campaign_id,
status=Post.IS_ACTIVE)
male, female = 0, 0
for stat_item in response_list[index]:
if 'sex' in stat_item:
m = list(
filter(
lambda sexes: 'm' in sexes.values(
), stat_item['sex']))
if len(m) == 1:
male += int(m[0]['visitors'])
f = list(
filter(
lambda sexes: 'f' in sexes.values(
), stat_item['sex']))
if len(f) == 1:
female += int(f[0]['visitors'])
for post in posts:
post_stats = PostStats.objects.get_or_create(
post=post)[0]
u_stat, _ = UnitedPostsStat.objects.get_or_create(
user=post.user,
group=post.group,
campaign=post.campaign)
if not female and not male:
post_stats.sex_m = 0
post_stats.sex_f = 0
else:
sex_m = round(male / (male + female) * 100,
1)
sex_f = round(
female / (male + female) * 100, 1)
post_stats.sex_m = sex_m
post_stats.sex_f = sex_f
post_stats.save()
else:
group = group_part[index].group
sentry_error = {
'error':
response_dict.get(
VK_MESSAGE_KEY.EXECUTE_ERRORS),
'group':
group.name
}
sentry_logger.warning(msg=sentry_error,
extra={
'response':
response_dict,
'execute_part':
execute_part,
'group_part': group_part,
'execute_parts':
execute_parts,
'group_parts':
group_parts
})
except Exception as e:
traceback_log = traceback.format_exc()
sentry_logger.error(msg=e,
extra={
'trace': traceback_log,
'locals': locals()
})
Group \
.objects \
.filter(pk__in=task_message.group_ids) \
.update(is_stat_fetching=False,
last_stat_fetch_date=timezone.make_aware(datetime.now(), timezone.get_default_timezone()))
return True
async def bulk_delete_video(
self, task_message: BulkDeleteVideoTaskMessage) -> bool:
user = get_object_or_none(User, pk=task_message.user_id)
if not user:
logger.error('User not found {}'.format(task_message.user_id))
return False
social_uid, access_token = self._get_vk_credentials(user)
videos_queryset = Video.objects.filter(pk__in=task_message.video_ids)
video_list = list(videos_queryset)
call_method_list = []
video_splits = split_list(video_list, 25)
for video_split in video_splits:
for video in video_split:
args = {
VK_MESSAGE_KEY.OWNER_ID: video.owner,
VK_MESSAGE_KEY.VIDEO_ID: video.vid
}
call = ExecuteMethod.construct(ExecuteMethod.VIDEO_ENTITY,
ExecuteMethod.DELETE_METHOD,
json.dumps(args))
call_method_list.append(call)
method = ExecuteMethod(
priority=task_message.priority,
access_token=access_token,
code=ExecuteMethod.construct_code(call_method_list))
response_dict = await method.execute()
if not response_dict:
error = DefaultError(error_key=ErrorCode.RESPONSE_IS_EMPTY_KEY,
text=ErrorCode.RESPONSE_IS_EMPTY_TEXT)
for video in video_list:
video.vk_response = error.to_json()
video.status = Video.DELETE_FAIL
video.save()
return False
if VK_MESSAGE_KEY.ERROR in response_dict:
error_dict = response_dict.get(VK_MESSAGE_KEY.ERROR)
error_msg = DefaultError.get_error_from_response(
error_dict=error_dict, error_key=VK_MESSAGE_KEY.ERROR_CODE)
sentry_logger.warning(msg="Error while deleting videos",
extra={
'error': response_dict,
'error_msg': error_msg,
'user_id': user.pk,
'username': user.get_full_name()
})
for video in video_list:
video.vk_response = error_msg
video.status = Video.DELETE_FAIL
video.save()
return False
response_list = response_dict.get(VK_MESSAGE_KEY.RESPONSE)
if VK_MESSAGE_KEY.EXECUTE_ERRORS in response_dict:
error_list = response_dict.get(VK_MESSAGE_KEY.EXECUTE_ERRORS)
false_response_videos = []
# порядок постов в сплите совпадает с порядком постов в ответе
for index, video in enumerate(video_split):
if not response_list[index]:
false_response_videos.append(video)
if false_response_videos:
for index, video in enumerate(false_response_videos):
concrete_error = error_list[index]
concrete_error_msg = DefaultError.get_error_from_response(
error_dict=concrete_error,
error_key=VK_MESSAGE_KEY.ERROR_MSG)
# error code 7, Access denied распознаем, как удаленные
error_message = concrete_error.get(
VK_MESSAGE_KEY.ERROR_MSG)
if error_message == ErrorCode.ACCESS_DENIED_KEY:
video.status = Video.IS_DELETED
else:
video.status = Video.DELETE_FAIL
video.vk_response = concrete_error_msg
video.save()
self.log_to_sentry(video,
concrete_error_msg,
level='warning')
else:
for index, video in enumerate(video_split):
if response_list[index]:
video.status = Video.IS_DELETED
video.save()
call_method_list[:] = []
videos_queryset.update(status=Video.IS_DELETED)
return True
async def get_stats_video(self,
task_message: GetStatsVideoMessage) -> bool:
videos = Video.objects.all().filter(pk__in=task_message.video_ids)
if not videos:
logger.error('Videos is empty {}'.format(task_message.video_ids))
return False
videos.update(is_stat_fetching=True)
try:
for owner, user_videos in groupby(
videos, lambda video: video.owner if video.owner else video
.user.social_auth.get(provider='vk-oauth2').uid):
user_videos = list(user_videos)
user = user_videos[0].user
social_uid, access_token = self._get_vk_credentials(user)
video_parts = split_list(user_videos, 200)
execute_calls = [
ExecuteMethod.construct(
ExecuteMethod.VIDEO_ENTITY, ExecuteMethod.GET_METHOD,
json.dumps({
VK_MESSAGE_KEY.OWNER_ID:
owner,
VK_MESSAGE_KEY.COUNT:
len(video_part),
VK_MESSAGE_KEY.VIDEOS:
format_videos_ids(owner, video_part)
})) for video_part in video_parts
]
execute_parts = split_list(execute_calls, 25)
video_execute_parts = split_list(video_parts, 25)
for index_part, execute_part in enumerate(execute_parts):
method = VideoExecuteMethod(
priority=task_message.priority,
access_token=access_token,
code=ExecuteMethod.construct_code(execute_part))
response_dict = await method.execute()
if not response_dict:
traceback_log = traceback.format_exc()
sentry_logger.error(
msg="False response on get stat Video",
extra={
'trace': traceback_log,
'locals': locals()
})
continue
response_list = response_dict.get(VK_MESSAGE_KEY.RESPONSE)
if VK_MESSAGE_KEY.ERROR in response_dict or VK_MESSAGE_KEY.EXECUTE_ERRORS in response_dict:
sentry_error = {'error': response_dict}
sentry_logger.warning(msg=sentry_error)
for index, videos_err in enumerate(
video_execute_parts[index_part]):
for video in videos_err:
video.is_stat_fetching = False
video.last_stat_fetch_date = datetime.now(
tzlocal())
video.save()
else:
video_execute_part = video_execute_parts[index_part]
for index, video_part in enumerate(video_execute_part):
if response_list[index]:
video_get_array = response_list[index]
# первый элемент - количество видео в одном видео гет
video_get_array.pop(0)
for video_part_index, video in enumerate(
video_part):
video.is_stat_fetching = False
video_array = []
if video_get_array:
video_array = list(
filter(
lambda vid_dict: vid_dict.get(
'vid') == video.vid,
video_get_array))
if len(video_array):
video_dict = video_array[0]
diff = video_dict["views"] - video.views
try:
if SuspiciousUser.objects.filter(
user=video.user,
campaign=video.campaign
).count() > 0:
views_tags = {
'type':
'views',
'campaign_pk':
video.campaign.pk,
'user_pk':
video.user.pk,
'user':
translit(str(video.user),
'ru',
reversed=True),
'group_pk':
video.group.pk,
'group_name':
translit(video.group.name,
'ru',
reversed=True)
}
StatsClient.event(
'video-seed__suspicious_users',
value=diff,
tags=views_tags)
except Exception as e:
sentry_logger.error(msg=e)
views = video_dict["views"]
video.views = views
video.is_stat_fetching = False
video.last_stat_fetch_date = datetime.now(
tzlocal())
video.save()
today = timezone.make_aware(
datetime.now(),
timezone.get_default_timezone()
).date()
views_stat, _ = ViewsStat.objects.get_or_create(
group=video.group,
campaign=video.campaign,
date__contains=today,
user=video.user)
if diff:
views_stat.views += diff
views_stat.save()
except Exception as e:
traceback_log = traceback.format_exc()
sentry_logger.error(msg=e,
extra={
'trace': traceback_log,
'locals': locals()
})
videos.update(is_stat_fetching=False,
last_stat_fetch_date=timezone.make_aware(
datetime.now(), timezone.get_default_timezone()))
return True