def update_all_videos(): """ Updates all videos with YouTube data """ batch_size = 50 qs = (YouTubeVideo.objects.order_by('modified').values_list('pk', flat=True)) start = 0 video_ids = () while len(video_ids) == batch_size or start == 0: video_ids = list(qs[start:start + batch_size]) deferred_manager.defer(update_videos, video_ids, _queue='update-youtube-video-data') if settings.SEARCH_INDEXING: project_video_ids = list( Video.objects.filter( youtube_video_id__in=video_ids).values_list('pk', flat=True)) deferred_manager.defer(index_all, model_type="Video", object_ids=project_video_ids, _queue='search-indexing') start += batch_size
def delete_user_search_document(sender, instance, **kwargs): """ Triggers a task to delete the User document """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_auto_complete_user, instance.email, _queue='search-indexing', _countdown=5)
def process_user_search_document(sender, instance, created, raw, **kwargs): """ Triggers a task to index the user """ if not raw and settings.SEARCH_INDEXING: deferred_manager.defer(index_auto_complete_user, instance.email, _queue='search-indexing', _countdown=2)
def delete_pending_user_search_document(sender, instance, **kwargs): """ Re-index the autocomplete object here because when a PendingUser is deleted there should be a User object with the same email """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_auto_complete_user, instance.email, _queue='search-indexing', _countdown=2)
def process_video_search_document(sender, instance, created, raw, **kwargs): """ Triggers a task to index the Video """ if not raw and settings.SEARCH_INDEXING: deferred_manager.defer(index_video_document, instance.pk, task_reference="video-{0}".format(instance.pk), unique_until=x_minutes(10), _queue='search-indexing')
def user_video_detail_deleted(sender, instance, **kwargs): """ Triggers a task to denormalise data onto the video object """ deferred_manager.defer(denormalise_video, instance.video_id, task_reference="video-denorm-{0}".format( instance.video_id), unique_until=x_minutes(2), _queue='denormalising', _countdown=2)
def defer_delete_user(user, deleted_by_user=None, email_user=True): """ Calls :func:`greenday_core.user_deletion.delete_user <greenday_core.user_deletion.delete_user>` via a deferred task """ deferred_manager.defer( delete_user, user.pk, deleted_by_user_id=deleted_by_user.pk if deleted_by_user else None, email_user=email_user, task_reference="user-{0:d}".format(user.pk), unique_until=timezone.now() + datetime.timedelta(hours=2), _queue="user-deletion")
def process_project_tag_search_document(sender, instance, created, raw, **kwargs): """ Triggers a task to index the related GlobalTag """ if not raw and settings.SEARCH_INDEXING: deferred_manager.defer(index_global_tag_document, instance.global_tag_id, task_reference="globaltag-{0}".format( instance.global_tag_id), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2)
def delete_video(sender, instance, **kwargs): """ If the video's YouTubeVideo is no longer related to any other videos then delete it """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_video_document, instance.pk, task_reference="video-{0}".format(instance.pk), unique_until=x_minutes(10), _queue='search-indexing') if not instance.youtube_video.projectvideos.count(): instance.youtube_video.delete()
def global_tag_deleted(sender, instance, **kwargs): """ Triggers a task to delete the GlobaTag document """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_global_tag_document, instance.pk, task_reference="globaltag-{0}".format( instance.pk), _queue='search-indexing', _countdown=2) manager = ImageManager('tag') manager.delete_linked_image_for_model(instance.pk)
def project_user_reprocess_project_search_document(sender, instance, *args, **kwargs): """ Triggers re-processing of project index when ProjectUser instances are created, updated or deleted. """ raw = kwargs.pop('raw', None) if not raw and settings.SEARCH_INDEXING: deferred_manager.defer(index_project_document, instance.project_id, task_reference="project-{0}".format( instance.project_id), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2)
def project_deleted(sender, instance, **kwargs): """ Triggers a task to delete the Project document """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_project_document, instance.pk, task_reference="project-{0}".format( instance.pk), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2) manager = ImageManager('project') manager.delete_linked_image_for_model(instance.pk)
def timed_video_comment_reprocess_video_search_document( sender, instance, *args, **kwargs): """ Triggers re-processing of video index when TimedVideoComment instances are created, updated or deleted. """ raw = kwargs.pop('raw', None) if not raw and settings.SEARCH_INDEXING: deferred_manager.defer(index_video_document, instance.video_id, task_reference="video-{0}".format( instance.video_id), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2)
def delete_project_tag(sender, instance, **kwargs): """ Triggers a task to index the related GlobalTag """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_global_tag_document, instance.global_tag_id, task_reference="globaltag-{0}".format( instance.global_tag_id), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2) if (instance.global_tag.created_from_project == instance.project and instance.project.tags_is_private): instance.global_tag.delete()
def global_tag_saved(sender, instance, created, raw, **kwargs): """ Triggers a task to index the GlobalTag """ if not raw: if settings.SEARCH_INDEXING: deferred_manager.defer(index_global_tag_document, instance.pk, task_reference="globaltag-{0}".format( instance.pk), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2) manager = ImageManager('tag') manager.update_linked_image_for_model(instance.image_url, instance.pk)
def video_tag_instance_deleted(sender, instance, **kwargs): """ Triggers a task to index the Video related to the VideoTagInstance """ if settings.SEARCH_INDEXING: deferred_manager.defer(index_video_document, instance.video_tag.video_id, task_reference="video-{0}".format( instance.video_tag.video_id), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2) global_tag_id = instance.video_tag.project_tag.global_tag_id deferred_manager.defer( index_global_tag_document, global_tag_id, task_reference="globaltag-{0}".format(global_tag_id), unique_until=x_minutes(10), _queue='search-indexing', _countdown=2) deferred_manager.defer(denormalise_project, instance.video_tag.project_id, task_reference="project-denorm-{0}".format( instance.video_tag.project_id), unique_until=x_minutes(2), _queue='denormalising', _countdown=2) deferred_manager.defer(denormalise_video, instance.video_tag.video_id, task_reference="video-denorm-{0}".format( instance.video_tag.video_id), unique_until=x_minutes(2), _queue='denormalising', _countdown=2)