def delete_objects_for_missing_videos(youtube_ids_in_filesystem,
videos_marked_at_all):
# VideoFile objects say they're available, but that don't actually exist.
deleted_video_ids = []
videos_flagged_for_download = set([
video.youtube_id for video in VideoFile.objects.filter(
flagged_for_download=True)
])
videos_needing_model_deletion_chunked = break_into_chunks(
videos_marked_at_all - youtube_ids_in_filesystem -
videos_flagged_for_download)
# Disconnect cache-invalidation listener to prevent it from being called multiple times
for chunk in videos_needing_model_deletion_chunked:
video_files_needing_model_deletion = VideoFile.objects.filter(
youtube_id__in=chunk)
deleted_video_ids += [
video_file.youtube_id
for video_file in video_files_needing_model_deletion
]
video_files_needing_model_deletion.delete()
if deleted_video_ids:
caching.invalidate_all_caches()
self.stdout.write(
"Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n"
% len(deleted_video_ids))
return deleted_video_ids
def delete_objects_for_missing_videos(youtube_ids_in_filesystem, videos_marked_at_all):
# VideoFile objects say they're available, but that don't actually exist.
deleted_video_ids = []
videos_flagged_for_download = set([video.youtube_id for video in VideoFile.objects.filter(flagged_for_download=True)])
videos_needing_model_deletion_chunked = break_into_chunks(videos_marked_at_all - youtube_ids_in_filesystem - videos_flagged_for_download)
for chunk in videos_needing_model_deletion_chunked:
video_files_needing_model_deletion = VideoFile.objects.filter(youtube_id__in=chunk)
video_files_needing_model_deletion.delete()
deleted_video_ids += [video_file.video_id for video_file in video_files_needing_model_deletion]
if deleted_video_ids:
self.stdout.write("Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n" % len(deleted_video_ids))
return deleted_video_ids
def update_objects_to_be_complete(youtube_ids_in_filesystem):
# Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False
updated_video_ids = []
for chunk in break_into_chunks(youtube_ids_in_filesystem):
video_files_needing_model_update = VideoFile.objects.filter(percent_complete=0, download_in_progress=False, youtube_id__in=chunk)
video_files_needing_model_update.update(percent_complete=100, flagged_for_download=False)
updated_video_ids += [i18n.get_video_id(video_file.youtube_id) for video_file in video_files_needing_model_update]
if updated_video_ids:
self.stdout.write("Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % len(updated_video_ids))
return updated_video_ids
def update_objects_to_be_complete(youtube_ids_in_filesystem):
# Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False
updated_video_ids = []
for chunk in break_into_chunks(youtube_ids_in_filesystem):
video_files_needing_model_update = VideoFile.objects.filter(percent_complete=0, download_in_progress=False, youtube_id__in=chunk)
video_files_needing_model_update.update(percent_complete=100, flagged_for_download=False)
updated_video_ids += [i18n.get_video_id(video_file.youtube_id) for video_file in video_files_needing_model_update]
if updated_video_ids:
self.stdout.write("Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % len(updated_video_ids))
return updated_video_ids
def delete_objects_for_missing_videos(youtube_ids_in_filesystem, videos_marked_at_all):
# VideoFile objects say they're available, but that don't actually exist.
deleted_video_ids = []
videos_flagged_for_download = set([video.youtube_id for video in VideoFile.objects.filter(flagged_for_download=True)])
videos_needing_model_deletion_chunked = break_into_chunks(videos_marked_at_all - youtube_ids_in_filesystem - videos_flagged_for_download)
# Disconnect cache-invalidation listener to prevent it from being called multiple times
for chunk in videos_needing_model_deletion_chunked:
video_files_needing_model_deletion = VideoFile.objects.filter(youtube_id__in=chunk)
deleted_video_ids += [video_file.youtube_id for video_file in video_files_needing_model_deletion]
video_files_needing_model_deletion.delete()
if deleted_video_ids:
caching.invalidate_all_caches()
self.stdout.write("Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n" % len(deleted_video_ids))
return deleted_video_ids
def start_video_download(request):
"""
API endpoint for launching the videodownload job.
"""
youtube_ids = OrderedSet(simplejson.loads(request.body or "{}").get("youtube_ids", []))
# One query per video (slow)
video_files_to_create = [id for id in youtube_ids if not get_object_or_None(VideoFile, youtube_id=id)]
# OK to do bulk_create; cache invalidation triggered via save download
for lang_code, lang_youtube_ids in divide_videos_by_language(video_files_to_create).iteritems():
VideoFile.objects.bulk_create([VideoFile(youtube_id=id, flagged_for_download=True, language=lang_code) for id in lang_youtube_ids])
# OK to update all, since we're not setting all props above.
# One query per chunk
for chunk in break_into_chunks(youtube_ids):
video_files_needing_model_update = VideoFile.objects.filter(download_in_progress=False, youtube_id__in=chunk).exclude(percent_complete=100)
video_files_needing_model_update.update(percent_complete=0, cancel_download=False, flagged_for_download=True)
force_job("videodownload", _("Download Videos"), locale=request.language)
return JsonResponseMessageSuccess(_("Launched video download process successfully."))
def start_video_download(request):
"""
API endpoint for launching the videodownload job.
"""
youtube_ids = OrderedSet(simplejson.loads(request.raw_post_data or "{}").get("youtube_ids", []))
# One query per video (slow)
video_files_to_create = [id for id in youtube_ids if not get_object_or_None(VideoFile, youtube_id=id)]
# OK to do bulk_create; cache invalidation triggered via save download
for lang_code, lang_youtube_ids in divide_videos_by_language(video_files_to_create).iteritems():
VideoFile.objects.bulk_create([VideoFile(youtube_id=id, flagged_for_download=True, language=lang_code) for id in lang_youtube_ids])
# OK to update all, since we're not setting all props above.
# One query per chunk
for chunk in break_into_chunks(youtube_ids):
video_files_needing_model_update = VideoFile.objects.filter(download_in_progress=False, youtube_id__in=chunk).exclude(percent_complete=100)
video_files_needing_model_update.update(percent_complete=0, cancel_download=False, flagged_for_download=True)
force_job("videodownload", _("Download Videos"), locale=request.language)
return JsonResponseMessageSuccess(_("Launched video download process successfully."))
def delete_objects_for_missing_videos(youtube_ids_in_filesystem,
videos_marked_at_all):
# VideoFile objects say they're available, but that don't actually exist.
deleted_video_ids = []
videos_flagged_for_download = set([
video.youtube_id for video in VideoFile.objects.filter(
flagged_for_download=True)
])
videos_needing_model_deletion_chunked = break_into_chunks(
videos_marked_at_all - youtube_ids_in_filesystem -
videos_flagged_for_download)
for chunk in videos_needing_model_deletion_chunked:
video_files_needing_model_deletion = VideoFile.objects.filter(
youtube_id__in=chunk)
video_files_needing_model_deletion.delete()
deleted_video_ids += [
video_file.video_id
for video_file in video_files_needing_model_deletion
]
if deleted_video_ids:
self.stdout.write(
"Deleted %d VideoFile models (because the videos didn't exist in the filesystem)\n"
% len(deleted_video_ids))
return deleted_video_ids