def add_missing_objects_to_db(youtube_ids_in_filesystem,
videos_marked_at_all):
# Files that exist, but are not in the DB, should be assumed to be good videos,
# and just needing to be added to the DB. Add them to the DB in this way,
# so that these files also trigger the update code below (and trigger cache invalidation)
youtube_ids_needing_model_creation = list(
youtube_ids_in_filesystem - videos_marked_at_all)
new_video_files = []
if youtube_ids_needing_model_creation:
for lang_code, youtube_ids in divide_videos_by_language(
youtube_ids_needing_model_creation).iteritems():
# OK to do bulk_create; cache invalidation triggered via save download
lang_video_files = [
VideoFile(youtube_id=id,
percent_complete=100,
download_in_progress=False,
language=lang_code) for id in youtube_ids
]
VideoFile.objects.bulk_create(lang_video_files)
new_video_files += lang_video_files
caching.invalidate_all_caches(
) # Do this within the loop, to update users ASAP
self.stdout.write(
"Created %d VideoFile models (and marked them as complete, since the files exist)\n"
% len(new_video_files))
return [
i18n.get_video_id(video_file.youtube_id)
for video_file in new_video_files
]
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError(
"This must only be run on distributed servers server.")
lang_code = lcode_to_ietf(options["lang_code"])
software_version = options["software_version"]
logging.info(
"Downloading language pack for lang_code=%s, software_version=%s" %
(lang_code, software_version))
# Download the language pack
try:
if options['file']:
self.start(
_("Using local language pack '%(filepath)s'") %
{"filepath": options['file']})
zip_filepath = options['file']
else:
self.start(
_("Downloading language pack '%(lang_code)s'") %
{"lang_code": lang_code})
zip_filepath = get_language_pack(lang_code,
software_version,
callback=self.cb)
# Unpack into locale directory
self.next_stage(
_("Unpacking language pack '%(lang_code)s'") %
{"lang_code": lang_code})
unpack_language(lang_code, zip_filepath=zip_filepath)
#
self.next_stage(
_("Creating static files for language pack '%(lang_code)s'") %
{"lang_code": lang_code})
update_jsi18n_file(lang_code)
self.next_stage(
_("Moving files to their appropriate local disk locations."))
move_dubbed_video_map(lang_code)
move_exercises(lang_code)
move_srts(lang_code)
move_video_sizes_file(lang_code)
self.next_stage(_("Invalidate caches"))
caching.invalidate_all_caches()
self.complete(
_("Finished processing language pack %(lang_code)s") %
{"lang_code": lang_code})
except Exception as e:
self.cancel(stage_status="error",
notes=_("Error: %(error_msg)s") %
{"error_msg": unicode(e)})
raise
def update_objects_to_be_complete(youtube_ids_in_filesystem):
# Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False
updated_video_ids = []
for chunk in break_into_chunks(youtube_ids_in_filesystem):
video_files_needing_model_update = VideoFile.objects.filter(percent_complete=0, download_in_progress=False, youtube_id__in=chunk)
video_files_needing_model_update.update(percent_complete=100, flagged_for_download=False)
updated_video_ids += [i18n.get_video_id(video_file.youtube_id) for video_file in video_files_needing_model_update]
if updated_video_ids:
caching.invalidate_all_caches()
self.stdout.write("Updated %d VideoFile models (to mark them as complete, since the files exist)\n" % len(updated_video_ids))
return updated_video_ids
def add_missing_objects_to_db(youtube_ids_in_filesystem, videos_marked_at_all):
# Files that exist, but are not in the DB, should be assumed to be good videos,
# and just needing to be added to the DB. Add them to the DB in this way,
# so that these files also trigger the update code below (and trigger cache invalidation)
youtube_ids_needing_model_creation = list(youtube_ids_in_filesystem - videos_marked_at_all)
new_video_files = []
if youtube_ids_needing_model_creation:
for lang_code, youtube_ids in divide_videos_by_language(youtube_ids_needing_model_creation).iteritems():
# OK to do bulk_create; cache invalidation triggered via save download
lang_video_files = [VideoFile(youtube_id=id, percent_complete=100, download_in_progress=False, language=lang_code) for id in youtube_ids]
VideoFile.objects.bulk_create(lang_video_files)
new_video_files += lang_video_files
caching.invalidate_all_caches() # Do this within the loop, to update users ASAP
self.stdout.write("Created %d VideoFile models (and marked them as complete, since the files exist)\n" % len(new_video_files))
return [i18n.get_video_id(video_file.youtube_id) for video_file in new_video_files]
def test_cache_invalidation(self):
"""Create the cache item, then invalidate it and show that it is deleted."""
# Get a random video id
n_videos = len(self.video_cache)
video_id = self.video_cache.keys()[10]#random.choice(self.video_cache.keys())
sys.stdout.write("Testing on video_id = %s\n" % video_id)
video_path = self.video_cache[video_id][0]['path']
# Clean the cache for this item
caching.expire_page(path=video_path, failure_ok=True)
# Create the cache item, and check it
self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
caching.regenerate_all_pages_related_to_videos(video_ids=[video_id])
self.assertTrue(caching.has_cache_key(path=video_path), "expect: Cache key exists after Django Client get")
# Invalidate the cache item, and check it
caching.invalidate_all_caches() # test the convenience function
self.assertTrue(not caching.has_cache_key(path=video_path), "expect: no cache key after expiring the page")
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError("This must only be run on distributed servers server.")
lang_code = lcode_to_ietf(options["lang_code"])
software_version = options["software_version"]
logging.info("Downloading language pack for lang_code=%s, software_version=%s" % (lang_code, software_version))
# Download the language pack
try:
if options['file']:
self.start(_("Using local language pack '%(filepath)s'") % {"filepath": options['file']})
zip_filepath = options['file']
else:
self.start(_("Downloading language pack '%(lang_code)s'") % {"lang_code": lang_code})
zip_filepath = get_language_pack(lang_code, software_version, callback=self.cb)
# Unpack into locale directory
self.next_stage(_("Unpacking language pack '%(lang_code)s'") % {"lang_code": lang_code})
unpack_language(lang_code, zip_filepath=zip_filepath)
#
self.next_stage(_("Creating static files for language pack '%(lang_code)s'") % {"lang_code": lang_code})
update_jsi18n_file(lang_code)
self.next_stage(_("Moving files to their appropriate local disk locations."))
move_dubbed_video_map(lang_code)
move_exercises(lang_code)
move_srts(lang_code)
move_video_sizes_file(lang_code)
self.next_stage(_("Invalidate caches"))
caching.invalidate_all_caches()
self.complete(_("Finished processing language pack %(lang_code)s") % {"lang_code": lang_code})
except Exception as e:
self.cancel(stage_status="error", notes=_("Error: %(error_msg)s") % {"error_msg": unicode(e)})
raise
def update_objects_to_be_complete(youtube_ids_in_filesystem):
# Files that exist, are in the DB, but have percent_complete=0, download_in_progress=False
updated_video_ids = []
for chunk in break_into_chunks(youtube_ids_in_filesystem):
video_files_needing_model_update = VideoFile.objects.filter(
percent_complete=0,
download_in_progress=False,
youtube_id__in=chunk)
video_files_needing_model_update.update(
percent_complete=100, flagged_for_download=False)
updated_video_ids += [
i18n.get_video_id(video_file.youtube_id)
for video_file in video_files_needing_model_update
]
if updated_video_ids:
caching.invalidate_all_caches()
self.stdout.write(
"Updated %d VideoFile models (to mark them as complete, since the files exist)\n"
% len(updated_video_ids))
return updated_video_ids
def test_cache_invalidation(self):
"""Create the cache item, then invalidate it and show that it is deleted."""
# Get a random video id
n_videos = len(self.video_cache)
video_id = self.video_cache.keys()[
10] #random.choice(self.video_cache.keys())
sys.stdout.write("Testing on video_id = %s\n" % video_id)
video_path = self.video_cache[video_id][0]['path']
# Clean the cache for this item
caching.expire_page(path=video_path, failure_ok=True)
# Create the cache item, and check it
self.assertTrue(not caching.has_cache_key(path=video_path),
"expect: no cache key after expiring the page")
caching.regenerate_all_pages_related_to_videos(video_ids=[video_id])
self.assertTrue(caching.has_cache_key(path=video_path),
"expect: Cache key exists after Django Client get")
# Invalidate the cache item, and check it
caching.invalidate_all_caches() # test the convenience function
self.assertTrue(not caching.has_cache_key(path=video_path),
"expect: no cache key after expiring the page")
