def map(exp_model):
if not feconf.DEV_MODE:
exp_id = exp_model.id
fs_old = fs_domain.AbstractFileSystem(
fs_domain.GcsFileSystem(exp_id))
# We have to make sure we pass the dir name without starting or
# ending with '/'.
image_urls = fs_old.listdir('image')
audio_urls = fs_old.listdir('audio')
image_filenames = [
GCS_IMAGE_ID_REGEX.match(url).group(3) for url in image_urls]
audio_filenames = [
GCS_AUDIO_ID_REGEX.match(url).group(3) for url in audio_urls]
for image_filename in image_filenames:
raw_image = fs_old.get('image/%s' % image_filename)
height, width = gae_image_services.get_image_dimensions(
raw_image)
filename_with_dimensions = (
html_validation_service.regenerate_image_filename_using_dimensions( # pylint: disable=line-too-long
image_filename, height, width))
exp_services.save_original_and_compressed_versions_of_image(
'ADMIN', filename_with_dimensions, exp_id, raw_image)
for audio_filename in audio_filenames:
filetype = audio_filename[audio_filename.rfind('.') + 1:]
raw_data = fs_old.get('audio/%s' % audio_filename)
fs = fs_domain.AbstractFileSystem(
fs_domain.GcsFileSystem('exploration/%s' % exp_id))
fs.commit(
'Admin', 'audio/%s' % audio_filename,
raw_data, mimetype='audio/%s' % filetype)
yield (ADDED_COMPRESSED_VERSIONS_OF_IMAGES, exp_id)
def map(exp_model):
if not constants.DEV_MODE:
exp_id = exp_model.id
fs_old = fs_domain.AbstractFileSystem(
fs_domain.GcsFileSystem(exp_id))
# We have to make sure we pass the dir name without starting or
# ending with '/'.
image_urls = fs_old.listdir('image')
audio_urls = fs_old.listdir('audio')
image_filenames = [
GCS_IMAGE_ID_REGEX.match(url).group(3) for url in image_urls]
audio_filenames = [
GCS_AUDIO_ID_REGEX.match(url).group(3) for url in audio_urls]
references_unicode_files = False
for image_filename in image_filenames:
try:
fs_old.get('image/%s' % image_filename)
except Exception:
references_unicode_files = True
break
if references_unicode_files:
image_filenames_str = '%s' % image_filenames
yield (
EXP_REFERENCES_UNICODE_FILES,
'Exp: %s, image filenames: %s' % (
exp_id, image_filenames_str.encode('utf-8')))
for image_filename in image_filenames:
try:
raw_image = fs_old.get(
'image/%s' % image_filename.encode('utf-8'))
height, width = gae_image_services.get_image_dimensions(
raw_image)
filename_with_dimensions = (
html_validation_service.regenerate_image_filename_using_dimensions( # pylint: disable=line-too-long
image_filename, height, width))
exp_services.save_original_and_compressed_versions_of_image( # pylint: disable=line-too-long
'ADMIN', filename_with_dimensions, exp_id, raw_image)
yield ('Copied file', 1)
except Exception:
error = traceback.format_exc()
logging.error(
'File %s in %s failed migration: %s' %
(image_filename.encode('utf-8'), exp_id, error))
yield (
ERROR_IN_FILENAME, 'Error when copying %s in %s: %s' % (
image_filename.encode('utf-8'), exp_id, error))
for audio_filename in audio_filenames:
filetype = audio_filename[audio_filename.rfind('.') + 1:]
raw_data = fs_old.get('audio/%s' % audio_filename)
fs = fs_domain.AbstractFileSystem(
fs_domain.GcsFileSystem('exploration/%s' % exp_id))
fs.commit(
'Admin', 'audio/%s' % audio_filename,
raw_data, mimetype='audio/%s' % filetype)
yield (ADDED_COMPRESSED_VERSIONS_OF_IMAGES, exp_id)
def map(exp_model):
if exp_model.deleted:
return
if not constants.DEV_MODE:
exp_id = exp_model.id
fs_internal = fs_domain.AbstractFileSystem(
fs_domain.GcsFileSystem(exp_id))
fs_external = fs_domain.AbstractFileSystem(
fs_domain.GcsFileSystem('exploration/' + exp_id))
# We have to make sure we pass the dir name without starting or
# ending with '/'.
image_urls_internal = fs_internal.listdir('image')
image_urls_external = fs_external.listdir('image')
image_filenames_internal = set([
GCS_IMAGE_ID_REGEX.match(url).group(3)
for url in image_urls_internal])
image_filenames_external = set([
GCS_EXTERNAL_IMAGE_ID_REGEX.match(url).group(3)
for url in image_urls_external])
image_filenames_internal_with_dimensions = set([])
for image_name in image_filenames_internal:
raw_image = fs_internal.get(
'image/%s' % image_name.encode('utf-8'))
height, width = gae_image_services.get_image_dimensions(
raw_image)
image_filenames_internal_with_dimensions.add(
html_validation_service.regenerate_image_filename_using_dimensions( # pylint: disable=line-too-long
image_name, height, width))
# Currently in editor.py and exp_services.py, all images are stored
# in exploration/{{exp_id}}/ external folder. So, we only need to
# check if all images in the internal folder are there in the
# external folder.
non_existent_images = (
image_filenames_internal_with_dimensions.difference(
image_filenames_external))
if len(non_existent_images) > 0:
yield (
exp_id,
'Missing Images: %s' % list(non_existent_images))
else:
yield (
ALL_IMAGES_VERIFIED, len(image_filenames_external))
def mock_get_filename_with_dimensions(filename, unused_exp_id):
return html_validation_service.regenerate_image_filename_using_dimensions(
filename, 490, 120)
def test_regenerate_image_filename_using_dimensions(self):
regenerated_name = (
html_validation_service.regenerate_image_filename_using_dimensions(
self.FILENAME, self.HEIGHT, self.WIDTH))
self.assertEqual(regenerated_name, 'abc_height_45_width_45.png')
