def test_listdir(self) -> None:
self.assertItemsEqual(self.fs.listdir(''),
[]) # type: ignore[no-untyped-call]
self.fs.commit('abc.png', b'file_contents')
self.fs.commit('abcd.png', b'file_contents_2')
self.fs.commit('abc/abcd.png', b'file_contents_3')
self.fs.commit('bcd/bcde.png', b'file_contents_4')
file_names = ['abc.png', 'abc/abcd.png', 'abcd.png', 'bcd/bcde.png']
self.assertItemsEqual(self.fs.listdir(''),
file_names) # type: ignore[no-untyped-call]
self.assertEqual(self.fs.listdir('abc'), ['abc/abcd.png'])
with self.assertRaisesRegex(
IOError, 'Invalid filepath'): # type: ignore[no-untyped-call]
self.fs.listdir('/abc')
with self.assertRaisesRegex( # type: ignore[no-untyped-call]
IOError, ('The dir_name should not start with /'
' or end with / : abc/')):
self.fs.listdir('abc/')
self.assertEqual(self.fs.listdir('fake_dir'), [])
new_fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
'eid2')
self.assertEqual(new_fs.listdir('assets'), [])
def test_invalid_filepaths_are_caught(self) -> None:
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, 'eid')
invalid_filepaths = [
'..', '../another_exploration', '../', '/..', '/abc'
]
for filepath in invalid_filepaths:
with self.assertRaisesRegex(
IOError,
'Invalid filepath'): # type: ignore[no-untyped-call]
fs.isfile(filepath)
with self.assertRaisesRegex(
IOError,
'Invalid filepath'): # type: ignore[no-untyped-call]
fs.get(filepath)
with self.assertRaisesRegex(
IOError,
'Invalid filepath'): # type: ignore[no-untyped-call]
fs.commit(filepath, b'raw_file')
with self.assertRaisesRegex(
IOError,
'Invalid filepath'): # type: ignore[no-untyped-call]
fs.delete(filepath)
with self.assertRaisesRegex(
IOError,
'Invalid filepath'): # type: ignore[no-untyped-call]
fs.listdir(filepath)
def test_save_original_and_compressed_versions_of_svg_image(self) -> None:
with utils.open_file(os.path.join(feconf.TESTS_DATA_DIR,
'test_svg.svg'),
'rb',
encoding=None) as f:
image_content = f.read()
with self.swap(constants, 'DEV_MODE', False):
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
self.EXPLORATION_ID)
self.assertFalse(fs.isfile('image/%s' % self.FILENAME))
self.assertFalse(
fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME))
self.assertFalse(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
fs_services.save_original_and_compressed_versions_of_image(
self.FILENAME, 'exploration', self.EXPLORATION_ID,
image_content, 'image', False)
self.assertTrue(fs.isfile('image/%s' % self.FILENAME))
self.assertTrue(
fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME))
self.assertTrue(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
original_image_content = fs.get('image/%s' % self.FILENAME)
compressed_image_content = fs.get('image/%s' %
self.COMPRESSED_IMAGE_FILENAME)
micro_image_content = fs.get('image/%s' %
self.MICRO_IMAGE_FILENAME)
self.assertEqual(original_image_content, image_content)
self.assertEqual(compressed_image_content, image_content)
self.assertEqual(micro_image_content, image_content)
def post(self, entity_type, entity_id):
"""Saves an image uploaded by a content creator."""
raw = self.normalized_request.get('image')
filename = self.normalized_payload.get('filename')
filename_prefix = self.normalized_payload.get('filename_prefix')
try:
file_format = image_validation_services.validate_image_and_filename(
raw, filename)
except utils.ValidationError as e:
raise self.InvalidInputException(e)
fs = fs_services.GcsFileSystem(entity_type, entity_id)
filepath = '%s/%s' % (filename_prefix, filename)
if fs.isfile(filepath):
raise self.InvalidInputException(
'A file with the name %s already exists. Please choose a '
'different name.' % filename)
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, entity_type, entity_id, raw, filename_prefix,
image_is_compressible)
self.render_json({'filename': filename})
def test_audio_upload_with_non_mp3_file(self):
self.login(self.EDITOR_EMAIL)
csrf_token = self.get_new_csrf_token()
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, '0')
with utils.open_file(os.path.join(feconf.TESTS_DATA_DIR,
self.TEST_AUDIO_FILE_FLAC),
'rb',
encoding=None) as f:
raw_audio = f.read()
self.assertFalse(fs.isfile('audio/%s' % self.TEST_AUDIO_FILE_FLAC))
with self.accepted_audio_extensions_swap:
self.post_json('%s/0' % self.AUDIO_UPLOAD_URL_PREFIX,
{'filename': self.TEST_AUDIO_FILE_FLAC},
csrf_token=csrf_token,
upload_files=[('raw_audio_file',
self.TEST_AUDIO_FILE_FLAC, raw_audio)
])
self.assertTrue(fs.isfile('audio/%s' % self.TEST_AUDIO_FILE_FLAC))
self.logout()
def validate_svg_filenames_in_math_rich_text(
entity_type, entity_id, html_string):
"""Validates the SVG filenames for each math rich-text components and
returns a list of all invalid math tags in the given HTML.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
html_string: str. The HTML string.
Returns:
list(str). A list of invalid math tags in the HTML string.
"""
soup = bs4.BeautifulSoup(html_string, 'html.parser')
error_list = []
for math_tag in soup.findAll(name='oppia-noninteractive-math'):
math_content_dict = (
json.loads(unescape_html(
math_tag['math_content-with-value'])))
svg_filename = (
objects.UnicodeString.normalize(math_content_dict['svg_filename']))
if svg_filename == '':
error_list.append(str(math_tag))
else:
fs = fs_services.GcsFileSystem(entity_type, entity_id)
filepath = 'image/%s' % svg_filename
if not fs.isfile(filepath):
error_list.append(str(math_tag))
return error_list
def setUp(self) -> None:
super(GcsFileSystemUnitTests, self).setUp()
self.USER_EMAIL = '*****@*****.**'
self.signup(self.USER_EMAIL, 'username')
self.user_id = self.get_user_id_from_email(
self.USER_EMAIL) # type: ignore[no-untyped-call]
self.fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
'eid')
def test_copy(self) -> None:
self.fs.commit('abc2.png', b'file_contents')
self.assertEqual(self.fs.listdir(''), ['abc2.png'])
destination_fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_QUESTION,
'question_id1')
self.assertEqual(destination_fs.listdir(''), [])
destination_fs.copy(self.fs.assets_path, 'abc2.png')
self.assertTrue(destination_fs.isfile('abc2.png'))
def test_validate_entity_parameters(self) -> None:
with self.assertRaisesRegex( # type: ignore[no-untyped-call]
utils.ValidationError, 'Invalid entity_id received: 1'):
# The argument `entity_id` of GcsFileSystem() can only accept
# string values, but here for testing purpose we are providing
# integer value. Thus to silent incompatible argument type MyPy
# error, we added an ignore statement here.
fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
1) # type: ignore[arg-type]
with self.assertRaisesRegex( # type: ignore[no-untyped-call]
utils.ValidationError, 'Entity id cannot be empty'):
fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, '')
with self.assertRaisesRegex( # type: ignore[no-untyped-call]
utils.ValidationError, 'Invalid entity_name received: '
'invalid_name.'):
fs_services.GcsFileSystem('invalid_name', 'exp_id')
def test_save_and_get_classifier_data(self) -> None:
"""Test that classifier data is stored and retrieved correctly."""
fs_services.save_classifier_data('exp_id', 'job_id',
self.classifier_data_proto)
filepath = 'job_id-classifier-data.pb.xz'
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
'exp_id')
classifier_data = utils.decompress_from_zlib(fs.get(filepath))
classifier_data_proto = text_classifier_pb2.TextClassifierFrozenModel()
classifier_data_proto.ParseFromString(classifier_data)
self.assertEqual(classifier_data_proto.model_json,
self.classifier_data_proto.model_json)
def setUp(self) -> None:
super(FileSystemClassifierDataTests, self).setUp()
self.fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
'exp_id')
self.classifier_data_proto = (
text_classifier_pb2.TextClassifierFrozenModel())
self.classifier_data_proto.model_json = json.dumps({
'param1':
40,
'param2': [34.2, 54.13, 95.23],
'submodel': {
'param1': 12
}
})
def test_save_original_and_compressed_versions_of_image(self) -> None:
with utils.open_file(os.path.join(feconf.TESTS_DATA_DIR, 'img.png'),
'rb',
encoding=None) as f:
original_image_content = f.read()
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
self.EXPLORATION_ID)
self.assertFalse(fs.isfile('image/%s' % self.FILENAME))
self.assertFalse(fs.isfile('image/%s' %
self.COMPRESSED_IMAGE_FILENAME))
self.assertFalse(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
fs_services.save_original_and_compressed_versions_of_image(
self.FILENAME, 'exploration', self.EXPLORATION_ID,
original_image_content, 'image', True)
self.assertTrue(fs.isfile('image/%s' % self.FILENAME))
self.assertTrue(fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME))
self.assertTrue(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
def get(self, page_context, page_identifier, asset_type, encoded_filename):
"""Returns an asset file.
Args:
page_context: str. The context of the page where the asset is
required.
page_identifier: str. The unique identifier for the particular
context. Valid page_context: page_identifier pairs:
exploration: exp_id
story: story_id
topic: topic_id
skill: skill_id
subtopic: topic_name of the topic that it is part of.
asset_type: str. Type of the asset, either image or audio.
encoded_filename: str. The asset filename. This
string is encoded in the frontend using encodeURIComponent().
"""
if not constants.EMULATOR_MODE:
raise self.PageNotFoundException
try:
filename = urllib.parse.unquote(encoded_filename)
file_format = filename[(filename.rfind('.') + 1):]
# If the following is not cast to str, an error occurs in the wsgi
# library because unicode gets used.
content_type = ('image/svg+xml' if file_format == 'svg' else
'%s/%s' % (asset_type, file_format))
self.response.headers['Content-Type'] = content_type
if page_context not in self._SUPPORTED_PAGE_CONTEXTS:
raise self.InvalidInputException
fs = fs_services.GcsFileSystem(page_context, page_identifier)
raw = fs.get('%s/%s' % (asset_type, filename))
self.response.cache_control.no_cache = None
self.response.cache_control.public = True
self.response.cache_control.max_age = 600
self.response.body_file = io.BytesIO(raw)
except Exception as e:
logging.exception('File not found: %s. %s' % (encoded_filename, e))
raise self.PageNotFoundException
def test_compress_image_on_prod_mode_with_small_image_size(self) -> None:
with utils.open_file(os.path.join(feconf.TESTS_DATA_DIR, 'img.png'),
'rb',
encoding=None) as f:
original_image_content = f.read()
with self.swap(constants, 'DEV_MODE', False):
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
self.EXPLORATION_ID)
self.assertFalse(fs.isfile('image/%s' % self.FILENAME))
self.assertFalse(
fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME))
self.assertFalse(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
fs_services.save_original_and_compressed_versions_of_image(
self.FILENAME, 'exploration', self.EXPLORATION_ID,
original_image_content, 'image', True)
self.assertTrue(fs.isfile('image/%s' % self.FILENAME))
self.assertTrue(
fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME))
self.assertTrue(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
original_image_content = fs.get('image/%s' % self.FILENAME)
compressed_image_content = fs.get('image/%s' %
self.COMPRESSED_IMAGE_FILENAME)
micro_image_content = fs.get('image/%s' %
self.MICRO_IMAGE_FILENAME)
self.assertEqual(
image_services.get_image_dimensions(original_image_content),
(32, 32))
self.assertEqual(
image_services.get_image_dimensions(compressed_image_content),
(25, 25))
self.assertEqual(
image_services.get_image_dimensions(micro_image_content),
(22, 22))
def post(self):
"""Generates structures for Android end-to-end tests.
This handler generates structures for Android end-to-end tests in
order to evaluate the integration of network requests from the
Android client to the backend. This handler should only be called
once (or otherwise raises an exception), and can only be used in
development mode (this handler is unavailable in production).
Note that the handler outputs an empty JSON dict when the request is
successful.
The specific structures that are generated:
Topic: A topic with both a test story and a subtopic.
Story: A story with 'android_interactions' as a exploration
node.
Exploration: 'android_interactions' from the local assets.
Subtopic: A dummy subtopic to validate the topic.
Skill: A dummy skill to validate the subtopic.
Raises:
Exception. When used in production mode.
InvalidInputException. The topic is already
created but not published.
InvalidInputException. The topic is already published.
"""
if not constants.DEV_MODE:
raise Exception('Cannot load new structures data in production.')
if topic_services.does_topic_with_name_exist('Android test'):
topic = topic_fetchers.get_topic_by_name('Android test')
topic_rights = topic_fetchers.get_topic_rights(topic.id,
strict=False)
if topic_rights.topic_is_published:
raise self.InvalidInputException(
'The topic is already published.')
raise self.InvalidInputException(
'The topic exists but is not published.')
exp_id = '26'
user_id = feconf.SYSTEM_COMMITTER_ID
# Generate new Structure id for topic, story, skill and question.
topic_id = topic_fetchers.get_new_topic_id()
story_id = story_services.get_new_story_id()
skill_id = skill_services.get_new_skill_id()
question_id = question_services.get_new_question_id()
# Create dummy skill and question.
skill = self._create_dummy_skill(skill_id, 'Dummy Skill for Android',
'<p>Dummy Explanation 1</p>')
question = self._create_dummy_question(question_id, 'Question 1',
[skill_id])
question_services.add_question(user_id, question)
question_services.create_new_question_skill_link(
user_id, question_id, skill_id, 0.3)
# Create and update topic to validate before publishing.
topic = topic_domain.Topic.create_default_topic(
topic_id, 'Android test', 'test-topic-one', 'description', 'fragm')
topic.update_url_fragment('test-topic')
topic.update_meta_tag_content('tag')
topic.update_page_title_fragment_for_web('page title for topic')
# Save the dummy image to the filesystem to be used as thumbnail.
with utils.open_file(os.path.join(feconf.TESTS_DATA_DIR,
'test_svg.svg'),
'rb',
encoding=None) as f:
raw_image = f.read()
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_TOPIC, topic_id)
fs.commit('%s/test_svg.svg' % (constants.ASSET_TYPE_THUMBNAIL),
raw_image,
mimetype='image/svg+xml')
# Update thumbnail properties.
topic_services.update_thumbnail_filename(topic, 'test_svg.svg')
topic.update_thumbnail_bg_color('#C6DCDA')
# Add other structures to the topic.
topic.add_canonical_story(story_id)
topic.add_uncategorized_skill_id(skill_id)
topic.add_subtopic(1, 'Test Subtopic Title', 'testsubtop')
# Update and validate subtopic.
topic_services.update_subtopic_thumbnail_filename(
topic, 1, 'test_svg.svg')
topic.update_subtopic_thumbnail_bg_color(1, '#FFFFFF')
topic.update_subtopic_url_fragment(1, 'suburl')
topic.move_skill_id_to_subtopic(None, 1, skill_id)
subtopic_page = (
subtopic_page_domain.SubtopicPage.create_default_subtopic_page(
1, topic_id))
# Upload local exploration to the datastore and enable feedback.
exp_services.load_demo(exp_id)
rights_manager.release_ownership_of_exploration(
user_services.get_system_user(), exp_id)
exp_services.update_exploration(
user_id, exp_id, [
exp_domain.ExplorationChange({
'cmd': exp_domain.CMD_EDIT_EXPLORATION_PROPERTY,
'property_name': 'correctness_feedback_enabled',
'new_value': True
})
], 'Changed correctness_feedback_enabled.')
# Add and update the exploration/node to the story.
story = story_domain.Story.create_default_story(
story_id, 'Android End to End testing', 'Description', topic_id,
'android-end-to-end-testing')
story.add_node('%s%d' % (story_domain.NODE_ID_PREFIX, 1),
'Testing with UI Automator')
story.update_node_description(
'%s%d' % (story_domain.NODE_ID_PREFIX, 1),
'To test all Android interactions')
story.update_node_exploration_id(
'%s%d' % (story_domain.NODE_ID_PREFIX, 1), exp_id)
# Save the dummy image to the filesystem to be used as thumbnail.
with utils.open_file(os.path.join(feconf.TESTS_DATA_DIR,
'test_svg.svg'),
'rb',
encoding=None) as f:
raw_image = f.read()
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_STORY, story_id)
fs.commit('%s/test_svg.svg' % (constants.ASSET_TYPE_THUMBNAIL),
raw_image,
mimetype='image/svg+xml')
story.update_node_thumbnail_filename(
'%s%d' % (story_domain.NODE_ID_PREFIX, 1), 'test_svg.svg')
story.update_node_thumbnail_bg_color(
'%s%d' % (story_domain.NODE_ID_PREFIX, 1), '#F8BF74')
# Update and validate the story.
story.update_meta_tag_content('tag')
story.update_thumbnail_filename('test_svg.svg')
story.update_thumbnail_bg_color(
constants.ALLOWED_THUMBNAIL_BG_COLORS['story'][0])
# Save the previously created structures
# (skill, story, topic, subtopic).
skill_services.save_new_skill(user_id, skill)
story_services.save_new_story(user_id, story)
topic_services.save_new_topic(user_id, topic)
subtopic_page_services.save_subtopic_page(
user_id, subtopic_page, 'Added subtopic', [
topic_domain.TopicChange({
'cmd': topic_domain.CMD_ADD_SUBTOPIC,
'subtopic_id': 1,
'title': 'Dummy Subtopic Title',
'url_fragment': 'dummy-fragment'
})
])
# Generates translation opportunities for the Contributor Dashboard.
exp_ids_in_story = story.story_contents.get_all_linked_exp_ids()
opportunity_services.add_new_exploration_opportunities(
story_id, exp_ids_in_story)
# Publish the story and topic.
topic_services.publish_story(topic_id, story_id, user_id)
topic_services.publish_topic(topic_id, user_id)
# Upload thumbnails to be accessible through AssetsDevHandler.
self._upload_thumbnail(topic_id, feconf.ENTITY_TYPE_TOPIC)
self._upload_thumbnail(story_id, feconf.ENTITY_TYPE_STORY)
self.render_json({})
def post(self, exploration_id):
"""Saves an audio file uploaded by a content creator."""
raw_audio_file = self.request.get('raw_audio_file')
filename = self.payload.get('filename')
allowed_formats = list(feconf.ACCEPTED_AUDIO_EXTENSIONS.keys())
if not raw_audio_file:
raise self.InvalidInputException('No audio supplied')
dot_index = filename.rfind('.')
extension = filename[dot_index + 1:].lower()
if dot_index in (-1, 0):
raise self.InvalidInputException(
'No filename extension: it should have '
'one of the following extensions: %s' % allowed_formats)
if extension not in feconf.ACCEPTED_AUDIO_EXTENSIONS:
raise self.InvalidInputException(
'Invalid filename extension: it should have '
'one of the following extensions: %s' % allowed_formats)
tempbuffer = io.BytesIO()
tempbuffer.write(raw_audio_file)
tempbuffer.seek(0)
try:
# For every accepted extension, use the mutagen-specific
# constructor for that type. This will catch mismatched audio
# types e.g. uploading a flac file with an MP3 extension.
if extension == 'mp3':
audio = mp3.MP3(tempbuffer)
else:
audio = mutagen.File(tempbuffer)
except mutagen.MutagenError as e:
# The calls to mp3.MP3() versus mutagen.File() seem to behave
# differently upon not being able to interpret the audio.
# mp3.MP3() raises a MutagenError whereas mutagen.File()
# seems to return None. It's not clear if this is always
# the case. Occasionally, mutagen.File() also seems to
# raise a MutagenError.
raise self.InvalidInputException(
'Audio not recognized as a %s file' % extension) from e
tempbuffer.close()
if audio is None:
raise self.InvalidInputException(
'Audio not recognized as a %s file' % extension)
if audio.info.length > feconf.MAX_AUDIO_FILE_LENGTH_SEC:
raise self.InvalidInputException(
'Audio files must be under %s seconds in length. The uploaded '
'file is %.2f seconds long.' %
(feconf.MAX_AUDIO_FILE_LENGTH_SEC, audio.info.length))
if len(
set(audio.mime).intersection(
set(feconf.ACCEPTED_AUDIO_EXTENSIONS[extension]))) == 0:
raise self.InvalidInputException(
'Although the filename extension indicates the file '
'is a %s file, it was not recognized as one. '
'Found mime types: %s' % (extension, audio.mime))
mimetype = audio.mime[0]
# Fetch the audio file duration from the Mutagen metadata.
duration_secs = audio.info.length
# For a strange, unknown reason, the audio variable must be
# deleted before opening cloud storage. If not, cloud storage
# throws a very mysterious error that entails a mutagen
# object being recursively passed around in app engine.
del audio
# Audio files are stored to the datastore in the dev env, and to GCS
# in production.
fs = fs_services.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION,
exploration_id)
fs.commit('%s/%s' % (self._FILENAME_PREFIX, filename),
raw_audio_file,
mimetype=mimetype)
self.render_json({
'filename': filename,
'duration_secs': duration_secs
})
