def get(self, exploration_id, encoded_filepath): """Returns an image. Args: exploration_id: the id of the exploration. encoded_filepath: a string representing the image filepath. This string is encoded in the frontend using encodeURIComponent(). """ try: filepath = urllib.unquote(encoded_filepath) file_format = filepath[(filepath.rfind('.') + 1):] # If the following is not cast to str, an error occurs in the wsgi # library because unicode gets used. self.response.headers['Content-Type'] = str( 'image/%s' % file_format) fs = fs_domain.AbstractFileSystem( fs_domain.ExplorationFileSystem( 'exploration/%s' % exploration_id)) raw = fs.get(filepath) self.response.cache_control.no_cache = None self.response.cache_control.public = True self.response.cache_control.max_age = 600 self.response.write(raw) except: raise self.PageNotFoundException
def get(self, exploration_id, filename): """Returns an audio file. Args: encoded_filepath: a string representing the audio filepath. This string is encoded in the frontend using encodeURIComponent(). """ if not constants.DEV_MODE: raise self.PageNotFoundException file_format = filename[(filename.rfind('.') + 1):] # If the following is not cast to str, an error occurs in the wsgi # library because unicode gets used. self.response.headers['Content-Type'] = str('audio/%s' % file_format) fs = fs_domain.AbstractFileSystem( fs_domain.ExplorationFileSystem('exploration/%s' % exploration_id)) try: raw = fs.get('%s/%s' % (self._AUDIO_PATH_PREFIX, filename)) except: raise self.PageNotFoundException self.response.cache_control.no_cache = None self.response.cache_control.public = True self.response.cache_control.max_age = 600 self.response.write(raw)
def test_save_original_and_compressed_versions_of_svg_image(self): with python_utils.open_file(os.path.join(feconf.TESTS_DATA_DIR, 'test_svg.svg'), 'rb', encoding=None) as f: image_content = f.read() with self.swap(constants, 'DEV_MODE', False): fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, self.EXPLORATION_ID)) self.assertFalse(fs.isfile('image/%s' % self.FILENAME)) self.assertFalse( fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME)) self.assertFalse(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME)) fs_services.save_original_and_compressed_versions_of_image( self.FILENAME, 'exploration', self.EXPLORATION_ID, image_content, 'image', False) self.assertTrue(fs.isfile('image/%s' % self.FILENAME)) self.assertTrue( fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME)) self.assertTrue(fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME)) original_image_content = fs.get('image/%s' % self.FILENAME) compressed_image_content = fs.get('image/%s' % self.COMPRESSED_IMAGE_FILENAME) micro_image_content = fs.get('image/%s' % self.MICRO_IMAGE_FILENAME) self.assertEqual(original_image_content, image_content) self.assertEqual(compressed_image_content, image_content) self.assertEqual(micro_image_content, image_content)
def test_copy_images(self): with python_utils.open_file(os.path.join(feconf.TESTS_DATA_DIR, 'img.png'), 'rb', encoding=None) as f: original_image_content = f.read() fs_services.save_original_and_compressed_versions_of_image( self.FILENAME, 'exploration', self.EXPLORATION_ID, original_image_content, 'image', True) destination_fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(feconf.ENTITY_TYPE_QUESTION, 'question_id1')) self.assertFalse(destination_fs.isfile('image/%s' % self.FILENAME)) self.assertFalse( destination_fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME)) self.assertFalse( destination_fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME)) fs_services.copy_images(feconf.ENTITY_TYPE_EXPLORATION, self.EXPLORATION_ID, feconf.ENTITY_TYPE_QUESTION, 'question_id1', ['image.png']) self.assertTrue(destination_fs.isfile('image/%s' % self.FILENAME)) self.assertTrue( destination_fs.isfile('image/%s' % self.COMPRESSED_IMAGE_FILENAME)) self.assertTrue( destination_fs.isfile('image/%s' % self.MICRO_IMAGE_FILENAME))
def get(self, exploration_id, asset_type, encoded_filename): """Returns an asset file. Args: exploration_id: str. The id of the exploration. asset_type: str. Type of the asset, either image or audio. encoded_filename: str. The asset filename. This string is encoded in the frontend using encodeURIComponent(). """ if not constants.DEV_MODE: raise self.PageNotFoundException if asset_type not in self._SUPPORTED_TYPES: raise Exception('%s is not a supported asset type.' % asset_type) try: filename = urllib.unquote(encoded_filename) file_format = filename[(filename.rfind('.') + 1):] # If the following is not cast to str, an error occurs in the wsgi # library because unicode gets used. self.response.headers['Content-Type'] = str( '%s/%s' % (asset_type, file_format)) fs = fs_domain.AbstractFileSystem( fs_domain.ExplorationFileSystem('exploration/%s' % exploration_id)) raw = fs.get('%s/%s' % (asset_type, filename)) self.response.cache_control.no_cache = None self.response.cache_control.public = True self.response.cache_control.max_age = 600 self.response.write(raw) except: raise self.PageNotFoundException
def map(exp_model): if not constants.DEV_MODE: exp_id = exp_model.id fs_old = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(exp_id)) # We have to make sure we pass the dir name without starting or # ending with '/'. image_urls = fs_old.listdir('image') audio_urls = fs_old.listdir('audio') for url in image_urls: catched_groups = GCS_IMAGE_ID_REGEX.match(url) if not catched_groups: yield (INVALID_GCS_URL, url.encode('utf-8')) else: try: filename = GCS_IMAGE_ID_REGEX.match(url).group(3) if fs_old.isfile('image/%s' % filename.encode('utf-8')): yield (FILE_FOUND_IN_GCS, filename.encode('utf-8')) except Exception: yield (ERROR_IN_FILENAME, url.encode('utf-8')) for url in audio_urls: catched_groups = GCS_AUDIO_ID_REGEX.match(url) if not catched_groups: yield (INVALID_GCS_URL, url)
def setUp(self): super(DatastoreBackedFileSystemUnitTests, self).setUp() self.user_email = '*****@*****.**' self.user_id = self.get_user_id_from_email(self.user_email) self.fs = fs_domain.AbstractFileSystem( fs_domain.DatastoreBackedFileSystem( fs_domain.ENTITY_TYPE_EXPLORATION, 'eid'))
def map(file_model): # This job is allowed to run only in Production environment since it # uses GcsFileSystem which can't be used in Development environment. if feconf.DEV_MODE: return instance_id = file_model.id filetype = instance_id[instance_id.rfind('.') + 1:] # To separate the image entries from the audio entries we get from the # FileSnapshotContentModel. if filetype in ALLOWED_IMAGE_EXTENSIONS: catched_groups = FILE_MODEL_ID_REGEX.match(instance_id) if not catched_groups: yield (WRONG_INSTANCE_ID, instance_id) else: filename = catched_groups.group(2) exploration_id = catched_groups.group(1) content = file_model.content fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(exploration_id)) if fs.isfile('image/%s' % filename): yield (FILE_ALREADY_EXISTS, file_model.id) else: fs.commit('ADMIN', 'image/%s' % filename, content, mimetype='image/%s' % filetype) if not fs.isfile('image/%s' % filename): yield ('Failed to commit file', instance_id) else: yield (FILE_COPIED, 1) else: yield ('Invalid filetype', filetype)
def _convert_state_v32_dict_to_v33_dict(cls, question_id, question_state_dict): """Converts from version 32 to 33. Version 33 adds dimensions to images in the oppia-noninteractive-image tags located inside tabs and collapsible blocks. Args: question_id: str. Question id. question_state_dict: dict. A dict where each key-value pair represents respectively, a state name and a dict used to initalize a State domain object. Returns: dict. The converted question_state_dict. """ file_system_class = fs_services.get_entity_file_system_class() exploration_fs = fs_domain.AbstractFileSystem( file_system_class(feconf.ENTITY_TYPE_QUESTION, question_id)) add_dimensions_to_image_tags = functools.partial( html_validation_service.add_dims_to_img_in_complex_rte, exploration_fs) question_state_dict = state_domain.State.convert_html_fields_in_state( question_state_dict, add_dimensions_to_image_tags) return question_state_dict
def validate_svg_filenames_in_math_rich_text(entity_type, entity_id, html_string): """Validates the SVG filenames for each math rich-text components and returns a list of all invalid math tags in the given HTML. Args: entity_type: str. The type of the entity. entity_id: str. The ID of the entity. html_string: str. The HTML string. Returns: list(str). A list of invalid math tags in the HTML string. """ soup = bs4.BeautifulSoup(html_string, 'html.parser') error_list = [] for math_tag in soup.findAll(name='oppia-noninteractive-math'): math_content_dict = (json.loads( unescape_html(math_tag['math_content-with-value']))) svg_filename = (objects.UnicodeString.normalize( math_content_dict['svg_filename'])) if svg_filename == '': error_list.append(python_utils.UNICODE(math_tag)) else: file_system_class = fs_services.get_entity_file_system_class() fs = fs_domain.AbstractFileSystem( file_system_class(entity_type, entity_id)) filepath = 'image/%s' % svg_filename if not fs.isfile(filepath): error_list.append(python_utils.UNICODE(math_tag)) return error_list
def post(self, entity_type, entity_id): """Saves an image uploaded by a content creator.""" raw = self.normalized_request.get('image') filename = self.normalized_payload.get('filename') filename_prefix = self.normalized_payload.get('filename_prefix') try: file_format = image_validation_services.validate_image_and_filename( raw, filename) except utils.ValidationError as e: raise self.InvalidInputException(e) file_system_class = fs_services.get_entity_file_system_class() fs = fs_domain.AbstractFileSystem( file_system_class(entity_type, entity_id)) filepath = '%s/%s' % (filename_prefix, filename) if fs.isfile(filepath): raise self.InvalidInputException( 'A file with the name %s already exists. Please choose a ' 'different name.' % filename) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, entity_type, entity_id, raw, filename_prefix, image_is_compressible) self.render_json({'filename': filename})
def test_copy_question_images_to_the_correct_storage_path(self): """Tests that the question images are copied to the correct storage path. """ file_system_class = fs_services.get_entity_file_system_class() question_fs = fs_domain.AbstractFileSystem( file_system_class(feconf.ENTITY_TYPE_QUESTION, self.QUESTION_ID)) # Assert that the storage paths do not exist before the job is run. self.assertFalse(question_fs.isfile('image/img.png')) self.assertFalse(question_fs.isfile('image/test_svg.svg')) # Start the job. job_id = (question_jobs_one_off.FixQuestionImagesStorageOneOffJob. create_new()) question_jobs_one_off.FixQuestionImagesStorageOneOffJob.enqueue(job_id) self.process_and_flush_pending_mapreduce_tasks() # Verify that the storage paths exist and the status is reported in the # job output. self.assertTrue(question_fs.isfile('image/img.png')) self.assertTrue(question_fs.isfile('image/test_svg.svg')) output = (question_jobs_one_off.FixQuestionImagesStorageOneOffJob. get_output(job_id)) expected = [[ u'question_image_copied', [ u'2 image paths were fixed for question id question_id ' u'with linked_skill_ids: ' u'[u\'skill_id_1\', u\'skill_id_2\']' ] ]] self.assertEqual(expected, [ast.literal_eval(x) for x in output])
def get(self, exploration_id): """Handles GET requests.""" fs = fs_domain.AbstractFileSystem( fs_domain.ExplorationFileSystem(exploration_id)) dir_list = fs.listdir('') self.render_json({'filepaths': dir_list})
def test_job_with_thumbnail_in_filesystem_logs_success(self): self.save_new_topic( self.TOPIC_ID, self.albert_id, name='A name', abbreviated_name='abbrev', description='description', thumbnail_size_in_bytes=None) # Save the dummy image to the filesystem to be used as thumbnail. with python_utils.open_file( os.path.join(feconf.TESTS_DATA_DIR, 'test_svg.svg'), 'rb', encoding=None) as f: raw_image = f.read() fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem( feconf.ENTITY_TYPE_TOPIC, self.TOPIC_ID)) fs.commit( '%s/topic.svg' % (constants.ASSET_TYPE_THUMBNAIL), raw_image, mimetype='image/svg+xml') # Start migration job on sample topic. job_id = ( topic_jobs_one_off.PopulateTopicThumbnailSizeOneOffJob.create_new() ) topic_jobs_one_off.PopulateTopicThumbnailSizeOneOffJob.enqueue(job_id) # This running without errors indicates that deleted topics are # skipped. self.process_and_flush_pending_mapreduce_tasks() output = ( topic_jobs_one_off.PopulateTopicThumbnailSizeOneOffJob.get_output( job_id)) expected = [[u'thumbnail_size_newly_added', 1]] self.assertEqual(expected, [ast.literal_eval(x) for x in output]) topic_services.delete_topic(self.albert_id, self.TOPIC_ID)
def test_audio_upload_with_non_mp3_file(self): self.login(self.EDITOR_EMAIL) csrf_token = self.get_new_csrf_token() file_system_class = fs_services.get_entity_file_system_class() fs = fs_domain.AbstractFileSystem( file_system_class(feconf.ENTITY_TYPE_EXPLORATION, '0')) with python_utils.open_file(os.path.join(feconf.TESTS_DATA_DIR, self.TEST_AUDIO_FILE_FLAC), mode='rb', encoding=None) as f: raw_audio = f.read() self.assertFalse(fs.isfile('audio/%s' % self.TEST_AUDIO_FILE_FLAC)) with self.accepted_audio_extensions_swap: self.post_json('%s/0' % self.AUDIO_UPLOAD_URL_PREFIX, {'filename': self.TEST_AUDIO_FILE_FLAC}, csrf_token=csrf_token, upload_files=[('raw_audio_file', 'unused_filename', raw_audio)]) self.assertTrue(fs.isfile('audio/%s' % self.TEST_AUDIO_FILE_FLAC)) self.logout()
def export_to_zip_file(exploration_id, version=None): """Returns a ZIP archive of the exploration.""" exploration = get_exploration_by_id(exploration_id, version=version) yaml_repr = exploration.to_yaml() o = StringIO.StringIO() with zipfile.ZipFile(o, mode='w', compression=zipfile.ZIP_DEFLATED) as zf: zf.writestr('%s.yaml' % exploration.title, yaml_repr) fs = fs_domain.AbstractFileSystem( fs_domain.ExplorationFileSystem(exploration_id)) dir_list = fs.listdir('') for filepath in dir_list: # Currently, the version number of all files is 1, since they are # not modifiable post-upload. # TODO(sll): When allowing editing of files, implement versioning # for them. file_contents = fs.get(filepath, version=1) str_filepath = 'assets/%s' % filepath assert isinstance(str_filepath, str) unicode_filepath = str_filepath.decode('utf-8') zf.writestr(unicode_filepath, file_contents) return o.getvalue()
def test_listdir(self): self.fs.commit('abc.png', 'file_contents') self.fs.commit('abcd.png', 'file_contents_2') self.fs.commit('abc/abcd.png', 'file_contents_3') self.fs.commit('bcd/bcde.png', 'file_contents_4') file_names = ['abc.png', 'abc/abcd.png', 'abcd.png', 'bcd/bcde.png'] self.assertEqual(self.fs.listdir(''), file_names) self.assertEqual(self.fs.listdir('abc'), ['abc/abcd.png']) with self.assertRaisesRegexp(IOError, 'Invalid filepath'): self.fs.listdir('/abc') with self.assertRaisesRegexp(IOError, ('The dir_name should not start with /' ' or end with / : abc/')): self.fs.listdir('abc/') self.assertEqual(self.fs.listdir('fake_dir'), []) new_fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, 'eid2')) self.assertEqual(new_fs.listdir('assets'), [])
def setUp(self): super(GcsFileSystemUnitTests, self).setUp() self.USER_EMAIL = '*****@*****.**' self.signup(self.USER_EMAIL, 'username') self.user_id = self.get_user_id_from_email(self.USER_EMAIL) self.fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, 'eid'))
def post(self, exploration_id): """Saves an image uploaded by a content creator.""" raw = self.request.get('image') filename = self.payload.get('filename') if not raw: raise self.InvalidInputException('No image supplied') allowed_formats = ', '.join( feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS.keys()) # Verify that the data is recognized as an image. file_format = imghdr.what(None, h=raw) if file_format not in feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS: raise self.InvalidInputException('Image not recognized') # Verify that the file type matches the supplied extension. if not filename: raise self.InvalidInputException('No filename supplied') if filename.rfind('.') == 0: raise self.InvalidInputException('Invalid filename') if '/' in filename or '..' in filename: raise self.InvalidInputException( 'Filenames should not include slashes (/) or consecutive dot ' 'characters.') if '.' not in filename: raise self.InvalidInputException( 'Image filename with no extension: it should have ' 'one of the following extensions: %s.' % allowed_formats) dot_index = filename.rfind('.') extension = filename[dot_index + 1:].lower() if (extension not in feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS[file_format]): raise self.InvalidInputException( 'Expected a filename ending in .%s, received %s' % (file_format, filename)) # Image files are stored to the datastore in the dev env, and to GCS # in production. file_system_class = (fs_domain.ExplorationFileSystem if (feconf.DEV_MODE or not constants.ENABLE_GCS_STORAGE_FOR_IMAGES) else fs_domain.GcsFileSystem) fs = fs_domain.AbstractFileSystem(file_system_class(exploration_id)) filepath = (filename if (feconf.DEV_MODE or not constants.ENABLE_GCS_STORAGE_FOR_IMAGES) else ('%s/%s' % (self._FILENAME_PREFIX, filename))) if fs.isfile(filepath): raise self.InvalidInputException( 'A file with the name %s already exists. Please choose a ' 'different name.' % filename) exp_services.save_original_and_compressed_versions_of_image( self.user_id, filename, exploration_id, raw) self.render_json({'filepath': filename})
def save_original_and_compressed_versions_of_image( filename, entity_type, entity_id, original_image_content, filename_prefix, image_is_compressible): """Saves the three versions of the image file. Args: filename: str. The name of the image file. entity_type: str. The type of the entity. entity_id: str. The id of the entity. original_image_content: str. The content of the original image. filename_prefix: str. The string to prefix to the filename. image_is_compressible: bool. Whether the image can be compressed or not. """ filepath = '%s/%s' % (filename_prefix, filename) filename_wo_filetype = filename[:filename.rfind('.')] filetype = filename[filename.rfind('.') + 1:] compressed_image_filename = '%s_compressed.%s' % ( filename_wo_filetype, filetype) compressed_image_filepath = '%s/%s' % ( filename_prefix, compressed_image_filename) micro_image_filename = '%s_micro.%s' % ( filename_wo_filetype, filetype) micro_image_filepath = '%s/%s' % (filename_prefix, micro_image_filename) file_system_class = get_entity_file_system_class() fs = fs_domain.AbstractFileSystem(file_system_class( entity_type, entity_id)) if image_is_compressible: compressed_image_content = gae_image_services.compress_image( original_image_content, 0.8) micro_image_content = gae_image_services.compress_image( original_image_content, 0.7) else: compressed_image_content = original_image_content micro_image_content = original_image_content # Because in case of CreateVersionsOfImageJob, the original image is # already there. Also, even if the compressed, micro versions for some # image exists, then this would prevent from creating another copy of # the same. if not fs.isfile(filepath.encode('utf-8')): fs.commit( filepath.encode('utf-8'), original_image_content, mimetype='image/%s' % filetype) if not fs.isfile(compressed_image_filepath.encode('utf-8')): fs.commit( compressed_image_filepath.encode('utf-8'), compressed_image_content, mimetype='image/%s' % filetype) if not fs.isfile(micro_image_filepath.encode('utf-8')): fs.commit( micro_image_filepath.encode('utf-8'), micro_image_content, mimetype='image/%s' % filetype)
def test_independence_of_file_systems(self): self.fs.commit(self.user_id, 'abc.png', 'file_contents') self.assertEqual(self.fs.get('abc.png'), 'file_contents') fs2 = fs_domain.AbstractFileSystem( fs_domain.ExplorationFileSystem('eid2')) with self.assertRaisesRegexp(IOError, r'File abc\.png .* not found'): fs2.get('abc.png')
def test_independence_of_file_systems(self): self.fs.commit(self.user_id, 'abc.png', 'file_contents') self.assertEqual(self.fs.get('abc.png'), 'file_contents') fs2 = fs_domain.AbstractFileSystem( fs_domain.DatastoreBackedFileSystem(feconf.ENTITY_TYPE_EXPLORATION, 'eid2')) with self.assertRaisesRegexp(IOError, r'File abc\.png .* not found'): fs2.get('abc.png')
def test_unexpected_content_rule(self): fs = fs_domain.AbstractFileSystem( fs_domain.DiskBackedFileSystem(TEST_DATA_DIR)) CANONICAL_DATA_DIR = os.path.join(TEST_DATA_DIR, 'canonical') canonical_fs = fs_domain.AbstractFileSystem( fs_domain.DiskBackedFileSystem(CANONICAL_DATA_DIR)) rule = tar_file_string.HasUnexpectedContent(['hello.c', 'Makefile' ]).set_fs(canonical_fs) file_name = 'incorrect-contents.tar.gz' encoded_content = base64.b64encode(fs.get(file_name, mode='rb')) self.assertTrue(rule.eval(encoded_content)) file_name = 'good.tar.gz' encoded_content = base64.b64encode(fs.get(file_name, mode='rb')) self.assertFalse(rule.eval(encoded_content))
def test_copy(self): self.fs.commit('abc2.png', 'file_contents') self.assertEqual(self.fs.listdir(''), ['abc2.png']) destination_fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(feconf.ENTITY_TYPE_QUESTION, 'question_id1')) self.assertEqual(destination_fs.listdir(''), []) destination_fs.copy(self.fs.impl.assets_path, 'abc2.png') self.assertTrue(destination_fs.isfile('abc2.png'))
def classify(exp_id, state, answer, params): """Normalize the answer and select among the answer groups the group in which the answer best belongs. The best group is decided by finding the first rule best satisfied by the answer. Returns a dict with the following keys: 'outcome': A dict representing the outcome of the answer group matched. 'answer_group_index': An index into the answer groups list indicating which one was selected as the group which this answer belongs to. This is equal to the number of answer groups if the default outcome was matched. 'classification_certainty': A normalized value within the range of [0, 1] representing at which confidence level the answer belongs in the chosen answer group. A certainty of 1 means it is the best possible match. A certainty of 0 means it is matched to the default outcome. 'rule_spec_index': An index into the rule specs list of the matched answer group which was selected that indicates which rule spec was matched. This is equal to 0 if the default outcome is selected. When the default rule is matched, outcome is the default_outcome of the state's interaction. """ interaction_instance = interaction_registry.Registry.get_interaction_by_id( state.interaction.id) normalized_answer = interaction_instance.normalize_answer(answer) fs = fs_domain.AbstractFileSystem(fs_domain.ExplorationFileSystem(exp_id)) input_type = interaction_instance.answer_type response = classify_hard_rule(state, params, input_type, normalized_answer, fs) if response is None: response = classify_soft_rule(state, params, input_type, normalized_answer, fs) if (interaction_instance.is_string_classifier_trainable and feconf.ENABLE_STRING_CLASSIFIER and response is None): response = classify_string_classifier_rule(state, normalized_answer) # The best matched group must match above a certain threshold. If no group # meets this requirement, then the default 'group' automatically matches # resulting in the outcome of the answer being the default outcome of the # state. if (response is not None and response['classification_certainty'] >= feconf.DEFAULT_ANSWER_GROUP_CLASSIFICATION_THRESHOLD): return response elif state.interaction.default_outcome is not None: return { 'outcome': state.interaction.default_outcome.to_dict(), 'answer_group_index': len(state.interaction.answer_groups), 'classification_certainty': 0.0, 'rule_spec_index': 0 } raise Exception( 'Something has seriously gone wrong with the exploration. Oppia does ' 'not know what to do with this answer. Please contact the ' 'exploration owner.')
def post(self, entity_type, entity_id): """Saves an image uploaded by a content creator.""" raw = self.request.get('image') filename = self.payload.get('filename') filename_prefix = self.payload.get('filename_prefix') if filename_prefix is None: filename_prefix = self._FILENAME_PREFIX if not raw: raise self.InvalidInputException('No image supplied') allowed_formats = ', '.join( list(feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS.keys())) # Verify that the data is recognized as an image. file_format = imghdr.what(None, h=raw) if file_format not in feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS: raise self.InvalidInputException('Image not recognized') # Verify that the file type matches the supplied extension. if not filename: raise self.InvalidInputException('No filename supplied') if filename.rfind('.') == 0: raise self.InvalidInputException('Invalid filename') if '/' in filename or '..' in filename: raise self.InvalidInputException( 'Filenames should not include slashes (/) or consecutive ' 'dot characters.') if '.' not in filename: raise self.InvalidInputException( 'Image filename with no extension: it should have ' 'one of the following extensions: %s.' % allowed_formats) dot_index = filename.rfind('.') extension = filename[dot_index + 1:].lower() if (extension not in feconf.ACCEPTED_IMAGE_FORMATS_AND_EXTENSIONS[file_format]): raise self.InvalidInputException( 'Expected a filename ending in .%s, received %s' % (file_format, filename)) file_system_class = fs_services.get_entity_file_system_class() fs = fs_domain.AbstractFileSystem( file_system_class(entity_type, entity_id)) filepath = '%s/%s' % (filename_prefix, filename) if fs.isfile(filepath): raise self.InvalidInputException( 'A file with the name %s already exists. Please choose a ' 'different name.' % filename) fs_services.save_original_and_compressed_versions_of_image( filename, entity_type, entity_id, raw, filename_prefix) self.render_json({'filename': filename})
def setUp(self): super(FileSystemClassifierDataTests, self).setUp() self.fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(feconf.ENTITY_TYPE_EXPLORATION, 'exp_id')) self.classifier_data = { 'param1': 40, 'param2': [34.2, 54.13, 95.23], 'submodel': { 'param1': 12 } }
def map(exp_model): if not feconf.DEV_MODE: exp_id = exp_model.id fs_old = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem(exp_id)) # We have to make sure we pass the dir name without starting or # ending with '/'. image_urls = fs_old.listdir('image') audio_urls = fs_old.listdir('audio') image_filenames = [ GCS_IMAGE_ID_REGEX.match(url).group(3) for url in image_urls ] audio_filenames = [ GCS_AUDIO_ID_REGEX.match(url).group(3) for url in audio_urls ] for image_filename in image_filenames: try: raw_image = fs_old.get('image/%s' % image_filename) height, width = gae_image_services.get_image_dimensions( raw_image) filename_with_dimensions = ( html_validation_service. regenerate_image_filename_using_dimensions( # pylint: disable=line-too-long image_filename, height, width)) exp_services.save_original_and_compressed_versions_of_image( # pylint: disable=line-too-long 'ADMIN', filename_with_dimensions, exp_id, raw_image) except Exception: yield (ERROR_IN_FILENAME, image_filename) for audio_filename in audio_filenames: filetype = audio_filename[audio_filename.rfind('.') + 1:] raw_data = fs_old.get('audio/%s' % audio_filename) fs = fs_domain.AbstractFileSystem( fs_domain.GcsFileSystem('exploration/%s' % exp_id)) fs.commit('Admin', 'audio/%s' % audio_filename, raw_data, mimetype='audio/%s' % filetype) yield (ADDED_COMPRESSED_VERSIONS_OF_IMAGES, exp_id)
def delete_classifier_data(exp_id, job_id): """Delete the classifier data from file. Args: exp_id: str. The id of the exploration. job_id: str. The id of the classifier training job model. """ filepath = '%s-classifier-data.json' % (job_id) file_system_class = get_entity_file_system_class() fs = fs_domain.AbstractFileSystem( file_system_class(feconf.ENTITY_TYPE_EXPLORATION, exp_id)) fs.delete(filepath)
def get(self, page_context, page_identifier, asset_type, encoded_filename): """Returns an asset file. Args: page_context: str. The context of the page where the asset is required. page_identifier: str. The unique identifier for the particular context. Valid page_context: page_identifier pairs: exploration: exp_id story: story_id topic: topic_id skill: skill_id subtopic: topic_name of the topic that it is part of. asset_type: str. Type of the asset, either image or audio. encoded_filename: str. The asset filename. This string is encoded in the frontend using encodeURIComponent(). """ if not constants.DEV_MODE: raise self.PageNotFoundException try: filename = python_utils.urllib_unquote(encoded_filename) file_format = filename[(filename.rfind('.') + 1):] # If the following is not cast to str, an error occurs in the wsgi # library because unicode gets used. self.response.headers[ 'Content-Type'] = python_utils.convert_to_bytes( '%s/%s' % (asset_type, file_format)) if page_context == feconf.ENTITY_TYPE_SUBTOPIC: entity_type = feconf.ENTITY_TYPE_TOPIC topic = topic_fetchers.get_topic_by_name(page_identifier) entity_id = topic.id elif (page_context == feconf.ENTITY_TYPE_EXPLORATION or page_context == feconf.ENTITY_TYPE_SKILL or page_context == feconf.ENTITY_TYPE_TOPIC or page_context == feconf.ENTITY_TYPE_STORY): entity_type = page_context entity_id = page_identifier else: raise self.InvalidInputException fs = fs_domain.AbstractFileSystem( fs_domain.DatastoreBackedFileSystem(entity_type, entity_id)) raw = fs.get('%s/%s' % (asset_type, filename)) self.response.cache_control.no_cache = None self.response.cache_control.public = True self.response.cache_control.max_age = 600 self.response.write(raw) except: raise self.PageNotFoundException