def _assert_validation_error( self, image, filename, expected_error_substring): """Checks that the image passes validation.""" with self.assertRaisesRegex( utils.ValidationError, expected_error_substring): image_validation_services.validate_image_and_filename( image, filename)
def _assert_image_validation_error(self, image: Union[str, bytes], filename: str, expected_error_substring: str) -> None: """Checks that the image passes validation.""" with self.assertRaisesRegex( # type: ignore[no-untyped-call] utils.ValidationError, expected_error_substring): image_validation_services.validate_image_and_filename( image, filename)
def validate_suggestion_images(files): """Validates the files dict. Args: files: dict. Data that needs to be validated. Returns: dict. Returns the dict after validation. """ for filename, raw_image in files.items(): image_validation_services.validate_image_and_filename( raw_image, filename) # The files argument do not represent any domain class, hence dict form # of the data is returned from here. return files
def post(self, entity_type, entity_id): """Saves an image uploaded by a content creator.""" raw = self.normalized_request.get('image') filename = self.normalized_payload.get('filename') filename_prefix = self.normalized_payload.get('filename_prefix') try: file_format = image_validation_services.validate_image_and_filename( raw, filename) except utils.ValidationError as e: raise self.InvalidInputException(e) fs = fs_services.GcsFileSystem(entity_type, entity_id) filepath = '%s/%s' % (filename_prefix, filename) if fs.isfile(filepath): raise self.InvalidInputException( 'A file with the name %s already exists. Please choose a ' 'different name.' % filename) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, entity_type, entity_id, raw, filename_prefix, image_is_compressible) self.render_json({'filename': filename})
def _upload_suggestion_images(files, suggestion, filenames): """Saves a suggestion's images to storage. Args: files: dict. Files containing a mapping of image filename to image blob. suggestion: BaseSuggestion. The suggestion for which images are being uploaded. filenames: list(str). The image filenames. """ suggestion_image_context = suggestion.image_context # TODO(#10513): Find a way to save the images before the suggestion is # created. for filename in filenames: image = files.get(filename) image = base64.decodebytes(image.encode('utf-8')) file_format = (image_validation_services.validate_image_and_filename( image, filename)) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, suggestion_image_context, suggestion.target_id, image, 'image', image_is_compressible) target_entity_html_list = suggestion.get_target_entity_html_strings() target_image_filenames = ( html_cleaner.get_image_filenames_from_html_strings( target_entity_html_list)) fs_services.copy_images(suggestion.target_type, suggestion.target_id, suggestion_image_context, suggestion.target_id, target_image_filenames)
def post(self): try: # The create_suggestion method needs to be run in transaction as it # generates multiple connected models (suggestion, feedback thread, # feedback message etc.) and all these models needs to be created # together, in a batch. suggestion = transaction_services.run_in_transaction( suggestion_services.create_suggestion, self.payload.get('suggestion_type'), self.payload.get('target_type'), self.payload.get('target_id'), self.payload.get('target_version_at_submission'), self.user_id, self.payload.get('change'), self.payload.get('description')) except utils.ValidationError as e: raise self.InvalidInputException(e) # TODO(#10513) : Find a way to save the images before the suggestion is # created. suggestion_image_context = suggestion.image_context # For suggestion which doesn't need images for rendering the # image_context is set to None. if suggestion_image_context is None: self.render_json(self.values) return new_image_filenames = ( suggestion.get_new_image_filenames_added_in_suggestion()) for filename in new_image_filenames: image = self.request.get(filename) if not image: logging.error( 'Image not provided for file with name %s when the ' ' suggestion with target id %s was created.' % (filename, suggestion.target_id)) raise self.InvalidInputException( 'No image data provided for file with name %s.' % (filename)) try: file_format = ( image_validation_services.validate_image_and_filename( image, filename)) except utils.ValidationError as e: raise self.InvalidInputException('%s' % (e)) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, suggestion_image_context, suggestion.target_id, image, 'image', image_is_compressible) target_entity_html_list = suggestion.get_target_entity_html_strings() target_image_filenames = ( html_cleaner.get_image_filenames_from_html_strings( target_entity_html_list)) fs_services.copy_images(suggestion.target_type, suggestion.target_id, suggestion_image_context, suggestion.target_id, target_image_filenames) self.render_json(self.values)
def post(self): """Handles POST requests.""" if (self.payload.get('suggestion_type') == feconf.SUGGESTION_TYPE_EDIT_STATE_CONTENT): raise self.InvalidInputException( 'Content suggestion submissions are no longer supported.') try: suggestion = suggestion_services.create_suggestion( self.payload.get('suggestion_type'), self.payload.get('target_type'), self.payload.get('target_id'), self.payload.get('target_version_at_submission'), self.user_id, self.payload.get('change'), self.payload.get('description')) except utils.ValidationError as e: raise self.InvalidInputException(e) # TODO(#10513) : Find a way to save the images before the suggestion is # created. suggestion_image_context = suggestion.image_context new_image_filenames = ( suggestion.get_new_image_filenames_added_in_suggestion()) for filename in new_image_filenames: image = self.request.get(filename) if not image: logging.exception( 'Image not provided for file with name %s when the ' ' suggestion with target id %s was created.' % ( filename, suggestion.target_id)) raise self.InvalidInputException( 'No image data provided for file with name %s.' % (filename)) try: file_format = ( image_validation_services.validate_image_and_filename( image, filename)) except utils.ValidationError as e: raise self.InvalidInputException('%s' % (e)) image_is_compressible = ( file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, suggestion_image_context, suggestion.target_id, image, 'image', image_is_compressible) target_entity_html_list = suggestion.get_target_entity_html_strings() target_image_filenames = ( html_cleaner.get_image_filenames_from_html_strings( target_entity_html_list)) fs_services.copy_images( suggestion.target_type, suggestion.target_id, suggestion_image_context, suggestion.target_id, target_image_filenames) self.render_json(self.values)
def post(self, topic_id): """Handles POST requests. Currently, this only adds the story to the canonical story id list of the topic. """ title = self.payload.get('title') description = self.payload.get('description') thumbnail_filename = self.payload.get('filename') thumbnail_bg_color = self.payload.get('thumbnailBgColor') raw_image = self.request.get('image') story_url_fragment = self.payload.get('story_url_fragment') story_domain.Story.require_valid_title(title) if story_services.does_story_exist_with_url_fragment( story_url_fragment): raise self.InvalidInputException( 'Story url fragment is not unique across the site.') new_story_id = story_services.get_new_story_id() story = story_domain.Story.create_default_story( new_story_id, title, description, topic_id, story_url_fragment) story_services.save_new_story(self.user_id, story) topic_services.add_canonical_story(self.user_id, topic_id, new_story_id) try: file_format = image_validation_services.validate_image_and_filename( raw_image, thumbnail_filename) except utils.ValidationError as e: raise self.InvalidInputException(e) entity_id = new_story_id filename_prefix = 'thumbnail' image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( thumbnail_filename, feconf.ENTITY_TYPE_STORY, entity_id, raw_image, filename_prefix, image_is_compressible) story_services.update_story(self.user_id, new_story_id, [ story_domain.StoryChange({ 'cmd': 'update_story_property', 'property_name': 'thumbnail_filename', 'old_value': None, 'new_value': thumbnail_filename }), story_domain.StoryChange({ 'cmd': 'update_story_property', 'property_name': 'thumbnail_bg_color', 'old_value': None, 'new_value': thumbnail_bg_color }), ], 'Added story thumbnail.') self.render_json({'storyId': new_story_id})
def map(item): if item.deleted: return fs = fs_domain.AbstractFileSystem(fs_domain.GcsFileSystem( feconf.ENTITY_TYPE_EXPLORATION, item.id)) filepaths = fs.listdir('image') count_of_unchanged_svgs = 0 filenames_of_modified_svgs = [] for filepath in filepaths: filename = filepath.split('/')[-1] if not re.match(constants.MATH_SVG_FILENAME_REGEX, filename): continue old_svg_image = fs.get(filepath) new_svg_image = ( html_validation_service.get_svg_with_xmlns_attribute( old_svg_image)) if new_svg_image == old_svg_image: count_of_unchanged_svgs += 1 continue try: image_validation_services.validate_image_and_filename( new_svg_image, filename) except Exception as e: yield ( 'FAILED validation', 'Exploration with id %s failed image validation for the ' 'filename %s with following error: %s' % ( item.id, filename, e)) else: fs.commit( filepath.encode('utf-8'), new_svg_image, mimetype='image/svg+xml') filenames_of_modified_svgs.append(filename) if count_of_unchanged_svgs: yield ('UNCHANGED', count_of_unchanged_svgs) if len(filenames_of_modified_svgs) > 0: yield ( 'SUCCESS - CHANGED Exp Id: %s' % item.id, filenames_of_modified_svgs)
def post(self): """Handles POST requests.""" name = self.payload.get('name') url_fragment = self.payload.get('url_fragment') description = self.payload.get('description') thumbnail_filename = self.payload.get('filename') thumbnail_bg_color = self.payload.get('thumbnailBgColor') raw_image = self.request.get('image') page_title_frag = self.payload.get('page_title_fragment') try: topic_domain.Topic.require_valid_name(name) except Exception as e: raise self.InvalidInputException( 'Invalid topic name, received %s.' % name) from e new_topic_id = topic_fetchers.get_new_topic_id() topic = topic_domain.Topic.create_default_topic( new_topic_id, name, url_fragment, description, page_title_frag) topic_services.save_new_topic(self.user_id, topic) try: file_format = image_validation_services.validate_image_and_filename( raw_image, thumbnail_filename) except utils.ValidationError as e: raise self.InvalidInputException(e) entity_id = new_topic_id filename_prefix = 'thumbnail' image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( thumbnail_filename, feconf.ENTITY_TYPE_TOPIC, entity_id, raw_image, filename_prefix, image_is_compressible) topic_services.update_topic_and_subtopic_pages( self.user_id, new_topic_id, [ topic_domain.TopicChange({ 'cmd': 'update_topic_property', 'property_name': 'thumbnail_filename', 'old_value': None, 'new_value': thumbnail_filename }), topic_domain.TopicChange({ 'cmd': 'update_topic_property', 'property_name': 'thumbnail_bg_color', 'old_value': None, 'new_value': thumbnail_bg_color }), ], 'Add topic thumbnail.') self.render_json({'topicId': new_topic_id})
def _upload_suggestion_images(request, suggestion, filenames): """Saves a suggestion's images to storage. Args: request: webapp2.Request. Request object containing a mapping of image filename to image blob. suggestion: BaseSuggestion. The suggestion for which images are being uploaded. filenames: list(str). The image filenames. """ suggestion_image_context = suggestion.image_context # TODO(#10513) : Find a way to save the images before the suggestion is # created. for filename in filenames: image = request.get(filename) if not image: logging.exception( 'Image not provided for file with name %s when the ' ' suggestion with target id %s was created.' % (filename, suggestion.target_id)) raise base.BaseHandler.InvalidInputException( 'No image data provided for file with name %s.' % (filename)) try: file_format = ( image_validation_services.validate_image_and_filename( image, filename)) except utils.ValidationError as e: raise base.BaseHandler.InvalidInputException('%s' % (e)) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, suggestion_image_context, suggestion.target_id, image, 'image', image_is_compressible) target_entity_html_list = suggestion.get_target_entity_html_strings() target_image_filenames = ( html_cleaner.get_image_filenames_from_html_strings( target_entity_html_list)) fs_services.copy_images(suggestion.target_type, suggestion.target_id, suggestion_image_context, suggestion.target_id, target_image_filenames)
def post(self): description = self.normalized_payload.get('description') linked_topic_ids = self.normalized_payload.get('linked_topic_ids') explanation_dict = self.normalized_payload.get('explanation_dict') rubrics = self.normalized_payload.get('rubrics') files = self.normalized_payload.get('files') new_skill_id = skill_services.get_new_skill_id() if linked_topic_ids is not None: topics = topic_fetchers.get_topics_by_ids(linked_topic_ids) for topic in topics: if topic is None: raise self.InvalidInputException topic_services.add_uncategorized_skill(self.user_id, topic.id, new_skill_id) if skill_services.does_skill_with_description_exist(description): raise self.InvalidInputException( 'Skill description should not be a duplicate.') skill = skill_domain.Skill.create_default_skill( new_skill_id, description, rubrics) skill.update_explanation(explanation_dict) image_filenames = skill_services.get_image_filenames_from_skill(skill) skill_services.save_new_skill(self.user_id, skill) for filename in image_filenames: base64_image = files.get(filename) bytes_image = base64.decodebytes(base64_image.encode('utf-8')) file_format = ( image_validation_services.validate_image_and_filename( bytes_image, filename)) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, feconf.ENTITY_TYPE_SKILL, skill.id, bytes_image, 'image', image_is_compressible) self.render_json({'skillId': new_skill_id})
def post(self, blog_post_id: str) -> None: """Stores thumbnail of the blog post in the datastore.""" blog_domain.BlogPost.require_valid_blog_post_id(blog_post_id) raw_image = self.normalized_request.get('image') thumbnail_filename = self.normalized_payload.get('thumbnail_filename') try: file_format = image_validation_services.validate_image_and_filename( raw_image, thumbnail_filename) except utils.ValidationError as e: raise self.InvalidInputException(e) entity_id = blog_post_id filename_prefix = 'thumbnail' image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( thumbnail_filename, feconf.ENTITY_TYPE_BLOG_POST, entity_id, raw_image, filename_prefix, image_is_compressible) self.render_json(self.values)
def post(self): description = self.payload.get('description') linked_topic_ids = self.payload.get('linked_topic_ids') explanation_dict = self.payload.get('explanation_dict') rubrics = self.payload.get('rubrics') if not isinstance(rubrics, list): raise self.InvalidInputException('Rubrics should be a list.') if not isinstance(explanation_dict, dict): raise self.InvalidInputException('Explanation should be a dict.') try: subtitled_html = ( state_domain.SubtitledHtml.from_dict(explanation_dict)) subtitled_html.validate() except Exception as e: raise self.InvalidInputException( 'Explanation should be a valid SubtitledHtml dict.') from e rubrics = [skill_domain.Rubric.from_dict(rubric) for rubric in rubrics] new_skill_id = skill_services.get_new_skill_id() if linked_topic_ids is not None: topics = topic_fetchers.get_topics_by_ids(linked_topic_ids) for topic in topics: if topic is None: raise self.InvalidInputException topic_services.add_uncategorized_skill(self.user_id, topic.id, new_skill_id) skill_domain.Skill.require_valid_description(description) if skill_services.does_skill_with_description_exist(description): raise self.InvalidInputException( 'Skill description should not be a duplicate.') skill = skill_domain.Skill.create_default_skill( new_skill_id, description, rubrics) skill.update_explanation( state_domain.SubtitledHtml.from_dict(explanation_dict)) image_filenames = skill_services.get_image_filenames_from_skill(skill) skill_services.save_new_skill(self.user_id, skill) image_validation_error_message_suffix = ( 'Please go to oppia.org/skill_editor/%s to edit ' 'the image.' % skill.id) for filename in image_filenames: image = self.request.get(filename) if not image: logging.exception( 'Image not provided for file with name %s when the skill ' 'with id %s was created.' % (filename, skill.id)) raise self.InvalidInputException( 'No image data provided for file with name %s. %s' % (filename, image_validation_error_message_suffix)) try: file_format = ( image_validation_services.validate_image_and_filename( image, filename)) except utils.ValidationError as e: e = '%s %s' % (e, image_validation_error_message_suffix) raise self.InvalidInputException(e) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, feconf.ENTITY_TYPE_SKILL, skill.id, image, 'image', image_is_compressible) self.render_json({'skillId': new_skill_id})
def post(self): """Handles POST requests.""" skill_ids = self.payload.get('skill_ids') if not skill_ids: raise self.InvalidInputException( 'skill_ids parameter isn\'t present in the payload') if len(skill_ids) > constants.MAX_SKILLS_PER_QUESTION: raise self.InvalidInputException( 'More than %d QuestionSkillLinks for one question ' 'is not supported.' % constants.MAX_SKILLS_PER_QUESTION) try: for skill_id in skill_ids: skill_domain.Skill.require_valid_skill_id(skill_id) except Exception as e: raise self.InvalidInputException('Skill ID(s) aren\'t valid: ', e) try: skill_fetchers.get_multi_skills(skill_ids) except Exception as e: raise self.PageNotFoundException(e) question_dict = self.payload.get('question_dict') if ((question_dict['id'] is not None) or ('question_state_data' not in question_dict) or ('language_code' not in question_dict) or (question_dict['version'] != 0)): raise self.InvalidInputException( 'Question Data should contain id, state data, language code, ' + 'and its version should be set as 0') question_dict['question_state_data_schema_version'] = ( feconf.CURRENT_STATE_SCHEMA_VERSION) question_dict['id'] = question_services.get_new_question_id() question_dict['linked_skill_ids'] = skill_ids try: question = question_domain.Question.from_dict(question_dict) except Exception as e: raise self.InvalidInputException('Question structure is invalid:', e) skill_difficulties = self.payload.get('skill_difficulties') if not skill_difficulties: raise self.InvalidInputException( 'skill_difficulties not present in the payload') if len(skill_ids) != len(skill_difficulties): raise self.InvalidInputException( 'Skill difficulties don\'t match up with skill IDs') try: skill_difficulties = [ float(difficulty) for difficulty in skill_difficulties ] except (ValueError, TypeError) as e: raise self.InvalidInputException( 'Skill difficulties must be a float value') from e if any((difficulty < 0 or difficulty > 1) for difficulty in skill_difficulties): raise self.InvalidInputException( 'Skill difficulties must be between 0 and 1') question_services.add_question(self.user_id, question) question_services.link_multiple_skills_for_question( self.user_id, question.id, skill_ids, skill_difficulties) html_list = question.question_state_data.get_all_html_content_strings() filenames = ( html_cleaner.get_image_filenames_from_html_strings(html_list)) image_validation_error_message_suffix = ( 'Please go to the question editor for question with id %s and edit ' 'the image.' % question.id) for filename in filenames: image = self.request.get(filename) if not image: logging.exception( 'Image not provided for file with name %s when the question' ' with id %s was created.' % (filename, question.id)) raise self.InvalidInputException( 'No image data provided for file with name %s. %s' % (filename, image_validation_error_message_suffix)) try: file_format = ( image_validation_services.validate_image_and_filename( image, filename)) except utils.ValidationError as e: e = '%s %s' % (e, image_validation_error_message_suffix) raise self.InvalidInputException(e) image_is_compressible = (file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, feconf.ENTITY_TYPE_QUESTION, question.id, image, 'image', image_is_compressible) self.values.update({'question_id': question.id}) self.render_json(self.values)
def update_suggestions_with_math_svgs(suggestions_raw_latex_to_image_data_dict): """Saves an SVG for each LaTeX string without an SVG in suggestions and updates the suggestions. TODO(#10045): Remove this function once all the math-rich text components in suggestions have a valid math SVG stored in the datastore. Args: suggestions_raw_latex_to_image_data_dict: dict(str, dict(str, LatexStringSvgImageData)). The dictionary having the key as a suggestion ID each value as dict with key as a LaTeX string and its value as LatexStringSvgImageData domain object. Raises: Exception. If any of the SVG images provided fail validation. """ suggestions_to_update = [] suggestion_ids = suggestions_raw_latex_to_image_data_dict.keys() suggestion_models_to_update = ( suggestion_models.GeneralSuggestionModel.get_multi(suggestion_ids)) for suggestion_model in suggestion_models_to_update: suggestion = get_suggestion_from_model(suggestion_model) suggestion_id = suggestion.suggestion_id exp_id = suggestion.target_id raw_latex_to_image_data_dict = ( suggestions_raw_latex_to_image_data_dict[suggestion_id]) add_svg_filenames_for_latex_strings_in_html_string = ( functools.partial( html_validation_service. add_svg_filenames_for_latex_strings_in_html_string, raw_latex_to_image_data_dict)) suggestion.convert_html_in_suggestion_change( add_svg_filenames_for_latex_strings_in_html_string) updated_html_string = ''.join(suggestion.get_all_html_content_strings()) filenames_mapping = ( html_validation_service. extract_svg_filename_latex_mapping_in_math_rte_components( updated_html_string)) for filename, raw_latex in filenames_mapping: # Some new filenames may already have the images saved from the math # rich-text editor, for these files we don't need to save the image # again. if raw_latex in raw_latex_to_image_data_dict.keys(): image_file = raw_latex_to_image_data_dict[raw_latex].raw_image image_validation_error_message_suffix = ( 'SVG image provided for latex %s failed validation' % ( raw_latex)) try: file_format = ( image_validation_services.validate_image_and_filename( image_file, filename)) except utils.ValidationError as e: e = '%s %s' % (e, image_validation_error_message_suffix) raise Exception(e) image_is_compressible = ( file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS) fs_services.save_original_and_compressed_versions_of_image( filename, feconf.ENTITY_TYPE_EXPLORATION, exp_id, image_file, 'image', image_is_compressible) suggestions_to_update.append(suggestion) _update_suggestions(suggestions_to_update, update_last_updated_time=False)