def _assert_validation_error(
self, image, filename, expected_error_substring):
"""Checks that the image passes validation."""
with self.assertRaisesRegex(
utils.ValidationError, expected_error_substring):
image_validation_services.validate_image_and_filename(
image, filename)
def _assert_image_validation_error(self, image: Union[str, bytes],
filename: str,
expected_error_substring: str) -> None:
"""Checks that the image passes validation."""
with self.assertRaisesRegex( # type: ignore[no-untyped-call]
utils.ValidationError, expected_error_substring):
image_validation_services.validate_image_and_filename(
image, filename)
def validate_suggestion_images(files):
"""Validates the files dict.
Args:
files: dict. Data that needs to be validated.
Returns:
dict. Returns the dict after validation.
"""
for filename, raw_image in files.items():
image_validation_services.validate_image_and_filename(
raw_image, filename)
# The files argument do not represent any domain class, hence dict form
# of the data is returned from here.
return files
def post(self, entity_type, entity_id):
"""Saves an image uploaded by a content creator."""
raw = self.normalized_request.get('image')
filename = self.normalized_payload.get('filename')
filename_prefix = self.normalized_payload.get('filename_prefix')
try:
file_format = image_validation_services.validate_image_and_filename(
raw, filename)
except utils.ValidationError as e:
raise self.InvalidInputException(e)
fs = fs_services.GcsFileSystem(entity_type, entity_id)
filepath = '%s/%s' % (filename_prefix, filename)
if fs.isfile(filepath):
raise self.InvalidInputException(
'A file with the name %s already exists. Please choose a '
'different name.' % filename)
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, entity_type, entity_id, raw, filename_prefix,
image_is_compressible)
self.render_json({'filename': filename})
def _upload_suggestion_images(files, suggestion, filenames):
"""Saves a suggestion's images to storage.
Args:
files: dict. Files containing a mapping of image
filename to image blob.
suggestion: BaseSuggestion. The suggestion for which images are being
uploaded.
filenames: list(str). The image filenames.
"""
suggestion_image_context = suggestion.image_context
# TODO(#10513): Find a way to save the images before the suggestion is
# created.
for filename in filenames:
image = files.get(filename)
image = base64.decodebytes(image.encode('utf-8'))
file_format = (image_validation_services.validate_image_and_filename(
image, filename))
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, suggestion_image_context, suggestion.target_id, image,
'image', image_is_compressible)
target_entity_html_list = suggestion.get_target_entity_html_strings()
target_image_filenames = (
html_cleaner.get_image_filenames_from_html_strings(
target_entity_html_list))
fs_services.copy_images(suggestion.target_type, suggestion.target_id,
suggestion_image_context, suggestion.target_id,
target_image_filenames)
def post(self):
try:
# The create_suggestion method needs to be run in transaction as it
# generates multiple connected models (suggestion, feedback thread,
# feedback message etc.) and all these models needs to be created
# together, in a batch.
suggestion = transaction_services.run_in_transaction(
suggestion_services.create_suggestion,
self.payload.get('suggestion_type'),
self.payload.get('target_type'), self.payload.get('target_id'),
self.payload.get('target_version_at_submission'), self.user_id,
self.payload.get('change'), self.payload.get('description'))
except utils.ValidationError as e:
raise self.InvalidInputException(e)
# TODO(#10513) : Find a way to save the images before the suggestion is
# created.
suggestion_image_context = suggestion.image_context
# For suggestion which doesn't need images for rendering the
# image_context is set to None.
if suggestion_image_context is None:
self.render_json(self.values)
return
new_image_filenames = (
suggestion.get_new_image_filenames_added_in_suggestion())
for filename in new_image_filenames:
image = self.request.get(filename)
if not image:
logging.error(
'Image not provided for file with name %s when the '
' suggestion with target id %s was created.' %
(filename, suggestion.target_id))
raise self.InvalidInputException(
'No image data provided for file with name %s.' %
(filename))
try:
file_format = (
image_validation_services.validate_image_and_filename(
image, filename))
except utils.ValidationError as e:
raise self.InvalidInputException('%s' % (e))
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, suggestion_image_context, suggestion.target_id,
image, 'image', image_is_compressible)
target_entity_html_list = suggestion.get_target_entity_html_strings()
target_image_filenames = (
html_cleaner.get_image_filenames_from_html_strings(
target_entity_html_list))
fs_services.copy_images(suggestion.target_type, suggestion.target_id,
suggestion_image_context, suggestion.target_id,
target_image_filenames)
self.render_json(self.values)
def post(self):
"""Handles POST requests."""
if (self.payload.get('suggestion_type') ==
feconf.SUGGESTION_TYPE_EDIT_STATE_CONTENT):
raise self.InvalidInputException(
'Content suggestion submissions are no longer supported.')
try:
suggestion = suggestion_services.create_suggestion(
self.payload.get('suggestion_type'),
self.payload.get('target_type'), self.payload.get('target_id'),
self.payload.get('target_version_at_submission'),
self.user_id, self.payload.get('change'),
self.payload.get('description'))
except utils.ValidationError as e:
raise self.InvalidInputException(e)
# TODO(#10513) : Find a way to save the images before the suggestion is
# created.
suggestion_image_context = suggestion.image_context
new_image_filenames = (
suggestion.get_new_image_filenames_added_in_suggestion())
for filename in new_image_filenames:
image = self.request.get(filename)
if not image:
logging.exception(
'Image not provided for file with name %s when the '
' suggestion with target id %s was created.' % (
filename, suggestion.target_id))
raise self.InvalidInputException(
'No image data provided for file with name %s.'
% (filename))
try:
file_format = (
image_validation_services.validate_image_and_filename(
image, filename))
except utils.ValidationError as e:
raise self.InvalidInputException('%s' % (e))
image_is_compressible = (
file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, suggestion_image_context, suggestion.target_id,
image, 'image', image_is_compressible)
target_entity_html_list = suggestion.get_target_entity_html_strings()
target_image_filenames = (
html_cleaner.get_image_filenames_from_html_strings(
target_entity_html_list))
fs_services.copy_images(
suggestion.target_type, suggestion.target_id,
suggestion_image_context, suggestion.target_id,
target_image_filenames)
self.render_json(self.values)
def post(self, topic_id):
"""Handles POST requests.
Currently, this only adds the story to the canonical story id list of
the topic.
"""
title = self.payload.get('title')
description = self.payload.get('description')
thumbnail_filename = self.payload.get('filename')
thumbnail_bg_color = self.payload.get('thumbnailBgColor')
raw_image = self.request.get('image')
story_url_fragment = self.payload.get('story_url_fragment')
story_domain.Story.require_valid_title(title)
if story_services.does_story_exist_with_url_fragment(
story_url_fragment):
raise self.InvalidInputException(
'Story url fragment is not unique across the site.')
new_story_id = story_services.get_new_story_id()
story = story_domain.Story.create_default_story(
new_story_id, title, description, topic_id, story_url_fragment)
story_services.save_new_story(self.user_id, story)
topic_services.add_canonical_story(self.user_id, topic_id,
new_story_id)
try:
file_format = image_validation_services.validate_image_and_filename(
raw_image, thumbnail_filename)
except utils.ValidationError as e:
raise self.InvalidInputException(e)
entity_id = new_story_id
filename_prefix = 'thumbnail'
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
thumbnail_filename, feconf.ENTITY_TYPE_STORY, entity_id, raw_image,
filename_prefix, image_is_compressible)
story_services.update_story(self.user_id, new_story_id, [
story_domain.StoryChange({
'cmd': 'update_story_property',
'property_name': 'thumbnail_filename',
'old_value': None,
'new_value': thumbnail_filename
}),
story_domain.StoryChange({
'cmd': 'update_story_property',
'property_name': 'thumbnail_bg_color',
'old_value': None,
'new_value': thumbnail_bg_color
}),
], 'Added story thumbnail.')
self.render_json({'storyId': new_story_id})
def map(item):
if item.deleted:
return
fs = fs_domain.AbstractFileSystem(fs_domain.GcsFileSystem(
feconf.ENTITY_TYPE_EXPLORATION, item.id))
filepaths = fs.listdir('image')
count_of_unchanged_svgs = 0
filenames_of_modified_svgs = []
for filepath in filepaths:
filename = filepath.split('/')[-1]
if not re.match(constants.MATH_SVG_FILENAME_REGEX, filename):
continue
old_svg_image = fs.get(filepath)
new_svg_image = (
html_validation_service.get_svg_with_xmlns_attribute(
old_svg_image))
if new_svg_image == old_svg_image:
count_of_unchanged_svgs += 1
continue
try:
image_validation_services.validate_image_and_filename(
new_svg_image, filename)
except Exception as e:
yield (
'FAILED validation',
'Exploration with id %s failed image validation for the '
'filename %s with following error: %s' % (
item.id, filename, e))
else:
fs.commit(
filepath.encode('utf-8'), new_svg_image,
mimetype='image/svg+xml')
filenames_of_modified_svgs.append(filename)
if count_of_unchanged_svgs:
yield ('UNCHANGED', count_of_unchanged_svgs)
if len(filenames_of_modified_svgs) > 0:
yield (
'SUCCESS - CHANGED Exp Id: %s' % item.id,
filenames_of_modified_svgs)
def post(self):
"""Handles POST requests."""
name = self.payload.get('name')
url_fragment = self.payload.get('url_fragment')
description = self.payload.get('description')
thumbnail_filename = self.payload.get('filename')
thumbnail_bg_color = self.payload.get('thumbnailBgColor')
raw_image = self.request.get('image')
page_title_frag = self.payload.get('page_title_fragment')
try:
topic_domain.Topic.require_valid_name(name)
except Exception as e:
raise self.InvalidInputException(
'Invalid topic name, received %s.' % name) from e
new_topic_id = topic_fetchers.get_new_topic_id()
topic = topic_domain.Topic.create_default_topic(
new_topic_id, name, url_fragment, description, page_title_frag)
topic_services.save_new_topic(self.user_id, topic)
try:
file_format = image_validation_services.validate_image_and_filename(
raw_image, thumbnail_filename)
except utils.ValidationError as e:
raise self.InvalidInputException(e)
entity_id = new_topic_id
filename_prefix = 'thumbnail'
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
thumbnail_filename, feconf.ENTITY_TYPE_TOPIC, entity_id, raw_image,
filename_prefix, image_is_compressible)
topic_services.update_topic_and_subtopic_pages(
self.user_id, new_topic_id, [
topic_domain.TopicChange({
'cmd': 'update_topic_property',
'property_name': 'thumbnail_filename',
'old_value': None,
'new_value': thumbnail_filename
}),
topic_domain.TopicChange({
'cmd': 'update_topic_property',
'property_name': 'thumbnail_bg_color',
'old_value': None,
'new_value': thumbnail_bg_color
}),
], 'Add topic thumbnail.')
self.render_json({'topicId': new_topic_id})
def _upload_suggestion_images(request, suggestion, filenames):
"""Saves a suggestion's images to storage.
Args:
request: webapp2.Request. Request object containing a mapping of image
filename to image blob.
suggestion: BaseSuggestion. The suggestion for which images are being
uploaded.
filenames: list(str). The image filenames.
"""
suggestion_image_context = suggestion.image_context
# TODO(#10513) : Find a way to save the images before the suggestion is
# created.
for filename in filenames:
image = request.get(filename)
if not image:
logging.exception(
'Image not provided for file with name %s when the '
' suggestion with target id %s was created.' %
(filename, suggestion.target_id))
raise base.BaseHandler.InvalidInputException(
'No image data provided for file with name %s.' % (filename))
try:
file_format = (
image_validation_services.validate_image_and_filename(
image, filename))
except utils.ValidationError as e:
raise base.BaseHandler.InvalidInputException('%s' % (e))
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, suggestion_image_context, suggestion.target_id, image,
'image', image_is_compressible)
target_entity_html_list = suggestion.get_target_entity_html_strings()
target_image_filenames = (
html_cleaner.get_image_filenames_from_html_strings(
target_entity_html_list))
fs_services.copy_images(suggestion.target_type, suggestion.target_id,
suggestion_image_context, suggestion.target_id,
target_image_filenames)
def post(self):
description = self.normalized_payload.get('description')
linked_topic_ids = self.normalized_payload.get('linked_topic_ids')
explanation_dict = self.normalized_payload.get('explanation_dict')
rubrics = self.normalized_payload.get('rubrics')
files = self.normalized_payload.get('files')
new_skill_id = skill_services.get_new_skill_id()
if linked_topic_ids is not None:
topics = topic_fetchers.get_topics_by_ids(linked_topic_ids)
for topic in topics:
if topic is None:
raise self.InvalidInputException
topic_services.add_uncategorized_skill(self.user_id, topic.id,
new_skill_id)
if skill_services.does_skill_with_description_exist(description):
raise self.InvalidInputException(
'Skill description should not be a duplicate.')
skill = skill_domain.Skill.create_default_skill(
new_skill_id, description, rubrics)
skill.update_explanation(explanation_dict)
image_filenames = skill_services.get_image_filenames_from_skill(skill)
skill_services.save_new_skill(self.user_id, skill)
for filename in image_filenames:
base64_image = files.get(filename)
bytes_image = base64.decodebytes(base64_image.encode('utf-8'))
file_format = (
image_validation_services.validate_image_and_filename(
bytes_image, filename))
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, feconf.ENTITY_TYPE_SKILL, skill.id, bytes_image,
'image', image_is_compressible)
self.render_json({'skillId': new_skill_id})
def post(self, blog_post_id: str) -> None:
"""Stores thumbnail of the blog post in the datastore."""
blog_domain.BlogPost.require_valid_blog_post_id(blog_post_id)
raw_image = self.normalized_request.get('image')
thumbnail_filename = self.normalized_payload.get('thumbnail_filename')
try:
file_format = image_validation_services.validate_image_and_filename(
raw_image, thumbnail_filename)
except utils.ValidationError as e:
raise self.InvalidInputException(e)
entity_id = blog_post_id
filename_prefix = 'thumbnail'
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
thumbnail_filename, feconf.ENTITY_TYPE_BLOG_POST, entity_id,
raw_image, filename_prefix, image_is_compressible)
self.render_json(self.values)
def post(self):
description = self.payload.get('description')
linked_topic_ids = self.payload.get('linked_topic_ids')
explanation_dict = self.payload.get('explanation_dict')
rubrics = self.payload.get('rubrics')
if not isinstance(rubrics, list):
raise self.InvalidInputException('Rubrics should be a list.')
if not isinstance(explanation_dict, dict):
raise self.InvalidInputException('Explanation should be a dict.')
try:
subtitled_html = (
state_domain.SubtitledHtml.from_dict(explanation_dict))
subtitled_html.validate()
except Exception as e:
raise self.InvalidInputException(
'Explanation should be a valid SubtitledHtml dict.') from e
rubrics = [skill_domain.Rubric.from_dict(rubric) for rubric in rubrics]
new_skill_id = skill_services.get_new_skill_id()
if linked_topic_ids is not None:
topics = topic_fetchers.get_topics_by_ids(linked_topic_ids)
for topic in topics:
if topic is None:
raise self.InvalidInputException
topic_services.add_uncategorized_skill(self.user_id, topic.id,
new_skill_id)
skill_domain.Skill.require_valid_description(description)
if skill_services.does_skill_with_description_exist(description):
raise self.InvalidInputException(
'Skill description should not be a duplicate.')
skill = skill_domain.Skill.create_default_skill(
new_skill_id, description, rubrics)
skill.update_explanation(
state_domain.SubtitledHtml.from_dict(explanation_dict))
image_filenames = skill_services.get_image_filenames_from_skill(skill)
skill_services.save_new_skill(self.user_id, skill)
image_validation_error_message_suffix = (
'Please go to oppia.org/skill_editor/%s to edit '
'the image.' % skill.id)
for filename in image_filenames:
image = self.request.get(filename)
if not image:
logging.exception(
'Image not provided for file with name %s when the skill '
'with id %s was created.' % (filename, skill.id))
raise self.InvalidInputException(
'No image data provided for file with name %s. %s' %
(filename, image_validation_error_message_suffix))
try:
file_format = (
image_validation_services.validate_image_and_filename(
image, filename))
except utils.ValidationError as e:
e = '%s %s' % (e, image_validation_error_message_suffix)
raise self.InvalidInputException(e)
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, feconf.ENTITY_TYPE_SKILL, skill.id, image, 'image',
image_is_compressible)
self.render_json({'skillId': new_skill_id})
def post(self):
"""Handles POST requests."""
skill_ids = self.payload.get('skill_ids')
if not skill_ids:
raise self.InvalidInputException(
'skill_ids parameter isn\'t present in the payload')
if len(skill_ids) > constants.MAX_SKILLS_PER_QUESTION:
raise self.InvalidInputException(
'More than %d QuestionSkillLinks for one question '
'is not supported.' % constants.MAX_SKILLS_PER_QUESTION)
try:
for skill_id in skill_ids:
skill_domain.Skill.require_valid_skill_id(skill_id)
except Exception as e:
raise self.InvalidInputException('Skill ID(s) aren\'t valid: ', e)
try:
skill_fetchers.get_multi_skills(skill_ids)
except Exception as e:
raise self.PageNotFoundException(e)
question_dict = self.payload.get('question_dict')
if ((question_dict['id'] is not None)
or ('question_state_data' not in question_dict)
or ('language_code' not in question_dict)
or (question_dict['version'] != 0)):
raise self.InvalidInputException(
'Question Data should contain id, state data, language code, '
+ 'and its version should be set as 0')
question_dict['question_state_data_schema_version'] = (
feconf.CURRENT_STATE_SCHEMA_VERSION)
question_dict['id'] = question_services.get_new_question_id()
question_dict['linked_skill_ids'] = skill_ids
try:
question = question_domain.Question.from_dict(question_dict)
except Exception as e:
raise self.InvalidInputException('Question structure is invalid:',
e)
skill_difficulties = self.payload.get('skill_difficulties')
if not skill_difficulties:
raise self.InvalidInputException(
'skill_difficulties not present in the payload')
if len(skill_ids) != len(skill_difficulties):
raise self.InvalidInputException(
'Skill difficulties don\'t match up with skill IDs')
try:
skill_difficulties = [
float(difficulty) for difficulty in skill_difficulties
]
except (ValueError, TypeError) as e:
raise self.InvalidInputException(
'Skill difficulties must be a float value') from e
if any((difficulty < 0 or difficulty > 1)
for difficulty in skill_difficulties):
raise self.InvalidInputException(
'Skill difficulties must be between 0 and 1')
question_services.add_question(self.user_id, question)
question_services.link_multiple_skills_for_question(
self.user_id, question.id, skill_ids, skill_difficulties)
html_list = question.question_state_data.get_all_html_content_strings()
filenames = (
html_cleaner.get_image_filenames_from_html_strings(html_list))
image_validation_error_message_suffix = (
'Please go to the question editor for question with id %s and edit '
'the image.' % question.id)
for filename in filenames:
image = self.request.get(filename)
if not image:
logging.exception(
'Image not provided for file with name %s when the question'
' with id %s was created.' % (filename, question.id))
raise self.InvalidInputException(
'No image data provided for file with name %s. %s' %
(filename, image_validation_error_message_suffix))
try:
file_format = (
image_validation_services.validate_image_and_filename(
image, filename))
except utils.ValidationError as e:
e = '%s %s' % (e, image_validation_error_message_suffix)
raise self.InvalidInputException(e)
image_is_compressible = (file_format
in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, feconf.ENTITY_TYPE_QUESTION, question.id, image,
'image', image_is_compressible)
self.values.update({'question_id': question.id})
self.render_json(self.values)
def update_suggestions_with_math_svgs(suggestions_raw_latex_to_image_data_dict):
"""Saves an SVG for each LaTeX string without an SVG in suggestions
and updates the suggestions.
TODO(#10045): Remove this function once all the math-rich text components in
suggestions have a valid math SVG stored in the datastore.
Args:
suggestions_raw_latex_to_image_data_dict:
dict(str, dict(str, LatexStringSvgImageData)). The dictionary
having the key as a suggestion ID each value as dict with key as
a LaTeX string and its value as LatexStringSvgImageData domain
object.
Raises:
Exception. If any of the SVG images provided fail validation.
"""
suggestions_to_update = []
suggestion_ids = suggestions_raw_latex_to_image_data_dict.keys()
suggestion_models_to_update = (
suggestion_models.GeneralSuggestionModel.get_multi(suggestion_ids))
for suggestion_model in suggestion_models_to_update:
suggestion = get_suggestion_from_model(suggestion_model)
suggestion_id = suggestion.suggestion_id
exp_id = suggestion.target_id
raw_latex_to_image_data_dict = (
suggestions_raw_latex_to_image_data_dict[suggestion_id])
add_svg_filenames_for_latex_strings_in_html_string = (
functools.partial(
html_validation_service.
add_svg_filenames_for_latex_strings_in_html_string,
raw_latex_to_image_data_dict))
suggestion.convert_html_in_suggestion_change(
add_svg_filenames_for_latex_strings_in_html_string)
updated_html_string = ''.join(suggestion.get_all_html_content_strings())
filenames_mapping = (
html_validation_service.
extract_svg_filename_latex_mapping_in_math_rte_components(
updated_html_string))
for filename, raw_latex in filenames_mapping:
# Some new filenames may already have the images saved from the math
# rich-text editor, for these files we don't need to save the image
# again.
if raw_latex in raw_latex_to_image_data_dict.keys():
image_file = raw_latex_to_image_data_dict[raw_latex].raw_image
image_validation_error_message_suffix = (
'SVG image provided for latex %s failed validation' % (
raw_latex))
try:
file_format = (
image_validation_services.validate_image_and_filename(
image_file, filename))
except utils.ValidationError as e:
e = '%s %s' % (e, image_validation_error_message_suffix)
raise Exception(e)
image_is_compressible = (
file_format in feconf.COMPRESSIBLE_IMAGE_FORMATS)
fs_services.save_original_and_compressed_versions_of_image(
filename, feconf.ENTITY_TYPE_EXPLORATION, exp_id,
image_file, 'image', image_is_compressible)
suggestions_to_update.append(suggestion)
_update_suggestions(suggestions_to_update, update_last_updated_time=False)
