def validate_math_content_attribute_in_html(html_string):
"""Validates the format of SVG filenames for each math rich-text components
and returns a list of all invalid math tags in the given HTML.
Args:
html_string: str. The HTML string.
Returns:
list(dict(str, str)). A list of dicts each having the invalid tags in
the HTML string and the corresponding exception raised.
"""
soup = bs4.BeautifulSoup(html_string, 'html.parser')
error_list = []
for math_tag in soup.findAll(name='oppia-noninteractive-math'):
math_content_dict = (json.loads(
unescape_html(math_tag['math_content-with-value'])))
try:
components.Math.validate(
{'math_content-with-value': math_content_dict})
except utils.ValidationError as e:
error_list.append({
'invalid_tag': python_utils.UNICODE(math_tag),
'error': python_utils.UNICODE(e)
})
return error_list
def _generate_dummy_skill_and_questions(self):
"""Generate and loads the database with a skill and 15 questions
linked to the skill.
Raises:
Exception. Cannot load new structures data in production mode.
Exception. User does not have enough rights to generate data.
"""
if constants.DEV_MODE:
if feconf.ROLE_ID_CURRICULUM_ADMIN not in self.user.roles:
raise Exception(
'User does not have enough rights to generate data.')
skill_id = skill_services.get_new_skill_id()
skill_name = 'Dummy Skill %s' % python_utils.UNICODE(
random.getrandbits(32))
skill = self._create_dummy_skill(
skill_id, skill_name, '<p>Dummy Explanation 1</p>')
skill_services.save_new_skill(self.user_id, skill)
for i in python_utils.RANGE(15):
question_id = question_services.get_new_question_id()
question_name = 'Question number %s %s' % (
python_utils.UNICODE(i), skill_name)
question = self._create_dummy_question(
question_id, question_name, [skill_id])
question_services.add_question(self.user_id, question)
question_difficulty = list(
constants.SKILL_DIFFICULTY_LABEL_TO_FLOAT.values())
random_difficulty = random.choice(question_difficulty)
question_services.create_new_question_skill_link(
self.user_id, question_id, skill_id, random_difficulty)
else:
raise Exception('Cannot generate dummy skills in production.')
def test_recursively_convert_to_str_with_dict(self):
test_var_1_in_unicode = python_utils.UNICODE('test_var_1')
test_var_2_in_unicode = python_utils.UNICODE('test_var_2')
test_var_3_in_bytes = test_var_1_in_unicode.encode(encoding='utf-8')
test_var_4_in_bytes = test_var_2_in_unicode.encode(encoding='utf-8')
test_dict = {
test_var_1_in_unicode: test_var_3_in_bytes,
test_var_2_in_unicode: test_var_4_in_bytes
}
self.assertEqual(test_dict, {
'test_var_1': b'test_var_1',
'test_var_2': b'test_var_2'
})
for key, val in test_dict.items():
self.assertEqual(type(key), python_utils.UNICODE)
self.assertEqual(type(val), builtins.bytes)
dict_in_str = python_utils._recursively_convert_to_str(test_dict) # pylint: disable=protected-access
self.assertEqual(dict_in_str, {
'test_var_1': 'test_var_1',
'test_var_2': 'test_var_2'
})
for key, val in dict_in_str.items():
self.assertEqual(type(key), python_utils.UNICODE)
self.assertEqual(type(val), python_utils.UNICODE)
def test_flushing_and_executing_tasks_produces_correct_behavior(
self) -> None:
self.assertEqual(self.unit_test_emulator.get_number_of_tasks(), 0)
self.unit_test_emulator.create_task(self.queue_name1,
self.url,
payload=self.payload1)
self.unit_test_emulator.create_task(self.queue_name2,
self.url,
payload=self.payload2)
self.assertEqual(self.unit_test_emulator.get_number_of_tasks(), 2)
self.unit_test_emulator.process_and_flush_tasks(
queue_name=self.queue_name1)
self.assertEqual(self.output, [
'Task Default in queue %s with payload %s is sent to %s.' %
(self.queue_name1, python_utils.UNICODE(self.payload1), self.url)
])
self.assertEqual(self.unit_test_emulator.get_number_of_tasks(), 1)
self.unit_test_emulator.process_and_flush_tasks()
self.assertEqual(self.output, [
'Task Default in queue %s with payload %s is sent to %s.' %
(self.queue_name1, python_utils.UNICODE(self.payload1), self.url),
'Task Default in queue %s with payload %s is sent to %s.' %
(self.queue_name2, python_utils.UNICODE(self.payload2), self.url),
])
self.assertEqual(self.unit_test_emulator.get_number_of_tasks(), 0)
def validate_svg_filenames_in_math_rich_text(entity_type, entity_id,
html_string):
"""Validates the SVG filenames for each math rich-text components and
returns a list of all invalid math tags in the given HTML.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
html_string: str. The HTML string.
Returns:
list(str). A list of invalid math tags in the HTML string.
"""
soup = bs4.BeautifulSoup(html_string, 'html.parser')
error_list = []
for math_tag in soup.findAll(name='oppia-noninteractive-math'):
math_content_dict = (json.loads(
unescape_html(math_tag['math_content-with-value'])))
svg_filename = (objects.UnicodeString.normalize(
math_content_dict['svg_filename']))
if svg_filename == '':
error_list.append(python_utils.UNICODE(math_tag))
else:
file_system_class = fs_services.get_entity_file_system_class()
fs = fs_domain.AbstractFileSystem(
file_system_class(entity_type, entity_id))
filepath = 'image/%s' % svg_filename
if not fs.isfile(filepath):
error_list.append(python_utils.UNICODE(math_tag))
return error_list
def _generate_id(cls, intent: str) -> str:
"""Generates an ID for a new SentEmailModel instance.
Args:
intent: str. The intent string, i.e. the purpose of the email.
Valid intent strings are defined in feconf.py.
Returns:
str. The newly-generated ID for the SentEmailModel instance.
Raises:
Exception. The id generator for SentEmailModel is producing
too many collisions.
"""
id_prefix = '%s.' % intent
for _ in range(base_models.MAX_RETRIES):
new_id = '%s.%s' % (id_prefix,
utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(
base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for SentEmailModel is producing too many '
'collisions.')
def get_collection_by_id(collection_id, strict=True, version=None):
"""Returns a domain object representing a collection.
Args:
collection_id: str. ID of the collection.
strict: bool. Whether to fail noisily if no collection with the given
id exists in the datastore.
version: int or None. The version number of the collection to be
retrieved. If it is None, the latest version will be retrieved.
Returns:
Collection or None. The domain object representing a collection with the
given id, or None if it does not exist.
"""
sub_namespace = python_utils.UNICODE(version) if version else None
cached_collection = caching_services.get_multi(
caching_services.CACHE_NAMESPACE_COLLECTION, sub_namespace,
[collection_id]).get(collection_id)
if cached_collection is not None:
return cached_collection
else:
collection_model = collection_models.CollectionModel.get(
collection_id, strict=strict, version=version)
if collection_model:
collection = get_collection_from_model(collection_model)
caching_services.set_multi(
caching_services.CACHE_NAMESPACE_COLLECTION, sub_namespace,
{collection_id: collection})
return collection
else:
return None
def render_template(self, filepath, iframe_restriction='DENY'):
"""Prepares an HTML response to be sent to the client.
Args:
filepath: str. The template filepath.
iframe_restriction: str or None. Possible values are
'DENY' and 'SAMEORIGIN':
DENY: Strictly prevents the template to load in an iframe.
SAMEORIGIN: The template can only be displayed in a frame
on the same origin as the page itself.
"""
# The 'no-store' must be used to properly invalidate the cache when we
# deploy a new version, using only 'no-cache' doesn't work properly.
self.response.cache_control.no_store = True
self.response.cache_control.must_revalidate = True
self.response.headers['Strict-Transport-Security'] = (
'max-age=31536000; includeSubDomains')
self.response.headers['X-Content-Type-Options'] = 'nosniff'
self.response.headers['X-Xss-Protection'] = '1; mode=block'
if iframe_restriction is not None:
if iframe_restriction in ['SAMEORIGIN', 'DENY']:
self.response.headers['X-Frame-Options'] = (
python_utils.UNICODE(iframe_restriction))
else:
raise Exception(
'Invalid X-Frame-Options: %s' % iframe_restriction)
self.response.expires = 'Mon, 01 Jan 1990 00:00:00 GMT'
self.response.pragma = 'no-cache'
self.response.write(load_template(filepath))
def _create_user_in_mailchimp_db(user_email: str) -> bool:
"""Creates a new user in the mailchimp database and handles the case where
the user was permanently deleted from the database.
Args:
user_email: str. Email ID of the user. Email is used to uniquely
identify the user in the mailchimp DB.
Returns:
bool. Whether the user was successfully added to the db. (This will be
False if the user was permanently deleted earlier and therefore cannot
be added back.)
Raises:
Exception. Any error (other than the one mentioned below) raised by the
mailchimp API.
"""
post_data = {'email_address': user_email, 'status': 'subscribed'}
client = _get_mailchimp_class()
try:
client.lists.members.create(feconf.MAILCHIMP_AUDIENCE_ID, post_data)
except mailchimpclient.MailChimpError as error:
error_message = ast.literal_eval(python_utils.UNICODE(error))
# This is the specific error message returned for the case where the
# user was permanently deleted from the Mailchimp database earlier.
# This was found by experimenting with the MailChimp API. Note that the
# error reference
# (https://mailchimp.com/developer/marketing/docs/errors/) is not
# comprehensive, since, under status 400, they only list a subset of the
# common error titles.
if error_message['title'] == 'Forgotten Email Not Subscribed':
return False
raise Exception(error_message['detail'])
return True
def _create_token(cls, user_id, issued_on):
"""Creates a new CSRF token.
Args:
user_id: str|None. The user_id for which the token is generated.
issued_on: float. The timestamp at which the token was issued.
Returns:
str. The generated CSRF token.
"""
cls.init_csrf_secret()
# The token has 4 parts: hash of the actor user id, hash of the page
# name, hash of the time issued and plain text of the time issued.
if user_id is None:
user_id = cls._USER_ID_DEFAULT
# Round time to seconds.
issued_on = python_utils.UNICODE(int(issued_on))
digester = hmac.new(CSRF_SECRET.value.encode('utf-8'))
digester.update(user_id.encode('utf-8'))
digester.update(b':')
digester.update(issued_on.encode('utf-8'))
digest = digester.digest()
# The b64encode returns bytes, so we first need to decode the returned
# bytes to string.
token = '%s/%s' % (
issued_on, base64.urlsafe_b64encode(digest).decode('utf-8'))
return token
def get_skill_by_id(skill_id, strict=True, version=None):
"""Returns a domain object representing a skill.
Args:
skill_id: str. ID of the skill.
strict: bool. Whether to fail noisily if no skill with the given
id exists in the datastore.
version: int or None. The version number of the skill to be
retrieved. If it is None, the latest version will be retrieved.
Returns:
Skill or None. The domain object representing a skill with the
given id, or None if it does not exist.
"""
sub_namespace = python_utils.UNICODE(version) if version else None
cached_skill = caching_services.get_multi(
caching_services.CACHE_NAMESPACE_SKILL, sub_namespace,
[skill_id]).get(skill_id)
if cached_skill is not None:
return cached_skill
else:
skill_model = skill_models.SkillModel.get(skill_id,
strict=strict,
version=version)
if skill_model:
skill = get_skill_from_model(skill_model)
caching_services.set_multi(caching_services.CACHE_NAMESPACE_SKILL,
sub_namespace, {skill_id: skill})
return skill
else:
return None
def update_developer_names(release_summary_lines):
"""Updates about-page.constants.ts file.
Args:
release_summary_lines: list(str). List of lines in
../release_summary.md.
"""
python_utils.PRINT('Updating about-page file...')
new_developer_names = get_new_contributors(release_summary_lines,
return_only_names=True)
with python_utils.open_file(ABOUT_PAGE_CONSTANTS_FILEPATH,
'r') as about_page_file:
about_page_lines = about_page_file.readlines()
start_index = about_page_lines.index(CREDITS_START_LINE) + 1
end_index = about_page_lines[start_index:].index(CREDITS_END_LINE) + 1
all_developer_names = about_page_lines[start_index:end_index]
for name in new_developer_names:
all_developer_names.append('%s\'%s\',\n' % (CREDITS_INDENT, name))
all_developer_names = sorted(list(set(all_developer_names)),
key=lambda s: s.lower())
about_page_lines[start_index:end_index] = all_developer_names
with python_utils.open_file(ABOUT_PAGE_CONSTANTS_FILEPATH,
'w') as about_page_file:
for line in about_page_lines:
about_page_file.write(python_utils.UNICODE(line))
python_utils.PRINT('Updated about-page file!')
def generate_id(cls, platform: str,
submitted_on_datetime: datetime.datetime) -> str:
"""Generates key for the instance of AppFeedbackReportModel class in the
required format with the arguments provided.
Args:
platform: str. The platform the user is the report from.
submitted_on_datetime: datetime.datetime. The datetime that the
report was submitted on in UTC.
Returns:
str. The generated ID for this entity using platform,
submitted_on_sec, and a random string, of the form
'[platform].[submitted_on_msec].[random hash]'.
"""
submitted_datetime_in_msec = utils.get_time_in_millisecs(
submitted_on_datetime)
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
random_hash = utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (platform, int(submitted_datetime_in_msec),
random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportModel is producing too '
'many collisions.')
def generate_id(cls, ticket_name: str) -> str:
"""Generates key for the instance of AppFeedbackReportTicketModel
class in the required format with the arguments provided.
Args:
ticket_name: str. The name assigned to the ticket on creation.
Returns:
str. The generated ID for this entity using the current datetime in
milliseconds (as the entity's creation timestamp), a SHA1 hash of
the ticket_name, and a random string, of the form
'[creation_datetime_msec]:[hash(ticket_name)]:[random hash]'.
"""
current_datetime_in_msec = utils.get_time_in_millisecs(
datetime.datetime.utcnow())
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
name_hash = utils.convert_to_hash(ticket_name,
base_models.ID_LENGTH)
random_hash = utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (int(current_datetime_in_msec), name_hash,
random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportTicketModel is producing too'
'many collisions.')
def get_topic_by_id(topic_id, strict=True, version=None):
"""Returns a domain object representing a topic.
Args:
topic_id: str. ID of the topic.
strict: bool. Whether to fail noisily if no topic with the given
id exists in the datastore.
version: int or None. The version number of the topic to be
retrieved. If it is None, the latest version will be retrieved.
Returns:
Topic or None. The domain object representing a topic with the
given id, or None if it does not exist.
"""
sub_namespace = python_utils.UNICODE(version) if version else None
cached_topic = caching_services.get_multi(
caching_services.CACHE_NAMESPACE_TOPIC, sub_namespace,
[topic_id]).get(topic_id)
if cached_topic is not None:
return cached_topic
else:
topic_model = topic_models.TopicModel.get(topic_id,
strict=strict,
version=version)
if topic_model:
topic = get_topic_from_model(topic_model)
caching_services.set_multi(caching_services.CACHE_NAMESPACE_TOPIC,
sub_namespace, {topic_id: topic})
return topic
else:
return None
def _generate_id(cls, exp_id: str) -> str:
"""Generates a unique id for the training job of the form
'[exp_id].[random hash of 16 chars]'.
Args:
exp_id: str. ID of the exploration.
Returns:
str. ID of the new ClassifierTrainingJobModel instance.
Raises:
Exception. The id generator for ClassifierTrainingJobModel is
producing too many collisions.
"""
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
new_id = '%s.%s' % (
exp_id,
utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for ClassifierTrainingJobModel is producing '
'too many collisions.')
def test_tasks_scheduled_for_immediate_execution_are_handled_correctly(
self) -> None:
self.dev_mode_emulator.create_task(self.queue_name1,
self.url,
payload=self.payload1)
self.dev_mode_emulator.create_task(self.queue_name2,
self.url,
payload=self.payload2)
# Allow the threads to execute the tasks scheduled immediately.
time.sleep(1)
self.assertEqual(self.output, [
'Task Default in queue %s with payload %s is sent to %s.' %
(self.queue_name1, python_utils.UNICODE(self.payload1), self.url),
'Task Default in queue %s with payload %s is sent to %s.' %
(self.queue_name2, python_utils.UNICODE(self.payload2), self.url),
])
def mock_task_handler(self,
url: str,
payload: Dict[str, Any],
queue_name: str,
task_name: Optional[str] = None) -> None:
self.output.append(
'Task %s in queue %s with payload %s is sent to %s.' %
(task_name if task_name else 'Default', queue_name,
python_utils.UNICODE(payload), url))
def _reload_exploration(self, exploration_id):
"""Reloads the exploration in dev_mode corresponding to the given
exploration id.
Args:
exploration_id: str. The exploration id.
Raises:
Exception. Cannot reload an exploration in production.
"""
if constants.DEV_MODE:
logging.info('[ADMIN] %s reloaded exploration %s' %
(self.user_id, exploration_id))
exp_services.load_demo(python_utils.UNICODE(exploration_id))
rights_manager.release_ownership_of_exploration(
user_services.get_system_user(),
python_utils.UNICODE(exploration_id))
else:
raise Exception('Cannot reload an exploration in production.')
def mock_print(*args):
"""Mock for python_utils.PRINT. Append the values to print to
task_stdout list.
Args:
*args: list(*). Variable length argument list of values to print
in the same line of output.
"""
self.task_stdout.append(
' '.join(python_utils.UNICODE(arg) for arg in args))
def _get_full_message_id(self, message_id):
"""Returns the full id of the message.
Args:
message_id: int. The id of the message for which we have to fetch
the complete message id.
Returns:
str. The full id corresponding to the given message id.
"""
return '.'.join([self.id, python_utils.UNICODE(message_id)])
def base64_from_int(value: int) -> str:
"""Converts the number into base64 representation.
Args:
value: int. Integer value for conversion into base64.
Returns:
str. Returns the base64 representation of the number passed.
"""
byte_value = (b'[' + python_utils.UNICODE(value).encode('utf-8') + b']')
return base64.b64encode(byte_value).decode('utf-8')
def _generate_id(cls, thread_id: str, message_id: int) -> str:
"""Generates full message ID given the thread ID and message ID.
Args:
thread_id: str. Thread ID of the thread to which the message
belongs.
message_id: int. Message ID of the message.
Returns:
str. Full message ID.
"""
return '.'.join([thread_id, python_utils.UNICODE(message_id)])
def to_ascii(input_string: str) -> str:
"""Change unicode characters in a string to ascii if possible.
Args:
input_string: str. String to convert.
Returns:
str. String containing the ascii representation of the input string.
"""
normalized_string = unicodedata.normalize(
'NFKD', python_utils.UNICODE(input_string))
return normalized_string.encode('ascii', 'ignore').decode('ascii')
def get_chrome_version():
"""Get the current version of Chrome.
Note that this only works on Linux systems. On macOS, for example,
the `google-chrome` command may not work.
Returns:
str. The version of Chrome we found.
"""
output = python_utils.UNICODE(
common.run_cmd(['google-chrome', '--version']))
chrome_version = ''.join(re.findall(r'([0-9]|\.)', output))
return chrome_version
def test_swap_to_always_raise_without_error_uses_empty_exception(self):
obj = mock.Mock()
obj.func = lambda: None
self.assertIsNone(obj.func())
with self.swap_to_always_raise(obj, 'func'):
try:
obj.func()
except Exception as e:
self.assertIs(type(e), Exception)
self.assertEqual(python_utils.UNICODE(e), '')
else:
self.fail(msg='obj.func() did not raise an Exception')
def fix_incorrectly_encoded_chars(html_string):
"""Replaces incorrectly encoded character with the correct one in a given
HTML string.
Args:
html_string: str. The HTML string to modify.
Returns:
str. The updated html string.
"""
return python_utils.UNICODE(
_process_string_with_components(html_string,
_replace_incorrectly_encoded_chars))
def convert_svg_diagram_tags_to_image_tags(html_string):
"""Renames all the oppia-noninteractive-svgdiagram on the server to
oppia-noninteractive-image and changes corresponding attributes.
Args:
html_string: str. The HTML string to check.
Returns:
str. The updated html string.
"""
return python_utils.UNICODE(
_process_string_with_components(html_string,
convert_svg_diagram_to_image_for_soup))
def test_update_developer_names(self):
with python_utils.open_file(
update_changelog_and_credits.ABOUT_PAGE_CONSTANTS_FILEPATH, 'r'
) as f:
about_page_lines = f.readlines()
start_index = about_page_lines.index(
update_changelog_and_credits.CREDITS_START_LINE) + 1
end_index = about_page_lines[start_index:].index(
update_changelog_and_credits.CREDITS_END_LINE) + 1
existing_developer_names = about_page_lines[start_index:end_index]
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.name = MOCK_ABOUT_PAGE_CONSTANTS_FILEPATH
with python_utils.open_file(
MOCK_ABOUT_PAGE_CONSTANTS_FILEPATH, 'w'
) as f:
for line in about_page_lines:
f.write(python_utils.UNICODE(line))
release_summary_lines = read_from_file(MOCK_RELEASE_SUMMARY_FILEPATH)
new_developer_names = update_changelog_and_credits.get_new_contributors(
release_summary_lines, return_only_names=True)
expected_developer_names = existing_developer_names
for name in new_developer_names:
expected_developer_names.append('%s\'%s\',\n' % (
update_changelog_and_credits.CREDITS_INDENT, name))
expected_developer_names = sorted(
list(set(expected_developer_names)), key=lambda s: s.lower())
with self.swap(
update_changelog_and_credits, 'ABOUT_PAGE_CONSTANTS_FILEPATH',
MOCK_ABOUT_PAGE_CONSTANTS_FILEPATH):
update_changelog_and_credits.update_developer_names(
release_summary_lines)
with python_utils.open_file(tmp_file.name, 'r') as f:
about_page_lines = f.readlines()
start_index = about_page_lines.index(
update_changelog_and_credits.CREDITS_START_LINE) + 1
end_index = about_page_lines[start_index:].index(
update_changelog_and_credits.CREDITS_END_LINE) + 1
actual_developer_names = about_page_lines[start_index:end_index]
self.assertEqual(actual_developer_names, expected_developer_names)
tmp_file.close()
if os.path.isfile(MOCK_ABOUT_PAGE_CONSTANTS_FILEPATH):
# Occasionally this temp file is not deleted.
os.remove(MOCK_ABOUT_PAGE_CONSTANTS_FILEPATH)
def assert_error_is_decorated(
self, actual_msg: str, decorated_msg: str
) -> None:
"""Asserts that decorate_beam_errors() raises with the right message.
Args:
actual_msg: str. The actual message raised originally.
decorated_msg: str. The expected decorated message produced by the
context manager.
"""
try:
with job_test_utils.decorate_beam_errors():
raise beam_testing_util.BeamAssertException(actual_msg)
except AssertionError as e:
self.assertMultiLineEqual(python_utils.UNICODE(e), decorated_msg)