def map(item):
if item.deleted or item.suggestion_type != (
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT):
return
suggestion = suggestion_services.get_suggestion_from_model(item)
exploration = exp_fetchers.get_exploration_by_id(suggestion.target_id)
if suggestion.change.state_name not in exploration.states:
return
state = exploration.states[suggestion.change.state_name]
subtitled_unicode_content_ids = []
customisation_args = state.interaction.customization_args
for ca_name in customisation_args:
subtitled_unicode_content_ids.extend(
state_domain.InteractionCustomizationArg.
traverse_by_schema_and_get(
customisation_args[ca_name].schema,
customisation_args[ca_name].value,
[schema_utils.SCHEMA_OBJ_TYPE_SUBTITLED_UNICODE],
lambda subtitled_unicode: subtitled_unicode.content_id))
if suggestion.change.content_id in subtitled_unicode_content_ids:
if suggestion.change.cmd == exp_domain.CMD_ADD_WRITTEN_TRANSLATION:
suggestion.change.data_format = (
schema_utils.SCHEMA_TYPE_UNICODE)
suggestion.change.translation_html = html_cleaner.strip_html_tags(
suggestion.change.translation_html)
item.change_cmd = suggestion.change.to_dict()
item.update_timestamps(update_last_updated_time=False)
item.put()
yield ('UPDATED',
'%s | %s' % (item.id, suggestion.change.content_id))
yield ('PROCESSED', item.id)
def test_strip_html_tags(self) -> None:
test_data: List[Tuple[str, str]] = [
(
'<a href="http://www.google.com">Hello</a>',
'Hello',
), (
'Just some text 12345',
'Just some text 12345',
), (
'<code>Unfinished HTML',
'Unfinished HTML',
), (
'<br/>',
'',
),
(
'A big mix <div>Hello</div> Yes <span>No</span>',
'A big mix Hello Yes No',
), (
'Text with\nnewlines',
'Text with\nnewlines',
)
]
for datum in test_data:
self.assertEqual(html_cleaner.strip_html_tags(datum[0]), datum[1])
def _get_plain_text_from_html_content_string(html_content_string):
"""Retrieves the plain text from the given html content string. RTE element
occurrences in the html are replaced by their corresponding rte component
name, capitalized in square brackets.
eg: <p>Sample1 <oppia-noninteractive-math></oppia-noninteractive-math>
Sample2 </p> will give as output: Sample1 [Math] Sample2.
Note: similar logic exists in the frontend in format-rte-preview.filter.ts.
Args:
html_content_string: str. The content html string to convert to plain
text.
Returns:
str. The plain text string from the given html content string.
"""
def _replace_rte_tag(rte_tag):
"""Replaces all of the <oppia-noninteractive-**> tags with their
corresponding rte component name in square brackets.
Args:
rte_tag: MatchObject. A matched object that contins the
oppia-noninteractive rte tags.
Returns:
str. The string to replace the rte tags with.
"""
# Retrieve the matched string from the MatchObject.
rte_tag_string = rte_tag.group(0)
# Get the name of the rte tag. The hyphen is there as an optional
# matching character to cover the case where the name of the rte
# component is more than one word.
rte_tag_name = re.search(
r'oppia-noninteractive-(\w|-)+', rte_tag_string)
# Retrieve the matched string from the MatchObject.
rte_tag_name_string = rte_tag_name.group(0)
# Get the name of the rte component.
rte_component_name_string = rte_tag_name_string.split('-')[2:]
# If the component name is more than word, connect the words with spaces
# to create a single string.
rte_component_name_string = ' '.join(rte_component_name_string)
# Captialize each word in the string.
capitalized_rte_component_name_string = (
rte_component_name_string.title())
formatted_rte_component_name_string = ' [%s] ' % (
capitalized_rte_component_name_string)
return formatted_rte_component_name_string
# Replace all the <oppia-noninteractive-**> tags with their rte component
# names capitalized in square brackets.
html_content_string_with_rte_tags_replaced = re.sub(
r'<oppia-noninteractive-[^>]+>(.*?)</oppia-noninteractive-[^>]+>',
_replace_rte_tag, html_content_string)
# Get rid of all of the other html tags.
plain_text = html_cleaner.strip_html_tags(
html_content_string_with_rte_tags_replaced)
# Remove trailing and leading whitespace and ensure that all words are
# separated by a single space.
plain_text_without_contiguous_whitespace = ' '.join(plain_text.split())
return plain_text_without_contiguous_whitespace
def _send_bulk_mail(
recipient_ids, sender_id, intent, email_subject, email_html_body, sender_email, sender_name, instance_id=None
):
"""Sends an email to all given recipients.
Args:
recipient_ids: list(str). The user IDs of the email recipients.
sender_id: str. The ID of the user sending the email.
intent: str. The intent string, i.e. the purpose of the email.
email_subject: str. The subject of the email.
email_html_body: str. The body (message) of the email.
sender_email: str. The sender's email address.
sender_name: str. The name to be shown in the "sender" field of the
email.
instance_id: str or None. The ID of the BulkEmailModel entity instance.
"""
_require_sender_id_is_valid(intent, sender_id)
recipients_settings = user_services.get_users_settings(recipient_ids)
recipient_emails = [user.email for user in recipients_settings]
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
"Original email HTML body does not match cleaned HTML body:\n"
"Original:\n%s\n\nCleaned:\n%s\n" % (email_html_body, cleaned_html_body)
)
return
raw_plaintext_body = (
cleaned_html_body.replace("<br/>", "\n")
.replace("<br>", "\n")
.replace("<li>", "<li>- ")
.replace("</p><p>", "</p>\n<p>")
)
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
def _send_bulk_mail_in_transaction(instance_id=None):
sender_name_email = "%s <%s>" % (sender_name, sender_email)
email_services.send_bulk_mail(
sender_name_email, recipient_emails, email_subject, cleaned_plaintext_body, cleaned_html_body
)
if instance_id is None:
instance_id = email_models.BulkEmailModel.get_new_id("")
email_models.BulkEmailModel.create(
instance_id,
recipient_ids,
sender_id,
sender_name_email,
intent,
email_subject,
cleaned_html_body,
datetime.datetime.utcnow(),
)
return transaction_services.run_in_transaction(_send_bulk_mail_in_transaction, instance_id)
def _send_bulk_mail(recipient_ids,
sender_id,
intent,
email_subject,
email_html_body,
sender_email,
sender_name,
instance_id=None):
"""Sends an email to all given recipients.
Args:
recipient_ids: list(str). The user IDs of the email recipients.
sender_id: str. The ID of the user sending the email.
intent: str. The intent string, i.e. the purpose of the email.
email_subject: str. The subject of the email.
email_html_body: str. The body (message) of the email.
sender_email: str. The sender's email address.
sender_name: str. The name to be shown in the "sender" field of the
email.
instance_id: str or None. The ID of the BulkEmailModel entity instance.
"""
_require_sender_id_is_valid(intent, sender_id)
recipients_settings = user_services.get_users_settings(recipient_ids)
recipient_emails = [user.email for user in recipients_settings]
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
'Original email HTML body does not match cleaned HTML body:\n'
'Original:\n%s\n\nCleaned:\n%s\n' %
(email_html_body, cleaned_html_body))
return
raw_plaintext_body = cleaned_html_body.replace('<br/>', '\n').replace(
'<br>', '\n').replace('<li>',
'<li>- ').replace('</p><p>', '</p>\n<p>')
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
def _send_bulk_mail_in_transaction(instance_id=None):
"""Sends the emails in bulk to the recipients."""
sender_name_email = '%s <%s>' % (sender_name, sender_email)
email_services.send_bulk_mail(sender_name_email, recipient_emails,
email_subject, cleaned_plaintext_body,
cleaned_html_body)
if instance_id is None:
instance_id = email_models.BulkEmailModel.get_new_id('')
email_models.BulkEmailModel.create(instance_id, recipient_ids,
sender_id, sender_name_email,
intent, email_subject,
cleaned_html_body,
datetime.datetime.utcnow())
transaction_services.run_in_transaction(_send_bulk_mail_in_transaction,
instance_id)
def _send_email(recipient_id,
sender_id,
intent,
email_subject,
email_html_body,
sender_email,
bcc_admin=False,
sender_name=None):
"""Sends an email to the given recipient.
This function should be used for sending all user-facing emails.
Raises an Exception if the sender_id is not appropriate for the given
intent. Currently we support only system-generated emails and emails
initiated by moderator actions.
"""
if sender_name is None:
sender_name = EMAIL_SENDER_NAME.value
_require_sender_id_is_valid(intent, sender_id)
recipient_email = user_services.get_email_from_user_id(recipient_id)
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
'Original email HTML body does not match cleaned HTML body:\n'
'Original:\n%s\n\nCleaned:\n%s\n' %
(email_html_body, cleaned_html_body))
return
raw_plaintext_body = cleaned_html_body.replace('<br/>', '\n').replace(
'<br>', '\n').replace('<li>',
'<li>- ').replace('</p><p>', '</p>\n<p>')
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
if email_models.SentEmailModel.check_duplicate_message(
recipient_id, email_subject, cleaned_plaintext_body):
log_new_error('Duplicate email:\n'
'Details:\n%s %s\n%s\n\n' %
(recipient_id, email_subject, cleaned_plaintext_body))
return
def _send_email_in_transaction():
sender_name_email = '%s <%s>' % (sender_name, sender_email)
email_services.send_mail(sender_name_email, recipient_email,
email_subject, cleaned_plaintext_body,
cleaned_html_body, bcc_admin)
email_models.SentEmailModel.create(recipient_id, recipient_email,
sender_id, sender_name_email,
intent, email_subject,
cleaned_html_body,
datetime.datetime.utcnow())
return transaction_services.run_in_transaction(_send_email_in_transaction)
def map(item):
"""Implements the map function (generator). Computes word counts of
translations suggestions and outputs suggestion metadata.
Args:
item: GeneralSuggestionModel. An instance of GeneralSuggestionModel.
Yields:
tuple(key, recent_activity_commits). Where:
key: str. The entity ID of the corresponding
TranslationContributionStatsModel.
dict. Has the keys:
suggestion_status: str. The translation suggestion status.
edited_by_reviewer: bool. Whether the translation suggestion
was edited by a reviewer.
content_word_count: int. The word count of the translation
suggestion content HTML.
last_updated_date: date. The last updated date of the
translation suggestion.
"""
if item.suggestion_type != feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT:
return
suggestion = suggestion_services.get_suggestion_from_model(item)
# Try to extract the topic ID from the corresponding exploration
# opportunity.
topic_id = ''
exp_id = suggestion.target_id
exp_opportunity_dict = (
opportunity_services.get_exploration_opportunity_summaries_by_ids(
[exp_id]))
exp_opportunity = exp_opportunity_dict[exp_id]
if exp_opportunity is not None:
topic_id = exp_opportunity.topic_id
# Count the number of words in the original content, ignoring any HTML
# tags and attributes.
content_plain_text = html_cleaner.strip_html_tags(
suggestion.change.content_html)
content_word_count = len(content_plain_text.split())
key = suggestion_models.TranslationContributionStatsModel.generate_id(
suggestion.language_code, suggestion.author_id, topic_id)
translation_contribution_stats_dict = {
'suggestion_status': suggestion.status,
'edited_by_reviewer': suggestion.edited_by_reviewer,
'content_word_count': content_word_count,
'last_updated_date': suggestion.last_updated.date().isoformat()
}
yield (key, translation_contribution_stats_dict)
def generate_summary_of_blog_post(content):
"""Generates the summary for a blog post from the content of the blog
post.
Args:
content: santized html str. The blog post content to be truncated.
Returns:
str. The summary of the blog post.
"""
raw_text = html_cleaner.strip_html_tags(content)
max_chars_in_summary = constants.MAX_CHARS_IN_BLOG_POST_SUMMARY - 3
summary = raw_text[:max_chars_in_summary] + '...'
return summary
def _send_email(
recipient_id, sender_id, intent, email_subject, email_html_body,
sender_email, bcc_admin=False, sender_name=None):
"""Sends an email to the given recipient.
This function should be used for sending all user-facing emails.
Raises an Exception if the sender_id is not appropriate for the given
intent. Currently we support only system-generated emails and emails
initiated by moderator actions.
"""
if sender_name is None:
sender_name = EMAIL_SENDER_NAME.value
_require_sender_id_is_valid(intent, sender_id)
recipient_email = user_services.get_email_from_user_id(recipient_id)
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
'Original email HTML body does not match cleaned HTML body:\n'
'Original:\n%s\n\nCleaned:\n%s\n' %
(email_html_body, cleaned_html_body))
return
raw_plaintext_body = cleaned_html_body.replace('<br/>', '\n').replace(
'<br>', '\n').replace('<li>', '<li>- ').replace('</p><p>', '</p>\n<p>')
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
if email_models.SentEmailModel.check_duplicate_message(
recipient_id, email_subject, cleaned_plaintext_body):
log_new_error(
'Duplicate email:\n'
'Details:\n%s %s\n%s\n\n' %
(recipient_id, email_subject, cleaned_plaintext_body))
return
def _send_email_in_transaction():
sender_name_email = '%s <%s>' % (sender_name, sender_email)
email_services.send_mail(
sender_name_email, recipient_email, email_subject,
cleaned_plaintext_body, cleaned_html_body, bcc_admin)
email_models.SentEmailModel.create(
recipient_id, recipient_email, sender_id, sender_name_email, intent,
email_subject, cleaned_html_body, datetime.datetime.utcnow())
return transaction_services.run_in_transaction(_send_email_in_transaction)
def _generate_stats(
suggestions: Iterable[suggestion_registry.SuggestionTranslateContent],
opportunity: Optional[opportunity_domain.ExplorationOpportunitySummary]
) -> Iterable[Tuple[str, Dict[str, Union[bool, int, str]]]]:
"""Generates translation contribution stats for each suggestion.
Args:
suggestions: iter(SuggestionTranslateContent). Suggestions for which
the stats should be generated.
opportunity: ExplorationOpportunitySummary. Opportunity for which
were the suggestions generated. Used to extract topic ID.
Yields:
tuple(str, Dict(str, *)). Tuple of key and suggestion stats dict.
The stats dictionary has four fields:
suggestion_status: str. What is the status of the suggestion.
edited_by_reviewer: bool. Whether the suggestion was edited by
the reviewer.
content_word_count: int. The word count of the content of
the suggestion.
last_updated_date: str. When was the suggestion last updated.
"""
# When opportunity is not available we leave the topic ID empty.
topic_id = ''
if opportunity is not None:
topic_id = opportunity.topic_id
for suggestion in suggestions:
# Count the number of words in the original content, ignoring any
# HTML tags and attributes.
content_plain_text = html_cleaner.strip_html_tags( # type: ignore[no-untyped-call]
suggestion.change.content_html) # type: ignore[attr-defined]
content_word_count = len(content_plain_text.split())
key = (suggestion_models.TranslationContributionStatsModel.
generate_id(suggestion.language_code, suggestion.author_id,
topic_id))
translation_contribution_stats_dict = {
'suggestion_status': suggestion.status,
'edited_by_reviewer': suggestion.edited_by_reviewer,
'content_word_count': content_word_count,
'last_updated_date':
suggestion.last_updated.date().isoformat()
}
yield (key, translation_contribution_stats_dict)
def _send_email(recipient_id, sender_id, intent, email_subject, email_html_body):
"""Sends an email to the given recipient.
This function should be used for sending all user-facing emails.
Raises an Exception if the sender_id is not appropriate for the given
intent. Currently we support only system-generated emails and emails
initiated by moderator actions.
"""
_require_sender_id_is_valid(intent, sender_id)
recipient_email = user_services.get_email_from_user_id(recipient_id)
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
"Original email HTML body does not match cleaned HTML body:\n"
"Original:\n%s\n\nCleaned:\n%s\n" % (email_html_body, cleaned_html_body)
)
return
raw_plaintext_body = cleaned_html_body.replace("<br/>", "\n").replace("<br>", "\n").replace("</p><p>", "</p>\n<p>")
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
def _send_email_in_transaction():
sender_email = "%s <%s>" % (EMAIL_SENDER_NAME.value, feconf.SYSTEM_EMAIL_ADDRESS)
email_services.send_mail(
sender_email, recipient_email, email_subject, cleaned_plaintext_body, cleaned_html_body
)
email_models.SentEmailModel.create(
recipient_id,
recipient_email,
sender_id,
sender_email,
intent,
email_subject,
cleaned_html_body,
datetime.datetime.utcnow(),
)
return transaction_services.run_in_transaction(_send_email_in_transaction)
def test_strip_html_tags(self):
TEST_DATA = [(
'<a href="http://www.google.com">Hello</a>',
'Hello',
), (
'Just some text 12345',
'Just some text 12345',
), (
'<code>Unfinished HTML',
'Unfinished HTML',
), (
'<br/>',
'',
), (
'A big mix <div>Hello</div> Yes <span>No</span>',
'A big mix Hello Yes No',
), (
'Text with\nnewlines',
'Text with\nnewlines',
)]
for datum in TEST_DATA:
self.assertEqual(html_cleaner.strip_html_tags(datum[0]), datum[1])
def _send_bulk_mail(
recipient_ids, sender_id, intent, email_subject, email_html_body,
sender_email, sender_name, instance_id=None):
"""Sends an email to all given recipients."""
_require_sender_id_is_valid(intent, sender_id)
recipients_settings = user_services.get_users_settings(recipient_ids)
recipient_emails = [user.email for user in recipients_settings]
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
'Original email HTML body does not match cleaned HTML body:\n'
'Original:\n%s\n\nCleaned:\n%s\n' %
(email_html_body, cleaned_html_body))
return
raw_plaintext_body = cleaned_html_body.replace('<br/>', '\n').replace(
'<br>', '\n').replace('<li>', '<li>- ').replace('</p><p>', '</p>\n<p>')
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
def _send_bulk_mail_in_transaction(instance_id=None):
sender_name_email = '%s <%s>' % (sender_name, sender_email)
email_services.send_bulk_mail(
sender_name_email, recipient_emails, email_subject,
cleaned_plaintext_body, cleaned_html_body)
if instance_id is None:
instance_id = email_models.BulkEmailModel.get_new_id('')
email_models.BulkEmailModel.create(
instance_id, recipient_ids, sender_id, sender_name_email, intent,
email_subject, cleaned_html_body, datetime.datetime.utcnow())
return transaction_services.run_in_transaction(
_send_bulk_mail_in_transaction, instance_id)
def _send_email(recipient_id,
sender_id,
intent,
email_subject,
email_html_body,
sender_email,
bcc_admin=False,
sender_name=None,
reply_to_id=None):
"""Sends an email to the given recipient.
This function should be used for sending all user-facing emails.
Raises an Exception if the sender_id is not appropriate for the given
intent. Currently we support only system-generated emails and emails
initiated by moderator actions.
Args:
recipient_id: str. The user ID of the recipient.
sender_id: str. The user ID of the sender.
intent: str. The intent string for the email, i.e. the purpose/type.
email_subject: str. The subject of the email.
email_html_body: str. The body (message) of the email.
sender_email: str. The sender's email address.
bcc_admin: bool. Whether to send a copy of the email to the admin's
email address.
sender_name: str or None. The name to be shown in the "sender" field of
the email.
reply_to_id: str or None. The unique reply-to id used in reply-to email
address sent to recipient.
"""
if sender_name is None:
sender_name = EMAIL_SENDER_NAME.value
_require_sender_id_is_valid(intent, sender_id)
recipient_email = user_services.get_email_from_user_id(recipient_id)
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
'Original email HTML body does not match cleaned HTML body:\n'
'Original:\n%s\n\nCleaned:\n%s\n' %
(email_html_body, cleaned_html_body))
return
raw_plaintext_body = cleaned_html_body.replace('<br/>', '\n').replace(
'<br>', '\n').replace('<li>',
'<li>- ').replace('</p><p>', '</p>\n<p>')
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
if email_models.SentEmailModel.check_duplicate_message(
recipient_id, email_subject, cleaned_plaintext_body):
log_new_error('Duplicate email:\n'
'Details:\n%s %s\n%s\n\n' %
(recipient_id, email_subject, cleaned_plaintext_body))
return
def _send_email_in_transaction():
"""Sends the email to a single recipient."""
sender_name_email = '%s <%s>' % (sender_name, sender_email)
email_services.send_mail(sender_name_email,
recipient_email,
email_subject,
cleaned_plaintext_body,
cleaned_html_body,
bcc_admin,
reply_to_id=reply_to_id)
email_models.SentEmailModel.create(recipient_id, recipient_email,
sender_id, sender_name_email,
intent, email_subject,
cleaned_html_body,
datetime.datetime.utcnow())
transaction_services.run_in_transaction(_send_email_in_transaction)
def _send_email(
recipient_id, sender_id, intent, email_subject, email_html_body,
sender_email, bcc_admin=False, sender_name=None, reply_to_id=None):
"""Sends an email to the given recipient.
This function should be used for sending all user-facing emails.
Raises an Exception if the sender_id is not appropriate for the given
intent. Currently we support only system-generated emails and emails
initiated by moderator actions.
Args:
recipient_id: str. The user ID of the recipient.
sender_id: str. The user ID of the sender.
intent: str. The intent string for the email, i.e. the purpose/type.
email_subject: str. The subject of the email.
email_html_body: str. The body (message) of the email.
sender_email: str. The sender's email address.
bcc_admin: bool. Whether to send a copy of the email to the admin's
email address.
sender_name: str or None. The name to be shown in the "sender" field of
the email.
reply_to_id: str or None. The unique reply-to id used in reply-to email
address sent to recipient.
"""
if sender_name is None:
sender_name = EMAIL_SENDER_NAME.value
_require_sender_id_is_valid(intent, sender_id)
recipient_email = user_services.get_email_from_user_id(recipient_id)
cleaned_html_body = html_cleaner.clean(email_html_body)
if cleaned_html_body != email_html_body:
log_new_error(
'Original email HTML body does not match cleaned HTML body:\n'
'Original:\n%s\n\nCleaned:\n%s\n' %
(email_html_body, cleaned_html_body))
return
raw_plaintext_body = cleaned_html_body.replace('<br/>', '\n').replace(
'<br>', '\n').replace('<li>', '<li>- ').replace('</p><p>', '</p>\n<p>')
cleaned_plaintext_body = html_cleaner.strip_html_tags(raw_plaintext_body)
if email_models.SentEmailModel.check_duplicate_message(
recipient_id, email_subject, cleaned_plaintext_body):
log_new_error(
'Duplicate email:\n'
'Details:\n%s %s\n%s\n\n' %
(recipient_id, email_subject, cleaned_plaintext_body))
return
def _send_email_in_transaction():
sender_name_email = '%s <%s>' % (sender_name, sender_email)
email_services.send_mail(
sender_name_email, recipient_email, email_subject,
cleaned_plaintext_body, cleaned_html_body, bcc_admin,
reply_to_id=reply_to_id)
email_models.SentEmailModel.create(
recipient_id, recipient_email, sender_id, sender_name_email, intent,
email_subject, cleaned_html_body, datetime.datetime.utcnow())
return transaction_services.run_in_transaction(_send_email_in_transaction)
def _generate_stats(
suggestions: Iterable[suggestion_registry.SuggestionTranslateContent],
opportunity: Optional[opportunity_domain.ExplorationOpportunitySummary]
) -> Tuple[str, result.Result[Dict[str, Union[bool, int, str]], str]]:
"""Generates translation contribution stats for each suggestion.
Args:
suggestions: iter(SuggestionTranslateContent). Suggestions for which
the stats should be generated.
opportunity: ExplorationOpportunitySummary. Opportunity for which
were the suggestions generated. Used to extract topic ID.
Yields:
tuple(str, Dict(str, *)). Tuple of key and suggestion stats dict.
The stats dictionary has four fields:
suggestion_status: str. What is the status of the suggestion.
edited_by_reviewer: bool. Whether the suggestion was edited by
the reviewer.
content_word_count: int. The word count of the content of
the suggestion.
last_updated_date: str. When was the suggestion last updated.
"""
# When opportunity is not available we leave the topic ID empty.
topic_id = ''
if opportunity is not None:
topic_id = opportunity.topic_id
for suggestion in suggestions:
key = (suggestion_models.TranslationContributionStatsModel.
generate_id(suggestion.language_code, suggestion.author_id,
topic_id))
try:
change = suggestion.change
# In the new translation command the content in set format is
# a list, content in unicode and html format is a string.
# This code normalizes the content to the list type so that
# we can easily count words.
if (change.cmd == exp_domain.CMD_ADD_WRITTEN_TRANSLATION and
state_domain.WrittenTranslation.is_data_format_list(
change.data_format)):
content_items = change.content_html
else:
content_items = [change.content_html]
content_word_count = 0
for item in content_items:
# Count the number of words in the original content,
# ignoring any HTML tags and attributes.
content_plain_text = html_cleaner.strip_html_tags(
item) # type: ignore[no-untyped-call,attr-defined]
content_word_count += len(content_plain_text.split())
translation_contribution_stats_dict = {
'suggestion_status':
suggestion.status,
'edited_by_reviewer':
suggestion.edited_by_reviewer,
'content_word_count':
content_word_count,
'last_updated_date':
(suggestion.last_updated.date().isoformat())
}
yield (key, result.Ok(translation_contribution_stats_dict))
except Exception as e:
yield (key,
result.Err('%s: %s' % (suggestion.suggestion_id, e)))
