def test_accumulate_one_more_than_limit_case_is_split(self) -> None:
accumulated_results = job_run_result.JobRunResult.accumulate([
job_run_result.JobRunResult(stdout='', stderr='a' * 2000),
job_run_result.JobRunResult(stdout='', stderr='b' * 3000),
])
self.assertEqual(len(accumulated_results), 2)
def test_unmigrated_story_with_unmigrated_rubric_is_migrated(self) -> None:
story_model = self.create_model(
story_models.StoryModel,
id=self.STORY_1_ID,
story_contents_schema_version=4,
title='title',
language_code='cs',
notes='notes',
description='description',
story_contents=self.unmigrated_contents,
corresponding_topic_id=self.TOPIC_1_ID,
url_fragment='urlfragment',
)
story_model.update_timestamps()
story_model.commit(feconf.SYSTEM_COMMITTER_ID, 'Create story',
[{
'cmd': story_domain.CMD_CREATE_NEW
}])
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='STORY PROCESSED SUCCESS: 1'),
job_run_result.JobRunResult(stdout='STORY MIGRATED SUCCESS: 1'),
job_run_result.JobRunResult(stdout='CACHE DELETION SUCCESS: 1')
])
migrated_story_model = story_models.StoryModel.get(self.STORY_1_ID)
self.assertEqual(migrated_story_model.version, 2)
self.assertEqual(migrated_story_model.story_contents_schema_version,
feconf.CURRENT_STORY_CONTENTS_SCHEMA_VERSION)
self.assertEqual(migrated_story_model.story_contents,
self.latest_contents)
def test_broken_cache_is_reported(self) -> None:
cache_swap = self.swap_to_always_raise(
caching_services, 'delete_multi',
Exception('cache deletion error'))
story_model = self.create_model(
story_models.StoryModel,
id=self.STORY_1_ID,
story_contents_schema_version=4,
title='title',
language_code='cs',
notes='notes',
description='description',
story_contents=self.latest_contents,
corresponding_topic_id=self.TOPIC_1_ID,
url_fragment='urlfragment',
)
story_model.update_timestamps()
story_model.commit(feconf.SYSTEM_COMMITTER_ID, 'Create story',
[{
'cmd': story_domain.CMD_CREATE_NEW
}])
with cache_swap:
self.assert_job_output_is([
job_run_result.JobRunResult(
stdout='STORY MIGRATED SUCCESS: 1'),
job_run_result.JobRunResult(
stdout='STORY PROCESSED SUCCESS: 1'),
job_run_result.JobRunResult(
stderr='CACHE DELETION ERROR: "cache deletion error": 1')
])
migrated_story_model = story_models.StoryModel.get(self.STORY_1_ID)
self.assertEqual(migrated_story_model.version, 2)
def test_counts_multiple_collection(self) -> None:
user1 = self.create_model(
user_models.UserSettingsModel,
id=self.USER_ID_1,
email='*****@*****.**',
roles=[feconf.ROLE_ID_COLLECTION_EDITOR]
)
user2 = self.create_model(
user_models.UserSettingsModel,
id=self.USER_ID_2,
email='*****@*****.**',
roles=[feconf.ROLE_ID_COLLECTION_EDITOR]
)
# Checking a user who has no collection.
user3 = self.create_model(
user_models.UserSettingsModel,
id=self.USER_ID_3,
email='*****@*****.**',
roles=[feconf.ROLE_ID_COLLECTION_EDITOR]
)
user1.update_timestamps()
user2.update_timestamps()
user3.update_timestamps()
collection1 = self.create_model(
collection_models.CollectionRightsModel,
id='col_1',
owner_ids=[self.USER_ID_1, self.USER_ID_2],
editor_ids=[self.USER_ID_1],
voice_artist_ids=[self.USER_ID_1],
community_owned=False,
status=constants.ACTIVITY_STATUS_PUBLIC,
viewable_if_private=False,
first_published_msec=0.2
)
collection1.update_timestamps()
collection2 = self.create_model(
collection_models.CollectionRightsModel,
id='col_2',
owner_ids=[self.USER_ID_2],
editor_ids=[self.USER_ID_1],
voice_artist_ids=[self.USER_ID_1],
community_owned=False,
status=constants.ACTIVITY_STATUS_PUBLIC,
viewable_if_private=False,
first_published_msec=0.2
)
collection2.update_timestamps()
self.put_multi([user1, user2, user3, collection1, collection2])
self.assert_job_output_is([
job_run_result.JobRunResult(
stdout=(
'collection_ids: [\'col_1\'], email: '
'[\'[email protected]\']')),
job_run_result.JobRunResult(
stdout=(
'collection_ids: [\'col_1\', \'col_2\'], email: '
'[\'[email protected]\']')
)
])
def test_equality(self):
a_result = job_run_result.JobRunResult(stdout='abc', stderr='123')
b_result = job_run_result.JobRunResult(stdout='def', stderr='456')
self.assertEqual(a_result, a_result)
self.assertEqual(b_result, b_result)
self.assertNotEqual(a_result, b_result)
def run(self) -> beam.PCollection[job_run_result.JobRunResult]:
user_settings_models = (
self.pipeline
| 'Get all UserSettingsModels' >>
(ndb_io.GetModels(user_models.UserSettingsModel.get_all())))
old_user_stats_models = (
self.pipeline
| 'Get all UserStatsModels' >>
(ndb_io.GetModels(user_models.UserStatsModel.get_all())))
# Creates UserStatsModels if it does not exists.
new_user_stats_models = (
(user_settings_models, old_user_stats_models)
| 'Merge models' >> beam.Flatten()
# Returns a PCollection of
# (model.id, (user_settings_models, user_stats_models)) or
# (model.id, (user_settings_models,)).
| 'Group models with same ID' >> beam.GroupBy(lambda m: m.id)
# Discards model.id from the PCollection.
| 'Get rid of key' >> beam.Values() # pylint: disable=no-value-for-parameter
# Only keep groupings that indicate that
# the UserStatsModel is missing.
| 'Filter pairs of models' >>
beam.Filter(lambda models: (len(list(models)) == 1 and isinstance(
list(models)[0], user_models.UserSettingsModel)))
# Choosing the first element.
| 'Transform tuples into models' >>
beam.Map(lambda models: list(models)[0])
# Creates the missing UserStatsModels.
| 'Create new user stat models' >> beam.ParDo(
CreateUserStatsModel()))
unused_put_result = (
(new_user_stats_models, old_user_stats_models)
| 'Merge new and old models together' >> beam.Flatten()
| 'Update the dashboard stats' >> beam.ParDo(
UpdateWeeklyCreatorStats())
| 'Put models into the datastore' >> ndb_io.PutModels())
new_user_stats_job_result = (
new_user_stats_models
| 'Count all new models' >> beam.combiners.Count.Globally()
| 'Only create result for new models when > 0' >>
(beam.Filter(lambda x: x > 0))
| 'Create result for new models' >>
beam.Map(lambda x: job_run_result.JobRunResult(
stdout='SUCCESS NEW %s' % x)))
old_user_stats_job_result = (
old_user_stats_models
| 'Count all old models' >> beam.combiners.Count.Globally()
| 'Only create result for old models when > 0' >>
(beam.Filter(lambda x: x > 0))
| 'Create result for old models' >>
beam.Map(lambda x: job_run_result.JobRunResult(
stdout='SUCCESS OLD %s' % x)))
return ((new_user_stats_job_result, old_user_stats_job_result)
| 'Merge new and old results together' >> beam.Flatten())
def test_accumulate(self) -> None:
single_job_run_result = job_run_result.JobRunResult.accumulate([
job_run_result.JobRunResult(stdout='abc', stderr=''),
job_run_result.JobRunResult(stdout='', stderr='123'),
job_run_result.JobRunResult(stdout='def', stderr='456'),
])[0]
self.assertItemsEqual( # type: ignore[no-untyped-call]
single_job_run_result.stdout.split('\n'), ['abc', 'def'])
self.assertItemsEqual( # type: ignore[no-untyped-call]
single_job_run_result.stderr.split('\n'), ['123', '456'])
def test_accumulate_one_less_than_limit_is_not_truncated(self) -> None:
accumulated_results = job_run_result.JobRunResult.accumulate([
job_run_result.JobRunResult(stdout='', stderr='a' * 1999),
job_run_result.JobRunResult(stdout='', stderr='b' * 3000),
])
self.assertEqual(len(accumulated_results), 1)
self.assertItemsEqual( # type: ignore[no-untyped-call]
accumulated_results[0].stderr.split('\n'),
['a' * 1999, 'b' * 3000])
def test_accumulate(self):
single_job_run_result = job_run_result.JobRunResult.accumulate([
job_run_result.JobRunResult(stdout='abc', stderr=''),
job_run_result.JobRunResult(stdout='', stderr='123'),
job_run_result.JobRunResult(stdout='def', stderr='456'),
])[0]
self.assertItemsEqual(single_job_run_result.stdout.split('\n'),
['abc', 'def'])
self.assertItemsEqual(single_job_run_result.stderr.split('\n'),
['123', '456'])
def test_accumulate_with_enormous_outputs(self):
accumulated_results = job_run_result.JobRunResult.accumulate([
job_run_result.JobRunResult(stdout='a' * 750, stderr='b' * 750),
job_run_result.JobRunResult(stdout='a' * 500, stderr='b' * 500),
job_run_result.JobRunResult(stdout='a' * 250, stderr='b' * 250),
job_run_result.JobRunResult(stdout='a' * 100, stderr='b' * 100),
job_run_result.JobRunResult(stdout='a' * 50, stderr='b' * 50),
])
# 100000 and 200000 are small enough ot fit as one, but the others will
# each need their own result.
self.assertEqual(len(accumulated_results), 3)
def test_unmigrated_skill_with_unmigrated_skill_contents_is_migrated(
self
) -> None:
skill_model = self.create_model(
skill_models.SkillModel,
id=self.SKILL_1_ID,
description='description',
misconceptions_schema_version=(
feconf.CURRENT_MISCONCEPTIONS_SCHEMA_VERSION),
rubric_schema_version=feconf.CURRENT_RUBRIC_SCHEMA_VERSION,
rubrics=self.latest_rubrics,
language_code='cs',
skill_contents_schema_version=3,
skill_contents={
'explanation': {
'content_id': 'content_id',
'html': '<b>bo ld\n</b>'
},
'worked_examples': [],
'recorded_voiceovers': {
'voiceovers_mapping': {
'content_id': {}
}
},
'written_translations': {
'translations_mapping': {
'content_id': {}
}
}
},
next_misconception_id=2,
all_questions_merged=False
)
skill_model.update_timestamps()
skill_model.commit(feconf.SYSTEM_COMMITTER_ID, 'Create skill', [{
'cmd': skill_domain.CMD_CREATE_NEW
}])
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SKILL PROCESSED SUCCESS: 1'),
job_run_result.JobRunResult(stdout='SKILL MIGRATED SUCCESS: 1'),
job_run_result.JobRunResult(stdout='CACHE DELETION SUCCESS: 1')
])
migrated_skill_model = skill_models.SkillModel.get(self.SKILL_1_ID)
self.assertEqual(migrated_skill_model.version, 2)
self.assertEqual(
migrated_skill_model.skill_contents_schema_version,
feconf.CURRENT_SKILL_CONTENTS_SCHEMA_VERSION)
self.assertEqual(
migrated_skill_model.skill_contents,
self.latest_skill_contents
)
def test_indexes_non_deleted_models(self) -> None:
for i in range(5):
exp_summary = self.create_model(
exp_models.ExpSummaryModel,
id='abcd%s' % i,
deleted=False,
title='title',
category='category',
objective='objective',
language_code='lang',
community_owned=False,
status=constants.ACTIVITY_STATUS_PUBLIC
)
exp_summary.update_timestamps()
exp_summary.put()
add_docs_to_index_swap = self.swap_with_checks(
platform_search_services,
'add_documents_to_index',
lambda _, __: None,
expected_args=[
(
[{
'id': 'abcd%s' % i,
'language_code': 'lang',
'title': 'title',
'category': 'category',
'tags': [],
'objective': 'objective',
'rank': 20,
}],
search_services.SEARCH_INDEX_EXPLORATIONS
) for i in range(5)
]
)
max_batch_size_swap = self.swap(
exp_search_indexing_jobs.IndexExplorationsInSearchJob,
'MAX_BATCH_SIZE', 1)
with add_docs_to_index_swap, max_batch_size_swap:
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS 1 models indexed'),
job_run_result.JobRunResult(stdout='SUCCESS 1 models indexed'),
job_run_result.JobRunResult(stdout='SUCCESS 1 models indexed'),
job_run_result.JobRunResult(stdout='SUCCESS 1 models indexed'),
job_run_result.JobRunResult(stdout='SUCCESS 1 models indexed'),
])
def test_job_deletes_all_opportunities(self) -> None:
skill_opportunity_model_1 = self.create_model(
opportunity_models.SkillOpportunityModel,
id='opportunity_id1',
skill_description='A skill description',
question_count=20,
)
skill_opportunity_model_1.update_timestamps()
skill_opportunity_model_1.put()
skill_opportunity_model_2 = self.create_model(
opportunity_models.SkillOpportunityModel,
id='opportunity_id2',
skill_description='A skill description',
question_count=20,
)
skill_opportunity_model_2.update_timestamps()
skill_opportunity_model_2.put()
all_skill_opportunity_models = list(
opportunity_models.SkillOpportunityModel.get_all())
self.assertEqual(len(all_skill_opportunity_models), 2)
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS: 2')
])
all_skill_opportunity_models = list(
opportunity_models.SkillOpportunityModel.get_all())
self.assertEqual(len(all_skill_opportunity_models), 0)
def test_generation_job_creates_new_models(self) -> None:
all_opportunity_models = list(
opportunity_models.SkillOpportunityModel.get_all())
self.assertEqual(len(all_opportunity_models), 0)
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS: 2')
])
opportunity_model_1 = (
opportunity_models.SkillOpportunityModel.get(
self.SKILL_1_ID))
# Ruling out the possibility of None for mypy type checking.
assert opportunity_model_1 is not None
self.assertEqual(opportunity_model_1.id, self.SKILL_1_ID)
self.assertEqual(
opportunity_model_1.skill_description,
self.SKILL_1_DESCRIPTION)
self.assertEqual(opportunity_model_1.question_count, 1)
opportunity_model_2 = (
opportunity_models.SkillOpportunityModel.get(
self.SKILL_2_ID))
assert opportunity_model_2 is not None
self.assertEqual(opportunity_model_2.id, self.SKILL_2_ID)
self.assertEqual(
opportunity_model_2.skill_description,
self.SKILL_2_DESCRIPTION)
self.assertEqual(opportunity_model_2.question_count, 1)
def test_updates_existing_stats_model_when_values_are_provided(
self) -> None:
user_settings_model = self.create_model(user_models.UserSettingsModel,
id=self.VALID_USER_ID_1,
email='*****@*****.**')
user_stats_model = self.create_model(
user_models.UserStatsModel,
id=self.VALID_USER_ID_1,
num_ratings=10,
average_ratings=4.5,
total_plays=22,
)
self.put_multi([user_settings_model, user_stats_model])
self.assert_job_output_is(
[job_run_result.JobRunResult(stdout='SUCCESS OLD 1')])
new_user_stats_model = (user_models.UserStatsModel.get(
self.VALID_USER_ID_1))
# Ruling out the possibility of None for mypy type checking.
assert new_user_stats_model is not None
self.assertEqual(new_user_stats_model.weekly_creator_stats_list, [{
self.formated_datetime: {
'num_ratings': 10,
'average_ratings': 4.5,
'total_plays': 22
}
}])
def test_skips_private_model(self) -> None:
exp_summary = self.create_model(
exp_models.ExpSummaryModel,
id='abcd',
deleted=False,
title='title',
category='category',
objective='objective',
language_code='lang',
community_owned=False,
status=constants.ACTIVITY_STATUS_PRIVATE
)
exp_summary.update_timestamps()
exp_summary.put()
add_docs_to_index_swap = self.swap_with_checks(
platform_search_services,
'add_documents_to_index',
lambda _, __: None,
expected_args=[([], search_services.SEARCH_INDEX_EXPLORATIONS)]
)
with add_docs_to_index_swap:
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS 1 models indexed')
])
def test_generation_job_counts_multiple_questions(self) -> None:
all_opportunity_models = list(
opportunity_models.SkillOpportunityModel.get_all())
self.assertEqual(len(all_opportunity_models), 0)
question_skill_link_model_1 = self.create_model(
question_models.QuestionSkillLinkModel,
question_id=self.QUESTION_1_ID,
skill_id=self.SKILL_2_ID,
skill_difficulty=1)
question_skill_link_model_1.update_timestamps()
datastore_services.put_multi([question_skill_link_model_1])
self.assert_job_output_is(
[job_run_result.JobRunResult(stdout='SUCCESS: 2')])
opportunity_model_1 = (opportunity_models.SkillOpportunityModel.get(
self.SKILL_1_ID))
# Ruling out the possibility of None for mypy type checking.
assert opportunity_model_1 is not None
self.assertEqual(opportunity_model_1.id, self.SKILL_1_ID)
self.assertEqual(opportunity_model_1.skill_description,
self.SKILL_1_DESCRIPTION)
self.assertEqual(opportunity_model_1.question_count, 1)
opportunity_model_2 = (opportunity_models.SkillOpportunityModel.get(
self.SKILL_2_ID))
assert opportunity_model_2 is not None
self.assertEqual(opportunity_model_2.id, self.SKILL_2_ID)
self.assertEqual(opportunity_model_2.skill_description,
self.SKILL_2_DESCRIPTION)
self.assertEqual(opportunity_model_2.question_count, 2)
def test_migrated_skill_is_not_migrated(self) -> None:
skill_model = self.create_model(
skill_models.SkillModel,
id=self.SKILL_1_ID,
description='description',
misconceptions_schema_version=(
feconf.CURRENT_MISCONCEPTIONS_SCHEMA_VERSION),
rubric_schema_version=feconf.CURRENT_RUBRIC_SCHEMA_VERSION,
rubrics=self.latest_rubrics,
language_code='cs',
skill_contents_schema_version=(
feconf.CURRENT_SKILL_CONTENTS_SCHEMA_VERSION),
skill_contents=self.latest_skill_contents,
next_misconception_id=2,
all_questions_merged=False
)
skill_model.update_timestamps()
skill_model.commit(feconf.SYSTEM_COMMITTER_ID, 'Create skill', [{
'cmd': skill_domain.CMD_CREATE_NEW
}])
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SKILL PROCESSED SUCCESS: 1')
])
unmigrated_skill_model = skill_models.SkillModel.get(self.SKILL_1_ID)
self.assertEqual(unmigrated_skill_model.version, 1)
def test_reports_failure_on_broken_model(self) -> None:
suggestion_model = self.create_model(
suggestion_models.GeneralSuggestionModel,
id='suggestion_id',
suggestion_type=feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT,
author_id=self.VALID_USER_ID_1,
change_cmd={
'cmd': exp_domain.CMD_ADD_WRITTEN_TRANSLATION,
'state_name': 'state',
'content_id': 'content_id',
'language_code': 'lang',
'content_html': 111,
'translation_html': '111 222 333',
'data_format': 'html'
},
score_category='irelevant',
status=suggestion_models.STATUS_IN_REVIEW,
target_type='exploration',
target_id=self.EXP_1_ID,
target_version_at_submission=0,
language_code=self.LANG_1
)
suggestion_model.update_timestamps()
suggestion_model.put()
self.assert_job_output_is([
job_run_result.JobRunResult(
stderr=(
'ERROR: "suggestion_id: argument cannot be of \'int\' '
'type, must be of text type": 1'
)
)
])
def test_generation_job_returns_initial_opportunity(self) -> None:
all_opportunity_models = list(
opportunity_models.ExplorationOpportunitySummaryModel.get_all())
self.assertEqual(len(all_opportunity_models), 0)
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS')
])
opportunity_model = (
opportunity_models.ExplorationOpportunitySummaryModel.get(
self.EXP_1_ID))
# Ruling out the possibility of None for mypy type checking.
assert opportunity_model is not None
self.assertEqual(opportunity_model.topic_id, self.TOPIC_1_ID)
self.assertEqual(opportunity_model.topic_name, 'topic title')
self.assertEqual(opportunity_model.story_id, self.STORY_1_ID)
self.assertEqual(opportunity_model.story_title, 'story title')
self.assertEqual(opportunity_model.chapter_title, 'node title')
self.assertEqual(opportunity_model.content_count, 1)
self.assertItemsEqual( # type: ignore[no-untyped-call]
opportunity_model.incomplete_translation_language_codes,
{l['id'] for l in constants.SUPPORTED_AUDIO_LANGUAGES} - {'cs'})
self.assertEqual(opportunity_model.translation_counts, {})
self.assertEqual(
opportunity_model.language_codes_needing_voice_artists, ['cs'])
def test_fails_when_existing_stats_has_wrong_schema_version(self) -> None:
user_settings_model = self.create_model(
user_models.UserSettingsModel,
id=self.VALID_USER_ID_1, email='*****@*****.**')
user_stats_model = self.create_model(
user_models.UserStatsModel,
id=self.VALID_USER_ID_1,
schema_version=0
)
self.put_multi([user_settings_model, user_stats_model])
with self.assertRaisesRegexp( # type: ignore[no-untyped-call]
Exception,
'Sorry, we can only process v1-v%d dashboard stats schemas at '
'present.' % feconf.CURRENT_DASHBOARD_STATS_SCHEMA_VERSION
):
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS OLD 1')
])
new_user_stats_model = (
user_models.UserStatsModel.get(self.VALID_USER_ID_1))
# Ruling out the possibility of None for mypy type checking.
assert new_user_stats_model is not None
self.assertEqual(new_user_stats_model.weekly_creator_stats_list, [])
def test_broken_skill_is_not_migrated(self) -> None:
story_model = self.create_model(
story_models.StoryModel,
id=self.STORY_1_ID,
story_contents_schema_version=4,
title='title',
language_code='cs',
notes='notes',
description='description',
story_contents=self.broken_contents,
corresponding_topic_id=self.TOPIC_1_ID,
url_fragment='urlfragment',
)
story_model.update_timestamps()
story_model.commit(feconf.SYSTEM_COMMITTER_ID, 'Create story',
[{
'cmd': story_domain.CMD_CREATE_NEW
}])
self.assert_job_output_is([
job_run_result.JobRunResult(stderr=(
'STORY PROCESSED ERROR: "(\'story_1_id\', ValidationError('
'\'Expected description to be a string, received 123\''
'))": 1'))
])
migrated_story_model = story_models.StoryModel.get(self.STORY_1_ID)
self.assertEqual(migrated_story_model.version, 1)
def test_counts_single_collection(self) -> None:
user = self.create_model(
user_models.UserSettingsModel,
id=self.USER_ID_1,
email='*****@*****.**',
roles=[feconf.ROLE_ID_COLLECTION_EDITOR]
)
user.update_timestamps()
collection = self.create_model(
collection_models.CollectionRightsModel,
id='col_1',
owner_ids=[self.USER_ID_1],
editor_ids=[self.USER_ID_1],
voice_artist_ids=[self.USER_ID_1],
community_owned=False,
status=constants.ACTIVITY_STATUS_PUBLIC,
viewable_if_private=False,
first_published_msec=0.2
)
collection.update_timestamps()
self.put_multi([user, collection])
self.assert_job_output_is([
job_run_result.JobRunResult(
stdout=(
'collection_ids: [\'col_1\'], email: [\'[email protected]\']'))
])
def run(self) -> beam.PCollection[job_run_result.JobRunResult]:
"""Returns a PCollection of 'SUCCESS' or 'FAILURE' results from
deleting ExplorationOpportunitySummaryModel.
Returns:
PCollection. A PCollection of 'SUCCESS' or 'FAILURE' results from
deleting ExplorationOpportunitySummaryModel.
"""
exp_opportunity_summary_model = (
self.pipeline
| 'Get all non-deleted opportunity models' >> ndb_io.GetModels(
opportunity_models.ExplorationOpportunitySummaryModel.get_all(
include_deleted=False)))
unused_delete_result = (exp_opportunity_summary_model
| beam.Map(lambda model: model.key)
| 'Delete all models' >> ndb_io.DeleteModels())
return (exp_opportunity_summary_model
| 'Count all new models' >> beam.combiners.Count.Globally()
| 'Only create result for new models when > 0' >>
(beam.Filter(lambda n: n > 0))
| 'Create result for new models' >>
beam.Map(lambda n: job_run_result.JobRunResult(
stdout='SUCCESS %s' % n)))
def test_job_runs_correctly_for_second_batch(self) -> None:
config_model = self.create_model(
config_models.ConfigPropertyModel,
id='batch_index_for_mailchimp',
value=1
)
config_model.update_timestamps()
config_model.commit('user_id_0', [])
expected_emails = [
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**'
]
expected_output = ','.join(expected_emails)
self.assert_job_output_is([
job_run_result.JobRunResult(stdout=expected_output)
])
def test_job_fails_correctly_with_request_error(self) -> None:
user_model = self.create_model(
user_models.UserSettingsModel,
id='user_id',
email='errored_email'
)
user_model.update_timestamps()
user_model.put()
# Half of the users have emails enabled.
preferences_model = self.create_model(
user_models.UserEmailPreferencesModel,
id='user_id',
site_updates=True
)
preferences_model.update_timestamps()
preferences_model.put()
mailchimp = self.MockMailchimpClass()
swapped_mailchimp = lambda: mailchimp
swap_mailchimp_context = self.swap(
mailchimp_population_jobs, '_get_mailchimp_class',
swapped_mailchimp)
with swap_mailchimp_context, self.swap_audience_id:
self.assert_job_output_is([
job_run_result.JobRunResult(
stdout='Server Issue')
])
def test_generation_job_fails_when_exp_id_is_not_available(self) -> None:
self.topic_model.canonical_story_references.append({
'story_id':
self.STORY_2_ID,
'story_is_published':
False
})
self.topic_model.update_timestamps()
story_model = self.create_model(story_models.StoryModel,
id=self.STORY_2_ID,
title='story 2 title',
language_code='cs',
story_contents_schema_version=1,
corresponding_topic_id=self.TOPIC_1_ID,
url_fragment='story',
story_contents={
'nodes': [{
'id':
'node',
'outline':
'outline',
'title':
'node 2 title',
'description':
'description',
'destination_node_ids':
['123'],
'acquired_skill_ids': [],
'exploration_id':
'missing_id',
'prerequisite_skill_ids': [],
'outline_is_finalized':
True
}],
'initial_node_id':
'abc',
'next_node_id':
'efg'
},
notes='note')
story_model.update_timestamps()
datastore_services.put_multi([self.topic_model, story_model])
all_opportunity_models = list(
opportunity_models.ExplorationOpportunitySummaryModel.get_all())
self.assertEqual(len(all_opportunity_models), 0)
self.assert_job_output_is([
job_run_result.JobRunResult(stderr=(
'ERROR: "Failed to regenerate opportunities for topic id: '
'topic_1_id, missing_exp_with_ids: [\'missing_id\'], '
'missing_story_with_ids: []": 1'))
])
all_opportunity_models = list(
opportunity_models.ExplorationOpportunitySummaryModel.get_all())
self.assertEqual(len(all_opportunity_models), 0)
def test_creates_stats_model_from_one_suggestion_in_set_format(
self
) -> None:
suggestion_model = self.create_model(
suggestion_models.GeneralSuggestionModel,
suggestion_type=feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT,
author_id=self.VALID_USER_ID_1,
change_cmd={
'cmd': exp_domain.CMD_ADD_WRITTEN_TRANSLATION,
'state_name': 'state',
'content_id': 'content_id',
'language_code': 'lang',
'content_html': ['111 a', '222 b', '333 c'],
'translation_html': ['111 a', '222 b', '333 c'],
'data_format': 'set_of_normalized_string'
},
score_category='irelevant',
status=suggestion_models.STATUS_IN_REVIEW,
target_type='exploration',
target_id=self.EXP_1_ID,
target_version_at_submission=0,
language_code=self.LANG_1
)
suggestion_model.update_timestamps()
suggestion_model.put()
self.assert_job_output_is([
job_run_result.JobRunResult(stdout='SUCCESS: 1')
])
translation_stats_model = (
suggestion_models.TranslationContributionStatsModel.get(
self.LANG_1, self.VALID_USER_ID_1, ''))
# Ruling out the possibility of None for mypy type checking.
assert translation_stats_model is not None
self.assertEqual(translation_stats_model.language_code, self.LANG_1)
self.assertEqual(
translation_stats_model.contributor_user_id, self.VALID_USER_ID_1)
self.assertEqual(translation_stats_model.topic_id, '')
self.assertEqual(
translation_stats_model.submitted_translations_count, 1)
self.assertEqual(
translation_stats_model.submitted_translation_word_count, 6)
self.assertEqual(translation_stats_model.accepted_translations_count, 0)
self.assertEqual(
translation_stats_model
.accepted_translations_without_reviewer_edits_count,
0
)
self.assertEqual(
translation_stats_model.accepted_translation_word_count, 0)
self.assertEqual(translation_stats_model.rejected_translations_count, 0)
self.assertEqual(
translation_stats_model.rejected_translation_word_count, 0)
self.assertItemsEqual( # type: ignore[no-untyped-call]
translation_stats_model.contribution_dates,
[datetime.date.today()]
)
def run(self) -> beam.PCollection[job_run_result.JobRunResult]:
"""Generates the translation contributins stats.
Returns:
PCollection. A PCollection of 'SUCCESS x' results, where x is
the number of generated stats..
"""
suggestions_grouped_by_target = (
self.pipeline
| 'Get all non-deleted suggestion models' >> ndb_io.GetModels(
suggestion_models.GeneralSuggestionModel.get_all(
include_deleted=False))
# We need to window the models so that CoGroupByKey below
# works properly.
| 'Filter translate suggestions' >> beam.Filter(lambda m: (
m.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT))
| 'Transform to suggestion domain object' >> beam.Map(
suggestion_services.get_suggestion_from_model)
| 'Group by target' >> beam.GroupBy(lambda m: m.target_id))
exp_opportunities = (
self.pipeline
| 'Get all non-deleted opportunity models' >> ndb_io.GetModels(
opportunity_models.ExplorationOpportunitySummaryModel.get_all(
include_deleted=False))
# We need to window the models so that CoGroupByKey below
# works properly.
| 'Transform to opportunity domain object' >>
beam.Map(opportunity_services.
get_exploration_opportunity_summary_from_model)
| 'Group by ID' >> beam.GroupBy(lambda m: m.id))
new_user_stats_models = (
{
'suggestion': suggestions_grouped_by_target,
'opportunity': exp_opportunities
}
| 'Merge models' >> beam.CoGroupByKey()
| 'Get rid of key' >> beam.Values() # pylint: disable=no-value-for-parameter
| 'Generate stats' >> beam.ParDo(lambda x: self._generate_stats(
x['suggestion'][0] if len(x['suggestion']) else [],
list(x['opportunity'][0])[0]
if len(x['opportunity']) else None))
| 'Combine the stats' >> beam.CombinePerKey(CombineStats())
| 'Generate models from stats' >> beam.MapTuple(
self._generate_translation_contribution_model))
unused_put_result = (
new_user_stats_models
| 'Put models into the datastore' >> ndb_io.PutModels())
return (new_user_stats_models
| 'Count all new models' >>
(beam.combiners.Count.Globally().without_defaults())
| 'Only create result for new models when > 0' >>
(beam.Filter(lambda x: x > 0))
| 'Create result for new models' >>
beam.Map(lambda x: job_run_result.JobRunResult(
stdout='SUCCESS %s' % x)))
def test_job_deletes_sent_email_model_with_user_as_recipient(self) -> None:
self.sent_email_model_with_recipient.update_timestamps()
self.sent_email_model_with_recipient.put()
self.assert_job_output_is(
[job_run_result.JobRunResult(stdout='SENT EMAILS SUCCESS: 1')])
self.assertIsNone(
email_models.SentEmailModel.get('sent_email_id', strict=False))
