def run(self) -> beam.PCollection[job_run_result.JobRunResult]: """Returns a PCollection of 'SUCCESS' or 'FAILURE' results from generating ExplorationOpportunitySummaryModel. Returns: PCollection. A PCollection of 'SUCCESS' or 'FAILURE' results from generating ExplorationOpportunitySummaryModel. """ topics = (self.pipeline | 'Get all non-deleted topic models' >> (ndb_io.GetModels( topic_models.TopicModel.get_all(include_deleted=False))) | 'Get topic from model' >> beam.Map( topic_fetchers.get_topic_from_model)) story_ids_to_story = ( self.pipeline | 'Get all non-deleted story models' >> ndb_io.GetModels( story_models.StoryModel.get_all(include_deleted=False)) | 'Get story from model' >> beam.Map( story_fetchers.get_story_from_model) | 'Combine stories and ids' >> beam.Map(lambda story: (story.id, story))) exp_ids_to_exp = ( self.pipeline | 'Get all non-deleted exp models' >> ndb_io.GetModels( exp_models.ExplorationModel.get_all(include_deleted=False)) | 'Get exploration from model' >> beam.Map( exp_fetchers.get_exploration_from_model) | 'Combine exploration and ids' >> beam.Map(lambda exp: (exp.id, exp))) stories_dict = beam.pvalue.AsDict(story_ids_to_story) exps_dict = beam.pvalue.AsDict(exp_ids_to_exp) opportunities_results = ( topics | beam.Map(self._generate_opportunities_related_to_topic, stories_dict=stories_dict, exps_dict=exps_dict)) unused_put_result = ( opportunities_results | 'Filter the results with SUCCESS status' >> beam.Filter(lambda result: result.is_ok()) | 'Fetch the models to be put' >> beam.FlatMap(lambda result: result.unwrap()) | 'Add ID as a key' >> beam.WithKeys(lambda model: model.id) # pylint: disable=no-value-for-parameter | 'Allow only one item per key' >> (beam.combiners.Sample.FixedSizePerKey(1)) | 'Remove the IDs' >> beam.Values() # pylint: disable=no-value-for-parameter | 'Flatten the list of lists of models' >> beam.FlatMap(lambda x: x) | 'Put models into the datastore' >> ndb_io.PutModels()) return (opportunities_results | 'Count the output' >> (job_result_transforms.ResultsToJobRunResults()))
def test_ok_results_with_prefix_correctly_outputs(self) -> None: transform_result = ( self.pipeline | beam.Create([result.Ok('ok'), result.Ok('ok')]) | job_result_transforms.ResultsToJobRunResults('PREFIX') ) self.assert_pcoll_equal( transform_result, [job_run_result.JobRunResult.as_stdout('PREFIX SUCCESS: 2')] )
def test_err_results_with_prefix_correctly_outputs(self) -> None: transform_result = ( self.pipeline | beam.Create([ result.Err('err 1'), result.Err('err 2'), result.Err('err 2') ]) | job_result_transforms.ResultsToJobRunResults('PRE') ) self.assert_pcoll_equal( transform_result, [ job_run_result.JobRunResult.as_stderr('PRE ERROR: "err 1": 1'), job_run_result.JobRunResult.as_stderr('PRE ERROR: "err 2": 2') ] )
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: """Returns a PCollection of 'SUCCESS' or 'FAILURE' results from the Elastic Search. Returns: PCollection. A PCollection of 'SUCCESS' or 'FAILURE' results from the Elastic Search. """ return ( self.pipeline | 'Get all non-deleted models' >> (ndb_io.GetModels( exp_models.ExpSummaryModel.get_all(include_deleted=False))) | 'Split models into batches' >> beam.transforms.util.BatchElements( max_batch_size=self.MAX_BATCH_SIZE) | 'Index batches of models' >> beam.ParDo( IndexExplorationSummaries()) | 'Count the output' >> (job_result_transforms.ResultsToJobRunResults()))
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: """Returns a PCollection of results from the skill migration. Returns: PCollection. A PCollection of results from the skill migration. """ unmigrated_skill_models = ( self.pipeline | 'Get all non-deleted skill models' >> (ndb_io.GetModels(skill_models.SkillModel.get_all())) # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Add skill model ID' >> beam.WithKeys( # pylint: disable=no-value-for-parameter lambda skill_model: skill_model.id)) skill_summary_models = ( self.pipeline | 'Get all non-deleted skill summary models' >> (ndb_io.GetModels(skill_models.SkillSummaryModel.get_all())) # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Add skill summary ID' >> beam.WithKeys( # pylint: disable=no-value-for-parameter lambda skill_summary_model: skill_summary_model.id)) migrated_skill_results = (unmigrated_skill_models | 'Transform and migrate model' >> beam.MapTuple(self._migrate_skill)) migrated_skills = ( migrated_skill_results | 'Filter oks' >> beam.Filter(lambda result_item: result_item.is_ok()) | 'Unwrap ok' >> beam.Map(lambda result_item: result_item.unwrap())) migrated_skill_job_run_results = ( migrated_skill_results | 'Generate results for migration' >> (job_result_transforms.ResultsToJobRunResults('SKILL PROCESSED'))) skill_changes = (unmigrated_skill_models | 'Generate skill changes' >> beam.FlatMapTuple( self._generate_skill_changes)) skill_objects_list = ( { 'skill_model': unmigrated_skill_models, 'skill_summary_model': skill_summary_models, 'skill': migrated_skills, 'skill_changes': skill_changes } | 'Merge objects' >> beam.CoGroupByKey() | 'Get rid of ID' >> beam.Values() # pylint: disable=no-value-for-parameter | 'Remove unmigrated skills' >> beam.Filter( lambda x: len(x['skill_changes']) > 0 and len(x['skill']) > 0) | 'Reorganize the skill objects' >> beam.Map( lambda objects: { 'skill_model': objects['skill_model'][0], 'skill_summary_model': objects['skill_summary_model'][0], 'skill': objects['skill'][0], 'skill_changes': objects['skill_changes'] })) skill_objects_list_job_run_results = ( skill_objects_list | 'Transform skill objects into job run results' >> (job_result_transforms.CountObjectsToJobRunResult('SKILL MIGRATED') )) cache_deletion_job_run_results = ( skill_objects_list | 'Delete skill from cache' >> beam.Map(lambda skill_object: self._delete_skill_from_cache( skill_object['skill'])) | 'Generate results for cache deletion' >> (job_result_transforms.ResultsToJobRunResults('CACHE DELETION'))) skill_models_to_put = ( skill_objects_list | 'Generate skill models to put' >> beam.FlatMap(lambda skill_objects: self._update_skill( skill_objects['skill_model'], skill_objects['skill'], skill_objects['skill_changes'], ))) skill_summary_models_to_put = ( skill_objects_list | 'Generate skill summary models to put' >> beam.Map(lambda skill_objects: self._update_skill_summary( skill_objects['skill'], skill_objects['skill_summary_model']))) unused_put_results = ( (skill_models_to_put, skill_summary_models_to_put) | 'Merge models' >> beam.Flatten() | 'Put models into the datastore' >> ndb_io.PutModels()) return ( (cache_deletion_job_run_results, migrated_skill_job_run_results, skill_objects_list_job_run_results) | beam.Flatten())
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: """Generates the translation contributins stats. Returns: PCollection. A PCollection of 'SUCCESS x' results, where x is the number of generated stats.. """ suggestions_grouped_by_target = ( self.pipeline | 'Get all non-deleted suggestion models' >> ndb_io.GetModels( suggestion_models.GeneralSuggestionModel.get_all( include_deleted=False)) # We need to window the models so that CoGroupByKey below # works properly. | 'Filter translate suggestions' >> beam.Filter(lambda m: ( m.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT)) | 'Transform to suggestion domain object' >> beam.Map( suggestion_services.get_suggestion_from_model) | 'Group by target' >> beam.GroupBy(lambda m: m.target_id)) exp_opportunities = ( self.pipeline | 'Get all non-deleted opportunity models' >> ndb_io.GetModels( opportunity_models.ExplorationOpportunitySummaryModel.get_all( include_deleted=False)) # We need to window the models so that CoGroupByKey below # works properly. | 'Transform to opportunity domain object' >> beam.Map(opportunity_services. get_exploration_opportunity_summary_from_model) | 'Group by ID' >> beam.GroupBy(lambda m: m.id)) user_stats_results = ( { 'suggestion': suggestions_grouped_by_target, 'opportunity': exp_opportunities } | 'Merge models' >> beam.CoGroupByKey() | 'Get rid of key' >> beam.Values() # pylint: disable=no-value-for-parameter | 'Generate stats' >> beam.ParDo(lambda x: self._generate_stats( x['suggestion'][0] if len(x['suggestion']) else [], list(x['opportunity'][0])[0] if len(x['opportunity']) else None))) user_stats_models = ( user_stats_results | 'Filter ok results' >> beam.Filter(lambda key_and_result: key_and_result[1].is_ok()) | 'Unpack result' >> beam.MapTuple(lambda key, result: (key, result.unwrap())) | 'Combine the stats' >> beam.CombinePerKey(CombineStats()) | 'Generate models from stats' >> beam.MapTuple( self._generate_translation_contribution_model)) user_stats_error_job_run_results = ( user_stats_results | 'Filter err results' >> beam.Filter(lambda key_and_result: key_and_result[1].is_err()) # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Remove keys' >> beam.Values() # pylint: disable=no-value-for-parameter | 'Transform result to job run result' >> (job_result_transforms.ResultsToJobRunResults())) unused_put_result = ( user_stats_models | 'Put models into the datastore' >> ndb_io.PutModels()) user_stats_models_job_run_results = ( user_stats_models | 'Create job run result' >> (job_result_transforms.CountObjectsToJobRunResult())) return ((user_stats_error_job_run_results, user_stats_models_job_run_results) | 'Merge job run results' >> beam.Flatten())
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: """Returns a PCollection of 'SUCCESS' or 'FAILURE' results from generating SkillOpportunityModel. Returns: PCollection. A PCollection of 'SUCCESS' or 'FAILURE' results from generating SkillOpportunityModel. """ question_skill_link_models = ( self.pipeline | 'Get all non-deleted QuestionSkillLinkModels' >> (ndb_io.GetModels( question_models.QuestionSkillLinkModel.get_all( include_deleted=False))) | 'Group QuestionSkillLinkModels by skill ID' >> beam.GroupBy(lambda n: n.skill_id)) skills = ( self.pipeline | 'Get all non-deleted SkillModels' >> (ndb_io.GetModels( skill_models.SkillModel.get_all(include_deleted=False))) | 'Get skill object from model' >> beam.Map( skill_fetchers.get_skill_from_model) | 'Group skill objects by skill ID' >> beam.GroupBy(lambda m: m.id)) skills_with_question_counts = ( { 'skill': skills, 'question_skill_links': question_skill_link_models } | 'Merge by skill ID' >> beam.CoGroupByKey() # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Remove skill IDs' >> beam.Values() # pylint: disable=no-value-for-parameter # We are using itertools.chain.from_iterable to flatten # question_skill_links from a 2D list into a 1D list. | 'Flatten skill and question_skill_links' >> beam.Map( lambda object: { 'skill': list(object['skill'][0])[0], 'question_skill_links': list( itertools.chain.from_iterable(object[ 'question_skill_links'])) })) opportunities_results = ( skills_with_question_counts | beam.Map(lambda object: self._create_skill_opportunity_model( object['skill'], object['question_skill_links']))) unused_put_result = ( opportunities_results | 'Filter the results with OK status' >> beam.Filter(lambda result: result.is_ok()) | 'Fetch the models to be put' >> beam.Map(lambda result: result.unwrap()) | 'Put models into the datastore' >> ndb_io.PutModels()) return (opportunities_results | 'Transform Results to JobRunResults' >> (job_result_transforms.ResultsToJobRunResults()))
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: """Returns a PCollection of results from the story migration. Returns: PCollection. A PCollection of results from the story migration. """ unmigrated_story_models = ( self.pipeline | 'Get all non-deleted story models' >> ( ndb_io.GetModels(story_models.StoryModel.get_all())) # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Add story keys' >> beam.WithKeys( # pylint: disable=no-value-for-parameter lambda story_model: story_model.id) ) story_summary_models = ( self.pipeline | 'Get all non-deleted story summary models' >> ( ndb_io.GetModels(story_models.StorySummaryModel.get_all())) # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Add story summary keys' >> beam.WithKeys( # pylint: disable=no-value-for-parameter lambda story_summary_model: story_summary_model.id) ) topics = ( self.pipeline | 'Get all non-deleted topic models' >> ( ndb_io.GetModels(topic_models.TopicModel.get_all())) | 'Transform model into domain object' >> beam.Map( topic_fetchers.get_topic_from_model) # Pylint disable is needed because pylint is not able to correctly # detect that the value is passed through the pipe. | 'Add topic keys' >> beam.WithKeys( # pylint: disable=no-value-for-parameter lambda topic: topic.id) ) topic_id_to_topic = beam.pvalue.AsDict(topics) migrated_story_results = ( unmigrated_story_models | 'Transform and migrate model' >> beam.MapTuple( self._migrate_story, topic_id_to_topic=topic_id_to_topic) ) migrated_stories = ( migrated_story_results | 'Filter oks' >> beam.Filter( lambda result_item: result_item.is_ok()) | 'Unwrap ok' >> beam.Map( lambda result_item: result_item.unwrap()) ) migrated_story_job_run_results = ( migrated_story_results | 'Generate results for migration' >> ( job_result_transforms.ResultsToJobRunResults('STORY PROCESSED')) ) story_changes = ( unmigrated_story_models | 'Generate story changes' >> beam.FlatMapTuple( self._generate_story_changes) ) story_objects_list = ( { 'story_model': unmigrated_story_models, 'story_summary_model': story_summary_models, 'story': migrated_stories, 'story_change': story_changes } | 'Merge objects' >> beam.CoGroupByKey() | 'Get rid of ID' >> beam.Values() # pylint: disable=no-value-for-parameter | 'Remove unmigrated stories' >> beam.Filter( lambda x: len(x['story_change']) > 0 and len(x['story']) > 0) | 'Reorganize the story objects' >> beam.Map(lambda objects: { 'story_model': objects['story_model'][0], 'story_summary_model': objects['story_summary_model'][0], 'story': objects['story'][0], 'story_change': objects['story_change'][0] }) ) story_objects_list_job_run_results = ( story_objects_list | 'Transform story objects into job run results' >> ( job_result_transforms.CountObjectsToJobRunResult( 'STORY MIGRATED')) ) cache_deletion_job_run_results = ( story_objects_list | 'Delete story from cache' >> beam.Map( lambda story_objects: self._delete_story_from_cache( story_objects['story'])) | 'Generate results for cache deletion' >> ( job_result_transforms.ResultsToJobRunResults('CACHE DELETION')) ) story_models_to_put = ( story_objects_list | 'Generate story models to put' >> beam.FlatMap( lambda story_objects: self._update_story( story_objects['story_model'], story_objects['story'], story_objects['story_change'], )) ) story_summary_models_to_put = ( story_objects_list | 'Generate story summary models to put' >> beam.Map( lambda story_objects: self._update_story_summary( story_objects['story'], story_objects['story_summary_model'] )) ) unused_put_results = ( (story_models_to_put, story_summary_models_to_put) | 'Merge models' >> beam.Flatten() | 'Put models into the datastore' >> ndb_io.PutModels() ) return ( ( cache_deletion_job_run_results, migrated_story_job_run_results, story_objects_list_job_run_results ) | beam.Flatten() )