Beispiel #1
0
    def test_generation_job_counts_multiple_questions(self) -> None:
        all_opportunity_models = list(
            opportunity_models.SkillOpportunityModel.get_all())
        self.assertEqual(len(all_opportunity_models), 0)

        question_skill_link_model_1 = self.create_model(
            question_models.QuestionSkillLinkModel,
            question_id=self.QUESTION_1_ID,
            skill_id=self.SKILL_2_ID,
            skill_difficulty=1)
        question_skill_link_model_1.update_timestamps()
        datastore_services.put_multi([question_skill_link_model_1])

        self.assert_job_output_is(
            [job_run_result.JobRunResult(stdout='SUCCESS: 2')])

        opportunity_model_1 = (opportunity_models.SkillOpportunityModel.get(
            self.SKILL_1_ID))
        # Ruling out the possibility of None for mypy type checking.
        assert opportunity_model_1 is not None
        self.assertEqual(opportunity_model_1.id, self.SKILL_1_ID)
        self.assertEqual(opportunity_model_1.skill_description,
                         self.SKILL_1_DESCRIPTION)
        self.assertEqual(opportunity_model_1.question_count, 1)

        opportunity_model_2 = (opportunity_models.SkillOpportunityModel.get(
            self.SKILL_2_ID))
        assert opportunity_model_2 is not None
        self.assertEqual(opportunity_model_2.id, self.SKILL_2_ID)
        self.assertEqual(opportunity_model_2.skill_description,
                         self.SKILL_2_DESCRIPTION)
        self.assertEqual(opportunity_model_2.question_count, 2)
Beispiel #2
0
def get_beam_job_runs(
    refresh: bool = True
) -> List[beam_job_domain.BeamJobRun]:
    """Returns all of the Apache Beam job runs recorded in the datastore.

    Args:
        refresh: bool. Whether to refresh the jobs' state before returning them.

    Returns:
        list(BeamJobRun). A list of every job run recorded in the datastore.
    """
    beam_job_run_models = list(beam_job_models.BeamJobRunModel.query())
    beam_job_runs = [
        get_beam_job_run_from_model(m) for m in beam_job_run_models
    ]

    if refresh:
        updated_beam_job_run_models = []

        for i, beam_job_run_model in enumerate(beam_job_run_models):
            if beam_job_runs[i].in_terminal_state:
                continue
            jobs_manager.refresh_state_of_beam_job_run_model(beam_job_run_model)
            beam_job_run_model.update_timestamps(update_last_updated_time=False)
            updated_beam_job_run_models.append(beam_job_run_model)
            beam_job_runs[i] = get_beam_job_run_from_model(beam_job_run_model)

        if updated_beam_job_run_models:
            datastore_services.put_multi(updated_beam_job_run_models)

    return beam_job_runs
Beispiel #3
0
    def test_generation_job_fails_when_exp_id_is_not_available(self) -> None:
        self.topic_model.canonical_story_references.append({
            'story_id':
            self.STORY_2_ID,
            'story_is_published':
            False
        })
        self.topic_model.update_timestamps()

        story_model = self.create_model(story_models.StoryModel,
                                        id=self.STORY_2_ID,
                                        title='story 2 title',
                                        language_code='cs',
                                        story_contents_schema_version=1,
                                        corresponding_topic_id=self.TOPIC_1_ID,
                                        url_fragment='story',
                                        story_contents={
                                            'nodes': [{
                                                'id':
                                                'node',
                                                'outline':
                                                'outline',
                                                'title':
                                                'node 2 title',
                                                'description':
                                                'description',
                                                'destination_node_ids':
                                                ['123'],
                                                'acquired_skill_ids': [],
                                                'exploration_id':
                                                'missing_id',
                                                'prerequisite_skill_ids': [],
                                                'outline_is_finalized':
                                                True
                                            }],
                                            'initial_node_id':
                                            'abc',
                                            'next_node_id':
                                            'efg'
                                        },
                                        notes='note')
        story_model.update_timestamps()

        datastore_services.put_multi([self.topic_model, story_model])

        all_opportunity_models = list(
            opportunity_models.ExplorationOpportunitySummaryModel.get_all())
        self.assertEqual(len(all_opportunity_models), 0)

        self.assert_job_output_is([
            job_run_result.JobRunResult(stderr=(
                'ERROR: "Failed to regenerate opportunities for topic id: '
                'topic_1_id, missing_exp_with_ids: [\'missing_id\'], '
                'missing_story_with_ids: []": 1'))
        ])

        all_opportunity_models = list(
            opportunity_models.ExplorationOpportunitySummaryModel.get_all())
        self.assertEqual(len(all_opportunity_models), 0)
    def setUp(self):
        super().setUp()
        story_summary_model = self.create_model(
            story_models.StorySummaryModel,
            id=self.STORY_1_ID,
            title='title',
            url_fragment='urlfragment',
            language_code='cs',
            description='description',
            node_titles=['title1', 'title2'],
            story_model_last_updated=datetime.datetime.utcnow(),
            story_model_created_on=datetime.datetime.utcnow(),
            version=1)
        topic_model = self.create_model(
            topic_models.TopicModel,
            id=self.TOPIC_1_ID,
            name='topic title',
            canonical_name='topic title',
            story_reference_schema_version=1,
            subtopic_schema_version=1,
            next_subtopic_id=1,
            language_code='cs',
            url_fragment='topic',
            canonical_story_references=[{
                'story_id': self.STORY_1_ID,
                'story_is_published': False
            }],
            page_title_fragment_for_web='fragm',
        )
        datastore_services.update_timestamps_multi(
            [topic_model, story_summary_model])
        datastore_services.put_multi([topic_model, story_summary_model])
        self.latest_contents = {
            'nodes': [{
                'id': 'node_1111',
                'title': 'title',
                'description': 'description',
                'thumbnail_filename': 'thumbnail_filename.svg',
                'thumbnail_bg_color': '#F8BF74',
                'thumbnail_size_in_bytes': None,
                'destination_node_ids': [],
                'acquired_skill_ids': [],
                'prerequisite_skill_ids': [],
                'outline': 'outline',
                'outline_is_finalized': True,
                'exploration_id': 'exp_id'
            }],
            'initial_node_id':
            'node_1111',
            'next_node_id':
            'node_2222'
        }
        self.broken_contents = copy.deepcopy(self.latest_contents)
        self.broken_contents['nodes'][0]['description'] = 123

        self.unmigrated_contents = copy.deepcopy(self.latest_contents)
        self.unmigrated_contents['nodes'][0]['thumbnail_size_in_bytes'] = 123
Beispiel #5
0
    def put_multi(self, model_list: Sequence[base_models.BaseModel]) -> None:
        """Puts the input models into the datastore.

        Args:
            model_list: list(Model). The NDB models to put into the datastore.
        """
        datastore_services.update_timestamps_multi(
            model_list, update_last_updated_time=False)
        datastore_services.put_multi(model_list)
Beispiel #6
0
def mark_outdated_models_as_deleted() -> None:
    """Mark models in MODEL_CLASSES_TO_MARK_AS_DELETED, as deleted if they were
    last updated more than four weeks ago.
    """
    date_before_which_to_mark_as_deleted = (
        datetime.datetime.utcnow() - feconf.PERIOD_TO_MARK_MODELS_AS_DELETED)
    models_to_mark_as_deleted: List[base_models.BaseModel] = []
    for model_class in MODEL_CLASSES_TO_MARK_AS_DELETED:
        models_to_mark_as_deleted.extend(
            model_class.query(
                model_class.last_updated < date_before_which_to_mark_as_deleted
            ).fetch())
    for model_to_mark_as_deleted in models_to_mark_as_deleted:
        model_to_mark_as_deleted.deleted = True
    datastore_services.update_timestamps_multi(models_to_mark_as_deleted)
    datastore_services.put_multi(models_to_mark_as_deleted)
Beispiel #7
0
    def delete_multi(  # type: ignore[override]
            cls,
            entity_ids: List[str],
            committer_id: str,
            commit_message: str,
            force_deletion: bool = False) -> None:
        """Deletes the given cls instances with the given entity_ids.

        Note that this extends the superclass method.

        Args:
            entity_ids: list(str). Ids of entities to delete.
            committer_id: str. The user_id of the user who committed the change.
            commit_message: str. The commit description message.
            force_deletion: bool. If True these models are deleted completely
                from storage, otherwise there are only marked as deleted.
                Default is False.
        """
        super(ExplorationModel,
              cls).delete_multi(entity_ids,
                                committer_id,
                                commit_message,
                                force_deletion=force_deletion)

        if not force_deletion:
            commit_log_models = []
            exp_rights_models = ExplorationRightsModel.get_multi(
                entity_ids, include_deleted=True)
            versioned_models = cls.get_multi(entity_ids, include_deleted=True)

            versioned_and_exp_rights_models = zip(versioned_models,
                                                  exp_rights_models)
            for model, rights_model in versioned_and_exp_rights_models:
                # Ruling out the possibility of None for mypy type checking.
                assert model is not None
                assert rights_model is not None
                exploration_commit_log = ExplorationCommitLogEntryModel.create(
                    model.id, model.version, committer_id,
                    feconf.COMMIT_TYPE_DELETE, commit_message,
                    [{
                        'cmd': cls.CMD_DELETE_COMMIT
                    }], rights_model.status, rights_model.community_owned)
                exploration_commit_log.exploration_id = model.id
                commit_log_models.append(exploration_commit_log)
            ExplorationCommitLogEntryModel.update_timestamps_multi(
                commit_log_models)
            datastore_services.put_multi(commit_log_models)
    def test_generation_job_fails_when_story_id_is_not_available(self) -> None:
        self.topic_model.canonical_story_references.append({
            'story_id': 'missing_id',
            'story_is_published': False
        })
        self.topic_model.update_timestamps()
        datastore_services.put_multi([self.topic_model])

        all_opportunity_models = list(
            opportunity_models.ExplorationOpportunitySummaryModel.get_all())
        self.assertEqual(len(all_opportunity_models), 0)

        self.assert_job_output_is([
            job_run_result.JobRunResult(stderr=(
                'FAILURE: Failed to regenerate opportunities for topic id: '
                'topic_1_id, missing_exp_with_ids: [], '
                'missing_story_with_ids: [\'missing_id\']'
            ))
        ])

        all_opportunity_models = list(
            opportunity_models.ExplorationOpportunitySummaryModel.get_all())
        self.assertEqual(len(all_opportunity_models), 0)
Beispiel #9
0
    def test_job_finds_math_explorations_with_rules(self) -> None:
        exp_model_1 = self.create_model(
            exp_models.ExplorationModel,
            id=self.EXP_1_ID,
            title='exploration 1 title',
            category='category',
            objective='objective',
            language_code='cs',
            init_state_name='state',
            states_schema_version=48,
            states={
                'init_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict(),
                'alg_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict(),
                'eq_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict(),
                'end_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict(),
            })
        exp_model_1.states['alg_state']['interaction']['id'] = (
            'AlgebraicExpressionInput')
        exp_model_1.states['alg_state']['interaction']['answer_groups'] = [{
            'rule_specs': [{
                'inputs': {
                    'x': 'a + b'
                },
                'rule_type': 'IsEquivalentTo'
            }, {
                'inputs': {
                    'x': 'a - b'
                },
                'rule_type': 'ContainsSomeOf'
            }]
        }]
        exp_model_1.states['eq_state']['interaction']['id'] = (
            'MathEquationInput')
        exp_model_1.states['eq_state']['interaction']['answer_groups'] = [{
            'rule_specs': [{
                'inputs': {
                    'x': 'x = y',
                    'y': 'both'
                },
                'rule_type': 'MatchesExactlyWith'
            }]
        }]
        exp_model_1.update_timestamps()

        exp_model_2 = self.create_model(
            exp_models.ExplorationModel,
            id=self.EXP_2_ID,
            title='exploration 2 title',
            category='category',
            objective='objective',
            language_code='cs',
            init_state_name='state',
            states_schema_version=48,
            states={
                'init_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict(),
                'num_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict(),
                'end_state':
                state_domain.State.
                create_default_state(  # type: ignore[no-untyped-call]
                    'state',
                    is_initial_state=True).to_dict()
            })
        exp_model_2.states['num_state']['interaction']['id'] = (
            'NumericExpressionInput')
        exp_model_2.states['num_state']['interaction']['answer_groups'] = [{
            'rule_specs': [{
                'inputs': {
                    'x': '1.2 + 3'
                },
                'rule_type': 'MatchesExactlyWith'
            }, {
                'inputs': {
                    'x': '1 - 2'
                },
                'rule_type': 'OmitsSomeOf'
            }]
        }]
        exp_model_2.update_timestamps()

        datastore_services.put_multi([exp_model_1, exp_model_2])

        self.assert_job_output_is([
            job_run_result.JobRunResult(
                stdout=('(\'exp_1_id\', \'alg_state\', '
                        '[\'IsEquivalentTo\', \'ContainsSomeOf\'])')),
            job_run_result.JobRunResult(stdout=(
                '(\'exp_1_id\', \'eq_state\', [\'MatchesExactlyWith\'])')),
            job_run_result.JobRunResult(
                stdout=('(\'exp_2_id\', \'num_state\', '
                        '[\'MatchesExactlyWith\', \'OmitsSomeOf\'])'))
        ])
    def test_generation_job_returns_multiple_opportunities_for_multiple_topics(
        self
    ) -> None:
        topic_model = self.create_model(
            topic_models.TopicModel,
            id=self.TOPIC_2_ID,
            name='topic 2 title',
            canonical_name='topic 2 title',
            story_reference_schema_version=1,
            subtopic_schema_version=1,
            next_subtopic_id=1,
            language_code='cs',
            url_fragment='topic',
            canonical_story_references=[{
                'story_id': self.STORY_2_ID,
                'story_is_published': False
            }])
        topic_model.update_timestamps()
        topic_rights_model = self.create_model(
            topic_models.TopicRightsModel, id=self.TOPIC_2_ID)
        topic_rights_model.update_timestamps()

        story_model = self.create_model(
            story_models.StoryModel,
            id=self.STORY_2_ID,
            title='story 2 title',
            language_code='cs',
            story_contents_schema_version=1,
            corresponding_topic_id=self.TOPIC_1_ID,
            url_fragment='story',
            story_contents={
                'nodes': [{
                    'id': 'node',
                    'outline': 'outline',
                    'title': 'node 2 title',
                    'description': 'description',
                    'destination_node_ids': ['123'],
                    'acquired_skill_ids': [],
                    'exploration_id': self.EXP_2_ID,
                    'prerequisite_skill_ids': [],
                    'outline_is_finalized': True
                }],
                'initial_node_id': 'abc',
                'next_node_id': 'efg'
            },
            notes='note')
        story_model.update_timestamps()

        exp_model = self.create_model(
            exp_models.ExplorationModel,
            id=self.EXP_2_ID,
            title='exploration 2 title',
            category='category',
            objective='objective',
            language_code='en',
            init_state_name='state1',
            states_schema_version=48,
            states={
                'state1': state_domain.State.create_default_state( # type: ignore[no-untyped-call]
                    'state1', is_initial_state=True
                ).to_dict(),
                'state2': state_domain.State.create_default_state( # type: ignore[no-untyped-call]
                    'state2'
                ).to_dict()
            })
        exp_model.update_timestamps()
        datastore_services.put_multi([
            exp_model, story_model, topic_model, topic_rights_model
        ])

        all_opportunity_models = list(
            opportunity_models.ExplorationOpportunitySummaryModel.get_all())
        self.assertEqual(len(all_opportunity_models), 0)

        self.assert_job_output_is([
            job_run_result.JobRunResult(stdout='SUCCESS'),
            job_run_result.JobRunResult(stdout='SUCCESS')
        ])

        all_opportunity_models = list(
            opportunity_models.ExplorationOpportunitySummaryModel.get_all())
        self.assertEqual(len(all_opportunity_models), 2)

        opportunity_model = (
            opportunity_models.ExplorationOpportunitySummaryModel.get(
                self.EXP_2_ID))
        # Ruling out the possibility of None for mypy type checking.
        assert opportunity_model is not None
        self.assertEqual(opportunity_model.topic_id, self.TOPIC_2_ID)
        self.assertEqual(opportunity_model.topic_name, 'topic 2 title')
        self.assertEqual(opportunity_model.story_id, self.STORY_2_ID)
        self.assertEqual(opportunity_model.story_title, 'story 2 title')
        self.assertEqual(opportunity_model.chapter_title, 'node 2 title')
        self.assertEqual(opportunity_model.content_count, 2)
        self.assertItemsEqual( # type: ignore[no-untyped-call]
            opportunity_model.incomplete_translation_language_codes,
            {l['id'] for l in constants.SUPPORTED_AUDIO_LANGUAGES} - {'en'})
        self.assertEqual(opportunity_model.translation_counts, {})
        self.assertEqual(
            opportunity_model.language_codes_needing_voice_artists, ['en'])
    def setUp(self) -> None:
        super().setUp()
        self.topic_model = self.create_model(
            topic_models.TopicModel,
            id=self.TOPIC_1_ID,
            name='topic title',
            canonical_name='topic title',
            story_reference_schema_version=1,
            subtopic_schema_version=1,
            next_subtopic_id=1,
            language_code='cs',
            url_fragment='topic',
            canonical_story_references=[{
                'story_id': self.STORY_1_ID,
                'story_is_published': False
            }])
        self.topic_model.update_timestamps()
        topic_rights_model = self.create_model(
            topic_models.TopicRightsModel, id=self.TOPIC_1_ID)
        topic_rights_model.update_timestamps()

        story_model = self.create_model(
            story_models.StoryModel,
            id=self.STORY_1_ID,
            title='story title',
            language_code='cs',
            story_contents_schema_version=1,
            corresponding_topic_id=self.TOPIC_1_ID,
            url_fragment='story',
            story_contents={
                'nodes': [{
                    'id': 'node',
                    'outline': 'outline',
                    'title': 'node title',
                    'description': 'description',
                    'destination_node_ids': ['123'],
                    'acquired_skill_ids': [],
                    'exploration_id': self.EXP_1_ID,
                    'prerequisite_skill_ids': [],
                    'outline_is_finalized': True
                }],
                'initial_node_id': 'abc',
                'next_node_id': 'efg'
            },
            notes='note')
        story_model.update_timestamps()

        exp_model = self.create_model(
            exp_models.ExplorationModel,
            id=self.EXP_1_ID,
            title='exploration title',
            category='category',
            objective='objective',
            language_code='cs',
            init_state_name='state',
            states_schema_version=48,
            states={
                'state': state_domain.State.create_default_state( # type: ignore[no-untyped-call]
                    'state', is_initial_state=True
                ).to_dict()
            })
        exp_model.update_timestamps()
        datastore_services.put_multi([
            exp_model, story_model, self.topic_model, topic_rights_model
        ])
Beispiel #12
0
    def setUp(self) -> None:
        super().setUp()

        question_skill_link_model_1 = self.create_model(
            question_models.QuestionSkillLinkModel,
            question_id=self.QUESTION_1_ID,
            skill_id=self.SKILL_1_ID,
            skill_difficulty=1
        )

        question_skill_link_model_2 = self.create_model(
            question_models.QuestionSkillLinkModel,
            question_id=self.QUESTION_2_ID,
            skill_id=self.SKILL_2_ID,
            skill_difficulty=1
        )
        question_skill_link_model_1.update_timestamps()
        question_skill_link_model_2.update_timestamps()

        skill_1_model = self.create_model(
            skill_models.SkillModel,
            id=self.SKILL_1_ID,
            description=self.SKILL_1_DESCRIPTION,
            language_code=constants.DEFAULT_LANGUAGE_CODE,
            misconceptions=[],
            rubrics=[],
            skill_contents={
                'explanation': {
                    'html': 'test explanation',
                    'content_id': 'explanation',
                },
                'worked_examples': [],
                'recorded_voiceovers': {
                    'voiceovers_mapping': {}
                },
                'written_translations': {
                    'translations_mapping': {
                        'content': {},
                        'default_outcome': {}
                    }
                }
            },
            next_misconception_id=0,
            misconceptions_schema_version=feconf
                .CURRENT_MISCONCEPTIONS_SCHEMA_VERSION,
            rubric_schema_version=feconf
                .CURRENT_RUBRIC_SCHEMA_VERSION,
            skill_contents_schema_version=feconf
                .CURRENT_SKILL_CONTENTS_SCHEMA_VERSION,
            superseding_skill_id='blah',
            all_questions_merged=False,
            prerequisite_skill_ids=[]
        )

        skill_2_model = self.create_model(
            skill_models.SkillModel,
            id=self.SKILL_2_ID,
            description=self.SKILL_2_DESCRIPTION,
            language_code=constants.DEFAULT_LANGUAGE_CODE,
            misconceptions=[],
            rubrics=[],
            skill_contents={
                'explanation': {
                    'html': 'test explanation',
                    'content_id': 'explanation',
                },
                'worked_examples': [],
                'recorded_voiceovers': {
                    'voiceovers_mapping': {}
                },
                'written_translations': {
                    'translations_mapping': {
                        'content': {},
                        'default_outcome': {}
                    }
                }
            },
            next_misconception_id=0,
            misconceptions_schema_version=feconf
                .CURRENT_MISCONCEPTIONS_SCHEMA_VERSION,
            rubric_schema_version=feconf
                .CURRENT_RUBRIC_SCHEMA_VERSION,
            skill_contents_schema_version=feconf
                .CURRENT_SKILL_CONTENTS_SCHEMA_VERSION,
            superseding_skill_id='blah',
            all_questions_merged=False,
            prerequisite_skill_ids=[]
        )
        skill_1_model.update_timestamps()
        skill_2_model.update_timestamps()

        datastore_services.put_multi([
            skill_1_model,
            skill_2_model,
            question_skill_link_model_1,
            question_skill_link_model_2
        ])