Пример #1
0
    def test_0101_run_reruns_once_record_has_been_modified(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task that does not force re-running on unmodified records
        task_data = dict(TestApi.task_data)
        task_data['force_run_on_unmodified_records'] = False
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                # Run once
                task_id = run_task(task_data['name'])
                execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
                assert execution.status == StatusMaster.completed
                # Mark first record as modified
                rec = get_record(small_rng[0])
                rec['unrelated_thing'] = 'unrelated_key'
                rec.commit()
                # Run again
                task_id = run_task(task_data['name'])

        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        assert get_record(small_rng[0])['field1'] == 3.4 * 2
        assert get_record(small_rng[1])['field1'] == 3.4
Пример #2
0
    def test_074_run_modifies_records_appropriately(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task that forces re-running on unmodified records
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'])

        # Ensure that it modifies the records as coded
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        for i in small_rng:
            assert get_record(i)['field1'] == 3.4

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'])

        # And that it picks up the changes it did last time on the next run
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        for i in small_rng:
            assert get_record(i)['field1'] == 3.4 * 2
Пример #3
0
def task_run():
    requested_task_names = request.values.getlist('task_names[]')
    tasks_in_db = CheckerRule.query.filter(
        CheckerRule.name.in_(requested_task_names)).all()
    if len(tasks_in_db) != len(requested_task_names):
        return jsonify({'error': 'Missing tasks requested'}), 400
    for task in tasks_in_db:
        run_task(task.name)
    return jsonify({})
Пример #4
0
    def test_036_run_task_with_missing_name_raises(self):
        from invenio_checker.api import run_task

        # Given a task name that's not in the database
        task_name = TestApi.test_entry_prefix + "does not exist"

        # Make sure the task complains before branching into celery
        with pytest.raises(NoResultFound):
            run_task(task_name)
Пример #5
0
def task_run():
    requested_task_names = request.values.getlist('task_names[]')
    tasks_in_db = CheckerRule.query.filter(
        CheckerRule.name.in_(requested_task_names)).all()
    if len(tasks_in_db) != len(requested_task_names):
        return jsonify({'error': 'Missing tasks requested'}), 400
    for task in tasks_in_db:
        run_task(task.name)
    return jsonify({})
Пример #6
0
    def test_0103_worker_that_spun_returns_result(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task

        # Given a task
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        # ..that conflicts with other workers for a while
        def conflict_yielder():
            conflicts = (
                {MagicMock(uuid=1), MagicMock(uuid=2)},
                {MagicMock(uuid=1)},
                {},
            )
            global conflict_check_count
            conflict_check_count = 0
            for conflict in conflicts:
                conflict_check_count += 1
                yield conflict

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_without_class):
            with patch('invenio_checker.models.Query', Query):
                with patch('invenio_checker.conftest.conftest_checker._worker_conflicts_with_currently_running',
                           side_effect=conflict_yielder()):
                    task_id = run_task(task_data['name'])

        # Ensure it finishes successfully
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        assert conflict_check_count == 3
Пример #7
0
    def test_0104_worker_does_not_conflict_with_others_when_not_using_record_related_fixtures(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task

        # Given a task
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        # ..that has no conflicts (since it should have None,None as allowed_{paths,recids})
        def m_worker_conflicts_with_currently_running(worker):
            assert worker.allowed_paths is None
            assert worker.allowed_recids is None
            return {}

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_without_any_record_fetching_fixture):
            with patch('invenio_checker.models.Query', Query):
                with patch('invenio_checker.conftest.conftest_checker._worker_conflicts_with_currently_running',
                           side_effect=m_worker_conflicts_with_currently_running):
                    task_id = run_task(task_data['name'])

        # Ensure it finishes successfully
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
Пример #8
0
    def test_0100_run_does_not_run_twice_on_records_that_did_not_change_since_last_run(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task that does not force re-running on unmodified records
        task_data = dict(TestApi.task_data)
        task_data['force_run_on_unmodified_records'] = False
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                run_task(task_data['name'])
                task_id = run_task(task_data['name'])

        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        for i in small_rng:
            assert get_record(i)['field1'] == 3.4
Пример #9
0
    def test_077_run_on_non_record_centric_calls_check_function(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task that does not force re-running on unmodified records
        task_data = dict(TestApi.task_data)
        task_data['force_run_on_unmodified_records'] = False
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records([6000000, 6000001])

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_non_record_centric):
            with patch('invenio_checker.models.Query', Query):
                run_task(task_data['name'])
                task_id = run_task(task_data['name'])

        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        assert get_record(6000000)['a_field'] == 'a_value_2'
        assert get_record(6000001)['a_field'] == 'another_value'
Пример #10
0
    def test_043_run_task_creates_execution_object_in_database(self):
        from invenio_checker.api import run_task, create_task

        # Given a task
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'])

        # Make sure the execution information is stored in the database
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).first()
        assert execution
Пример #11
0
    def test_055_run_reports_successful_completion_without_class(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import run_task, create_task

        # Given a task with a check that uses a class
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_without_class):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'])

        # Make sure it reports successful completion
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
Пример #12
0
    def test_0110_run_with_multiple_functions_in_check_file_fails(self):
        from invenio_checker.api import create_task, run_task
        from invenio_checker.clients.master import StatusMaster

        # Given a task..
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        # ..where the file has too many check functions
        with patch('invenio_checker.models.CheckerRule.filepath', filepath_two_check_functions):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'])

        # Ensure that it failed
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.failed
Пример #13
0
    def test_0102_run_without_record_fixture_runs_once(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task
        task_data = TestApi.task_data
        # Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_without_record_fixture):
            task_id = run_task(task_data['name'])

        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        assert get_record(small_rng[0])['this_record_has_id_1'] == True
        assert 'this_record_has_id_1' not in get_record(small_rng[1])
Пример #14
0
    def test_077_run_with_dry_run_does_not_call_reporters_for_exceptions(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task that forces re-running on unmodified records
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_raises_exception_every_time):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'], dry_run=True)

        # Ensure that it modifies the records as coded
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.failed
        assert reported_exceptions == 0
Пример #15
0
    def test_0105_run_initialized_reporters_only_when_not_spinning(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task, create_reporter

        # Given a task with a reporter
        task_data = TestApi.task_data
        new_task = create_task(task_data)
        new_reporter = create_reporter(TestApi.reporter_data)
        Query = get_Query(task_data)
        self.create_records(small_rng)

        # ..while tracking the reporter's initialization
        reporterA = reimport_module('tests.demo_package.checkerext.reporters.reporterA')
        reporterA.get_reporter = MagicMock()

        # ..as well as calls to the task conflict resolver
        conflict_resolver = Mock(side_effect=({MagicMock(uuid=1)}, {}))
        mock_manager = Mock()
        mock_manager.attach_mock(reporterA.get_reporter, 'get_reporter')
        mock_manager.attach_mock(conflict_resolver, 'conflict_resolver')

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_without_class):
            with patch('invenio_checker.models.Query', Query):
                with patch('invenio_checker.models.CheckerReporter.module', reporterA):
                    with patch('invenio_checker.conftest.conftest_checker._worker_conflicts_with_currently_running',
                               conflict_resolver):
                        task_id = run_task(task_data['name'])

        # (Better safe than (very) sorry)
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed

        # Ensure that the reporter was not initialized before no conflicts were remaining
        from ..conftest import contains_sublist
        assert contains_sublist(
            [call[0] for call in mock_manager.mock_calls],  # func names
            [
                'conflict_resolver',
                'conflict_resolver',
                'get_reporter',
            ]
        )
Пример #16
0
    def test_0106_run_does_not_start_when_file_is_missing(self):
        """
        ..note::
            It shouldn't be necessary to have records for this to fail but
            that's how it is. FIXME
        """
        from invenio_checker.api import create_task, run_task
        from invenio_checker.clients.master import StatusMaster

        # Given a task..
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        # ..whose check file is absent
        with patch('invenio_checker.models.CheckerRule.filepath', None):
            with patch('invenio_checker.models.Query', Query):
                with pytest.raises(Exception):
                    task_id = run_task(task_data['name'])
Пример #17
0
    def test_075_run_with_dry_run_does_not_modify_records(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import create_task, run_task
        from invenio_records.api import get_record

        # Given a task that forces re-running on unmodified records
        task_data = TestApi.task_data
        Query = get_Query(task_data)
        create_task(task_data)
        self.create_records(small_rng)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                task_id = run_task(task_data['name'], dry_run=True)

        # Ensure that it modifies the records as coded
        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        for i in small_rng:
            assert 'field1' not in get_record(i)
Пример #18
0
    def test_0110_run_calls_reporters_when_check_wants_to_log(self):
        from invenio_checker.clients.master import StatusMaster
        from invenio_checker.api import run_task, create_task, create_reporter

        # Given a task..
        task_data = TestApi.task_data
        new_task = create_task(task_data)
        # ..with a reporter attached
        new_reporter = create_reporter(TestApi.reporter_data)
        # and some records in the database
        self.create_records(small_rng)

        reporterA = reimport_module('tests.demo_package.checkerext.reporters.reporterA')
        Query = get_Query(task_data)

        with patch('invenio_checker.models.CheckerRule.filepath', filepath_with_class):
            with patch('invenio_checker.models.Query', Query):
                with patch('invenio_checker.models.CheckerReporter.module', reporterA):
                    task_id = run_task(task_data['name'])

        execution = CheckerRuleExecution.query.filter(CheckerRuleExecution.uuid == task_id).one()
        assert execution.status == StatusMaster.completed
        assert reported_reports == len(small_rng)
Пример #19
0
def submit_task():
    """Insert or modify an existing task and its reporters."""
    from invenio_checker.clients.supervisor import run_task

    def failure(type_, errors):
        assert type_ in ('general', 'validation')
        return jsonify({'failure_type': type_, 'errors': errors}), 400

    def success():
        return jsonify({})

    # Recreate the forms that we have previously served to the user so that we
    # can validate.
    form_origin = get_NewTaskForm(request.form)
    form_plugin = get_ArgForm(request.form['plugin'], request.form)
    if not (form_origin.validate() & form_plugin.validate()):
        form_errors = defaultdict(list)
        for field, errors in chain(form_origin.errors.items(),
                                   form_plugin.errors.items()):
            form_errors[field].extend(errors)
        return failure('validation', form_errors)

    # Get a dictionary that we can pass as kwargs to the database object,
    form_for_db = dict(form_origin.data)
    # but first, pop metadata out of it.
    modify = form_for_db.pop('modify')
    original_name = form_for_db.pop('original_name')
    requested_action = form_for_db.pop('requested_action')
    reporter_names = set(form_for_db.pop('reporters'))
    form_for_db['arguments'] = form_plugin.data_for_db

    try:

        # Create or edit task
        if modify:
            task = edit_task(original_name, form_for_db, commit=False)
        else:
            task = create_task(form_for_db, commit=False)

        # Create or edit reporters and attach them as well
        for reporter_name in reporter_names:
            form_reporter = get_ArgForm(reporter_name, request.form)
            try:
                reporter = get_reporter_db(form_reporter.plugin_name,
                                           original_name)
            except NoResultFound:
                create_reporter(
                    {
                        'plugin': form_reporter.plugin_name,
                        'rule_name': task.name,
                        'arguments': form_reporter.data_for_db
                    },
                    commit=False)
            else:
                edit_reporter(reporter, {
                    'plugin': form_reporter.plugin_name,
                    'rule_name': task.name,
                    'arguments': form_reporter.data_for_db
                },
                              commit=False)
        if modify:
            # Delete reporters that are no longer selected
            attached_reporter_plugin_names = {
                reporter.plugin
                for reporter in task.reporters
            }
            for plugin_name_to_remove in attached_reporter_plugin_names - reporter_names:
                ex_reporter = get_reporter_db(plugin_name_to_remove,
                                              original_name)
                remove_reporter(ex_reporter, commit=False)

        db.session.commit()
    except Exception as e:
        db.session.rollback()
        return failure('general', format_exc())

    if requested_action.startswith('submit_run'):
        try:
            run_task(task.name)
        except Exception as e:
            return failure('general', format_exc())

    return success()
Пример #20
0
def submit_task():
    """Insert or modify an existing task and its reporters."""
    from invenio_checker.clients.supervisor import run_task

    def failure(type_, errors):
        assert type_ in ('general', 'validation')
        return jsonify({'failure_type': type_, 'errors': errors}), 400

    def success():
        return jsonify({})

    # Recreate the forms that we have previously served to the user so that we
    # can validate.
    form_origin = get_NewTaskForm(request.form)
    form_plugin = get_ArgForm(request.form['plugin'], request.form)
    if not (form_origin.validate() & form_plugin.validate()):
        form_errors = defaultdict(list)
        for field, errors in chain(form_origin.errors.items(),
                                   form_plugin.errors.items()):
            form_errors[field].extend(errors)
        return failure('validation', form_errors)

    # Get a dictionary that we can pass as kwargs to the database object,
    form_for_db = dict(form_origin.data)
    # but first, pop metadata out of it.
    modify = form_for_db.pop('modify')
    original_name = form_for_db.pop('original_name')
    requested_action = form_for_db.pop('requested_action')
    reporter_names = set(form_for_db.pop('reporters'))
    form_for_db['arguments'] = form_plugin.data_for_db

    try:

        # Create or edit task
        if modify:
            task = edit_task(original_name, form_for_db, commit=False)
        else:
            task = create_task(form_for_db, commit=False)

        # Create or edit reporters and attach them as well
        for reporter_name in reporter_names:
            form_reporter = get_ArgForm(reporter_name, request.form)
            try:
                reporter = get_reporter_db(form_reporter.plugin_name, original_name)
            except NoResultFound:
                create_reporter({'plugin': form_reporter.plugin_name,
                                 'rule_name': task.name,
                                 'arguments': form_reporter.data_for_db},
                                 commit=False)
            else:
                edit_reporter(reporter,
                              {'plugin': form_reporter.plugin_name,
                               'rule_name': task.name,
                               'arguments': form_reporter.data_for_db},
                              commit=False)
        if modify:
            # Delete reporters that are no longer selected
            attached_reporter_plugin_names = {reporter.plugin for reporter in task.reporters}
            for plugin_name_to_remove in attached_reporter_plugin_names - reporter_names:
                ex_reporter = get_reporter_db(plugin_name_to_remove, original_name)
                remove_reporter(ex_reporter, commit=False)

        db.session.commit()
    except Exception as e:
        db.session.rollback()
        return failure('general', format_exc())

    if requested_action.startswith('submit_run'):
        try:
            run_task(task.name)
        except Exception as e:
            return failure('general', format_exc())

    return success()