예제 #1
0
def main():

    parser = get_argparser()

    args = vars(parser.parse_args())

    output_file = args.pop('file')

    # parse additional key=value pairs
    keyvals = {}
    for var in args['keyval']:
        key, value = var.split('=', 1)
        keyvals[key] = value
    args['keyvals'] = keyvals
    args.pop('keyval')

    detail = CheckDetail(**args)

    if os.path.isfile(output_file):
        # if output file exists, parse its content and append new detail to it
        f = open(output_file, 'r+')
        yaml = f.read()
        details = import_YAML(yaml) if yaml else []
        details.append(detail)
        f.seek(0)
        f.write(export_YAML(details))
        f.close()
    else:
        # if output file doesn't exist, create it
        f = open(output_file, 'w')
        f.write(export_YAML([detail]))
        f.close()
예제 #2
0
    def process(self, params, arg_data):
        for param in ['item', 'type', 'checkname', 'file']:
            if param not in params.keys():
                raise CheckbDirectiveError('Mandatory parameter missing: %s' %
                                           param)
        item = params['item']
        itemtype = params['type']
        checkname = params['checkname']
        xunitfile = params['file']

        aggregation = params.get('aggregation', 'allpass')

        if aggregation not in self.aggregations:
            raise CheckbDirectiveError(
                "Aggregation '%s' is not one of: %s" %
                (aggregation, ', '.join(self.aggregations)))

        with open(xunitfile) as xmlfile:
            testsuite, testresult = xunitparser.parse(xmlfile)

        if aggregation == 'none':
            details = []
            for tc in testsuite:
                outcome = 'PASSED' if tc.good else 'FAILED'
                details.append(
                    CheckDetail(item=item,
                                checkname="%s.%s" % (checkname, tc.methodname),
                                report_type=itemtype,
                                outcome=outcome,
                                artifact=xunitfile))
            return export_YAML(details)

        elif aggregation == 'allpass':
            passed = len([tc for tc in testsuite if tc.good])
            failed = len([tc for tc in testsuite if not tc.good])
            final_outcome = 'PASSED' if failed == 0 else 'FAILED'
            note = CheckDetail.create_multi_item_summary(['PASSED'] * passed +
                                                         ['FAILED'] * failed)

            return export_YAML(
                CheckDetail(item=item,
                            report_type=itemtype,
                            outcome=final_outcome,
                            checkname=checkname,
                            note=note,
                            artifact=xunitfile))

        else:
            assert False, "This should never happen, aggregation is %r" % aggregation
    def setup(self, tmpdir):
        self.artifactsdir = tmpdir.mkdir('artifacts')
        self.report_filename = 'report.html'

        self.cd = check.CheckDetail(
            item='foo_bar',
            report_type=check.ReportType.KOJI_BUILD,
            outcome='NEEDS_INSPECTION',
            note='foo_bar note',
            output=["foo\nbar"],
            keyvals={
                "foo": "moo1",
                "bar": "moo2"
            },
            artifact='artifact.log',
        )

        self.yaml = check.export_YAML(self.cd)

        self.ref_params = {
            'results': self.yaml,
            'artifactsdir': self.artifactsdir.strpath,
        }
        self.ref_arg_data = {
            'resultsdb_job_id': 1,
            'checkname': 'test_resultsdb_report',
            'jobid': 'all/123',
            'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701',
        }

        self.rd = create_report_directive.CreateReportDirective()
예제 #4
0
    def test_custom_report_type(self):
        '''Test that user can specify a custom CheckDetail.report_type'''
        self.cd.report_type = 'My Custom Type'
        yaml_output = check.export_YAML(self.cd)
        yaml_obj = yaml.safe_load(yaml_output)['results'][0]

        assert yaml_obj['type'] == self.cd.report_type
예제 #5
0
    def test_nonempty_run(self, monkeypatch):
        test_args = mock.MagicMock()
        test_args.__dict__ = {
            'file': 'somefile',
            'report_type': 'beer',
            'item': 'punkipa',
            'keyval': [],
            'outcome': 'PASSED'
        }
        stub_parser = mock.MagicMock()
        stub_parser.parse_args = mock.MagicMock(return_value=test_args)
        stub_get_argparser = mock.MagicMock(return_value=stub_parser)
        monkeypatch.setattr(checkb_result, 'get_argparser', stub_get_argparser)

        stub_file = mock.MagicMock()
        stub_file.write = save_output
        stub_file.read = lambda: FILE

        monkeypatch.setattr(os.path, 'isfile',
                            mock.MagicMock(return_value=True))

        with mock.patch(builtin_open,
                        mock.MagicMock(return_value=stub_file),
                        create=True):
            checkb_result.main()

        assert OUTPUT == export_YAML(
            import_YAML(FILE) + [CheckDetail(**vars(test_args))])
예제 #6
0
    def setup(self, monkeypatch):
        '''Run this before every test invocation'''

        self.cd = check.CheckDetail(
            item='foo_bar',
            report_type=check.ReportType.KOJI_BUILD,
            outcome='NEEDS_INSPECTION',
            note='foo_bar note',
            output=["foo\nbar"],
            keyvals={"foo": "moo1", "bar": "moo2"},
            checkname='qa.test_resultsdb_report',
            )

        self.yaml = check.export_YAML(self.cd)

        self.ref_input = {'results': self.yaml}
        self.ref_arg_data = {
            'resultsdb_job_id': 1,
            'jobid': 'all/123',
            'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701',
            'artifactsdir': '/some/directory/',
            'task': '/taskdir',
            'item': 'firefox-45.0.2-1.fc23'
            }

        self.ref_resultdata = {u'id': 1234}

        self.ref_jobid = 1234
        self.ref_uuid = 'c25237a4-b6b3-11e4-b98a-3c970e018701'
        self.ref_refurl = u'http://example.com/%s' % self.cd.checkname
        self.ref_jobdata = {u'end_time': None,
                            u'href': u'http://127.0.0.1/api/v2.0/jobs/%d' % self.ref_jobid,
                            u'id': self.ref_jobid,
                            u'name': self.cd.checkname,
                            u'ref_url': self.ref_refurl,
                            u'results': [],
                            u'results_count': 0,
                            u'start_time': None,
                            u'status': u'SCHEDULED'}

        self.stub_rdb = mock.Mock(**{
            'get_testcase.return_value': {},
            'create_job.return_value': self.ref_jobdata,
            'create_result.return_value': self.ref_resultdata,
            })
        self.test_rdb = resultsdb_directive.ResultsdbDirective(self.stub_rdb)

        # while it appears useless, this actually sets config in several tests
        monkeypatch.setattr(config, '_config', None)
        self.conf = config.get_config()
        self.conf.report_to_resultsdb = True

        monkeypatch.setattr(configparser, 'ConfigParser', StubConfigParser)
예제 #7
0
    def test_report_artifact_in_log_url(self, monkeypatch):
        """Checks whether artifact is correctly mapped to log_url"""
        cd = check.CheckDetail(item='foo_bar',
                               report_type=check.ReportType.KOJI_BUILD,
                               outcome='NEEDS_INSPECTION',
                               note='foo_bar note',
                               output=["foo\nbar"],
                               keyvals={
                                   "foo": "moo1",
                                   "bar": "moo2"
                               },
                               checkname='qa.test_resultsdb_report',
                               artifact='digest/logs/logfile.log')
        monkeypatch.setattr(self.test_rdb, 'get_artifact_path',
                            lambda *x: 'digest/logs/logfile.log')

        yaml = check.export_YAML(cd)
        ref_input = {'results': yaml}

        self.test_rdb.process(ref_input, self.ref_arg_data)

        # Given the input data, the resultsdb should be called once, and only
        #   once, calling "create_result".
        # This assert failing either means that more calls were added in the
        #   source code, or that a bug is present, and "create_result" is
        #   called multiple times.
        # we expect rdb to be called 4 times: create job, update to RUNNING,
        # check for testcase, report result and complete job

        assert len(self.stub_rdb.mock_calls) == 2

        # Select the first call of "create_result" method.
        # This could be written as self.stub_rdb.calls()[0] at the moment, but
        #   this is more future-proof, and accidental addition of resultsdb
        #   calls is handled by the previous assert.
        call = [
            call for call in self.stub_rdb.mock_calls
            if call[0] == 'create_result'
        ][0]
        # Select the keyword arguments of that call
        call_data = call[2]

        # the log url depends on the arg_data, so construct it here
        ref_log_url = '%s/%s/%s' %\
                      (self.conf.artifacts_baseurl, self.ref_arg_data['uuid'], cd.artifact)

        assert call_data['ref_url'] == ref_log_url
예제 #8
0
    def test_import_exported(self):
        yaml_output = check.export_YAML(self.cd)

        cds = check.import_YAML(yaml_output)

        assert len(cds) == 1
        assert isinstance(cds[0], check.CheckDetail)

        cd = cds[0]
        assert cd.item == self.cd.item
        assert cd.outcome == self.cd.outcome
        assert cd.note == self.cd.note
        assert cd.report_type == self.cd.report_type
        assert cd.keyvals == self.keyvals
        assert cd.checkname == self.checkname
        assert cd.artifact == self.artifact
        assert cd._internal == self._internal
예제 #9
0
    def test_minimal(self):
        '''Test that empty CheckDetail values don't produce empty YAML lines,
        for example 'note:' should not be present if there's no
        CheckDetail.note'''
        cd = check.CheckDetail('foo')
        yaml_output = check.export_YAML(cd)
        yaml_obj = yaml.safe_load(yaml_output)['results'][0]

        # the output should look like this:
        #
        # results:
        #   - item: XXX
        #     outcome: XXX
        #
        # ('outcome:' is technically not a mandatory line, but with CheckDetail
        # export we produce it every time)
        assert len(yaml_obj) == 2
예제 #10
0
    def test_multi(self):
        '''Test export with multiple item sections'''
        cd2 = check.CheckDetail(item='foobar-1.2-3.fc20',
                                outcome='FAILED',
                                note='dependency error',
                                report_type=check.ReportType.BODHI_UPDATE)
        cd3 = check.CheckDetail(item='f20-updates',
                                outcome='INFO',
                                note='2 stale updates',
                                report_type=check.ReportType.KOJI_TAG)
        yaml_output = check.export_YAML([self.cd, cd2, cd3])
        yaml_obj = yaml.safe_load(yaml_output)

        assert len(yaml_obj['results']) == 3
        assert yaml_obj['results'][0]['item'] == self.cd.item
        assert yaml_obj['results'][1]['item'] == cd2.item
        assert yaml_obj['results'][2]['item'] == cd3.item
예제 #11
0
    def test_config_reporting_disabled(self):
        """Checks config option that disables reporting."""
        conf = config.get_config()

        conf.report_to_resultsdb = False

        yaml = self.test_rdb.process(self.ref_input, self.ref_arg_data)
        cds = check.import_YAML(yaml)
        my_cd = check.import_YAML(check.export_YAML(self.cd))
        # return value should be the same YAML
        assert len(cds) == 1
        assert cds[0].__dict__ == my_cd[0].__dict__

        # no call should have been made
        assert len(self.stub_rdb.mock_calls) == 0

        config._config = None
예제 #12
0
    def test_invalid_keyvals(self):
        '''Test export with keyvals containing reserved keys.'''
        cd = deepcopy(self.cd)

        for key in check.RESERVED_KEYS:
            cd.keyvals[key] = 'foo'

        yaml_output = check.export_YAML(cd)
        yaml_obj = yaml.load(yaml_output, Loader=yaml.SafeLoader)['results'][0]

        assert yaml_obj['item'] == self.item
        assert yaml_obj['outcome'] == self.outcome
        assert yaml_obj['note'] == self.note
        assert yaml_obj['type'] == self.report_type
        assert yaml_obj['foo'] == self.keyvals['foo']
        assert yaml_obj['moo'] == self.keyvals['moo']
        assert yaml_obj['checkname'] == self.checkname
        assert yaml_obj['_internal'] == self._internal
예제 #13
0
    def test_single_yaml(self):
        '''Test export with a single item section.'''
        yaml_output = check.export_YAML(self.cd)
        yaml_obj = yaml.safe_load(yaml_output)

        assert type(yaml_obj) is dict
        assert type(yaml_obj['results']) is list

        yaml_obj = yaml_obj['results'][0]

        assert yaml_obj['item'] == self.item
        assert yaml_obj['outcome'] == self.outcome
        assert yaml_obj['note'] == self.note
        assert yaml_obj['type'] == self.report_type
        assert yaml_obj['foo'] == self.keyvals['foo']
        assert yaml_obj['moo'] == self.keyvals['moo']
        assert yaml_obj['checkname'] == self.checkname
        assert yaml_obj['_internal'] == self._internal
예제 #14
0
    def test_keyval_parse(self, monkeypatch):
        test_args = mock.MagicMock()
        test_args.__dict__ = {
            'file': 'somefile',
            'report_type': 'beer',
            'item': 'punkipa',
            'keyval': ['hop=Simcoe'],
            'outcome': 'PASSED'
        }
        stub_parser = mock.MagicMock()
        stub_parser.parse_args = mock.MagicMock(return_value=test_args)
        stub_get_argparser = mock.MagicMock(return_value=stub_parser)
        monkeypatch.setattr(checkb_result, 'get_argparser', stub_get_argparser)

        stub_file = mock.MagicMock()
        stub_file.write = save_output

        with mock.patch(builtin_open,
                        mock.MagicMock(return_value=stub_file),
                        create=True):
            checkb_result.main()

        assert OUTPUT == export_YAML(CheckDetail(**vars(test_args)))
예제 #15
0
    def process(self, params, arg_data):
        # checking if reporting is enabled is done after importing yaml which
        # serves as validation of input results

        if 'file' in params and 'results' in params:
            raise CheckbDirectiveError(
                "Either `file` or `results` can be used, not both.")

        try:
            if params.get('file', None):
                with open(params['file']) as resultfile:
                    params['results'] = resultfile.read()

            check_details = check.import_YAML(params['results'])
            log.debug("YAML output parsed OK.")
        except (CheckbValueError, IOError) as e:
            raise CheckbDirectiveError("Failed to load results: %s" % e)

        for detail in check_details:
            if not (detail.item and detail.report_type and detail.checkname):
                raise CheckbDirectiveError(
                    "The resultsdb directive requires 'item', 'type' and "
                    "'checkname' to be present in the YAML data.")

        conf = config.get_config()
        if not conf.report_to_resultsdb:
            log.info(
                "Reporting to ResultsDB is disabled. Once enabled, the "
                "following would get reported:\n%s", params['results'])
            return check.export_YAML(check_details)

        artifactsdir_url = '%s/%s' % (self.artifacts_baseurl, arg_data['uuid'])

        # for now, we're creating the resultsdb group at reporting time
        group_data = self.create_resultsdb_group(uuid=arg_data['uuid'])

        log.info('Posting %s results to ResultsDB...' % len(check_details))
        for detail in check_details:
            checkname = detail.checkname

            # find out if the task is allowed to post results into the namespace
            if config.get_config().profile == config.ProfileName.PRODUCTION:
                self.check_namespace(checkname, arg_data)

            self.ensure_testcase_exists(checkname)
            result_log_url = artifactsdir_url
            if detail.artifact:
                artifact_path = self.get_artifact_path(
                    arg_data['artifactsdir'], detail.artifact)
                if artifact_path:
                    result_log_url = "%s/%s" % (artifactsdir_url,
                                                artifact_path)
            try:
                result = self.resultsdb.create_result(outcome=detail.outcome,
                                                      testcase=checkname,
                                                      groups=[group_data],
                                                      note=detail.note or None,
                                                      ref_url=result_log_url,
                                                      item=detail.item,
                                                      type=detail.report_type,
                                                      **detail.keyvals)
                log.debug('Result saved in ResultsDB:\n%s',
                          pprint.pformat(result))
                detail._internal['resultsdb_result_id'] = result['id']

            except resultsdb_api.ResultsDBapiException as e:
                log.error(e)
                log.error("Failed to store to ResultsDB: `%s` `%s` `%s`",
                          detail.item, checkname, detail.outcome)

        return check.export_YAML(check_details)
예제 #16
0
 def test_invalid_missing_item(self):
     '''Test invalid input parameters'''
     with pytest.raises(exc.CheckbValueError):
         self.cd.item = None
         check.export_YAML(self.cd)
예제 #17
0
 def update_input(self):
     self.yaml = check.export_YAML(self.cd)
     self.ref_input = {'results': self.yaml}