def main(): parser = get_argparser() args = vars(parser.parse_args()) output_file = args.pop('file') # parse additional key=value pairs keyvals = {} for var in args['keyval']: key, value = var.split('=', 1) keyvals[key] = value args['keyvals'] = keyvals args.pop('keyval') detail = CheckDetail(**args) if os.path.isfile(output_file): # if output file exists, parse its content and append new detail to it f = open(output_file, 'r+') yaml = f.read() details = import_YAML(yaml) if yaml else [] details.append(detail) f.seek(0) f.write(export_YAML(details)) f.close() else: # if output file doesn't exist, create it f = open(output_file, 'w') f.write(export_YAML([detail])) f.close()
def process(self, params, arg_data): for param in ['item', 'type', 'checkname', 'file']: if param not in params.keys(): raise CheckbDirectiveError('Mandatory parameter missing: %s' % param) item = params['item'] itemtype = params['type'] checkname = params['checkname'] xunitfile = params['file'] aggregation = params.get('aggregation', 'allpass') if aggregation not in self.aggregations: raise CheckbDirectiveError( "Aggregation '%s' is not one of: %s" % (aggregation, ', '.join(self.aggregations))) with open(xunitfile) as xmlfile: testsuite, testresult = xunitparser.parse(xmlfile) if aggregation == 'none': details = [] for tc in testsuite: outcome = 'PASSED' if tc.good else 'FAILED' details.append( CheckDetail(item=item, checkname="%s.%s" % (checkname, tc.methodname), report_type=itemtype, outcome=outcome, artifact=xunitfile)) return export_YAML(details) elif aggregation == 'allpass': passed = len([tc for tc in testsuite if tc.good]) failed = len([tc for tc in testsuite if not tc.good]) final_outcome = 'PASSED' if failed == 0 else 'FAILED' note = CheckDetail.create_multi_item_summary(['PASSED'] * passed + ['FAILED'] * failed) return export_YAML( CheckDetail(item=item, report_type=itemtype, outcome=final_outcome, checkname=checkname, note=note, artifact=xunitfile)) else: assert False, "This should never happen, aggregation is %r" % aggregation
def setup(self, tmpdir): self.artifactsdir = tmpdir.mkdir('artifacts') self.report_filename = 'report.html' self.cd = check.CheckDetail( item='foo_bar', report_type=check.ReportType.KOJI_BUILD, outcome='NEEDS_INSPECTION', note='foo_bar note', output=["foo\nbar"], keyvals={ "foo": "moo1", "bar": "moo2" }, artifact='artifact.log', ) self.yaml = check.export_YAML(self.cd) self.ref_params = { 'results': self.yaml, 'artifactsdir': self.artifactsdir.strpath, } self.ref_arg_data = { 'resultsdb_job_id': 1, 'checkname': 'test_resultsdb_report', 'jobid': 'all/123', 'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701', } self.rd = create_report_directive.CreateReportDirective()
def test_custom_report_type(self): '''Test that user can specify a custom CheckDetail.report_type''' self.cd.report_type = 'My Custom Type' yaml_output = check.export_YAML(self.cd) yaml_obj = yaml.safe_load(yaml_output)['results'][0] assert yaml_obj['type'] == self.cd.report_type
def test_nonempty_run(self, monkeypatch): test_args = mock.MagicMock() test_args.__dict__ = { 'file': 'somefile', 'report_type': 'beer', 'item': 'punkipa', 'keyval': [], 'outcome': 'PASSED' } stub_parser = mock.MagicMock() stub_parser.parse_args = mock.MagicMock(return_value=test_args) stub_get_argparser = mock.MagicMock(return_value=stub_parser) monkeypatch.setattr(checkb_result, 'get_argparser', stub_get_argparser) stub_file = mock.MagicMock() stub_file.write = save_output stub_file.read = lambda: FILE monkeypatch.setattr(os.path, 'isfile', mock.MagicMock(return_value=True)) with mock.patch(builtin_open, mock.MagicMock(return_value=stub_file), create=True): checkb_result.main() assert OUTPUT == export_YAML( import_YAML(FILE) + [CheckDetail(**vars(test_args))])
def setup(self, monkeypatch): '''Run this before every test invocation''' self.cd = check.CheckDetail( item='foo_bar', report_type=check.ReportType.KOJI_BUILD, outcome='NEEDS_INSPECTION', note='foo_bar note', output=["foo\nbar"], keyvals={"foo": "moo1", "bar": "moo2"}, checkname='qa.test_resultsdb_report', ) self.yaml = check.export_YAML(self.cd) self.ref_input = {'results': self.yaml} self.ref_arg_data = { 'resultsdb_job_id': 1, 'jobid': 'all/123', 'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701', 'artifactsdir': '/some/directory/', 'task': '/taskdir', 'item': 'firefox-45.0.2-1.fc23' } self.ref_resultdata = {u'id': 1234} self.ref_jobid = 1234 self.ref_uuid = 'c25237a4-b6b3-11e4-b98a-3c970e018701' self.ref_refurl = u'http://example.com/%s' % self.cd.checkname self.ref_jobdata = {u'end_time': None, u'href': u'http://127.0.0.1/api/v2.0/jobs/%d' % self.ref_jobid, u'id': self.ref_jobid, u'name': self.cd.checkname, u'ref_url': self.ref_refurl, u'results': [], u'results_count': 0, u'start_time': None, u'status': u'SCHEDULED'} self.stub_rdb = mock.Mock(**{ 'get_testcase.return_value': {}, 'create_job.return_value': self.ref_jobdata, 'create_result.return_value': self.ref_resultdata, }) self.test_rdb = resultsdb_directive.ResultsdbDirective(self.stub_rdb) # while it appears useless, this actually sets config in several tests monkeypatch.setattr(config, '_config', None) self.conf = config.get_config() self.conf.report_to_resultsdb = True monkeypatch.setattr(configparser, 'ConfigParser', StubConfigParser)
def test_report_artifact_in_log_url(self, monkeypatch): """Checks whether artifact is correctly mapped to log_url""" cd = check.CheckDetail(item='foo_bar', report_type=check.ReportType.KOJI_BUILD, outcome='NEEDS_INSPECTION', note='foo_bar note', output=["foo\nbar"], keyvals={ "foo": "moo1", "bar": "moo2" }, checkname='qa.test_resultsdb_report', artifact='digest/logs/logfile.log') monkeypatch.setattr(self.test_rdb, 'get_artifact_path', lambda *x: 'digest/logs/logfile.log') yaml = check.export_YAML(cd) ref_input = {'results': yaml} self.test_rdb.process(ref_input, self.ref_arg_data) # Given the input data, the resultsdb should be called once, and only # once, calling "create_result". # This assert failing either means that more calls were added in the # source code, or that a bug is present, and "create_result" is # called multiple times. # we expect rdb to be called 4 times: create job, update to RUNNING, # check for testcase, report result and complete job assert len(self.stub_rdb.mock_calls) == 2 # Select the first call of "create_result" method. # This could be written as self.stub_rdb.calls()[0] at the moment, but # this is more future-proof, and accidental addition of resultsdb # calls is handled by the previous assert. call = [ call for call in self.stub_rdb.mock_calls if call[0] == 'create_result' ][0] # Select the keyword arguments of that call call_data = call[2] # the log url depends on the arg_data, so construct it here ref_log_url = '%s/%s/%s' %\ (self.conf.artifacts_baseurl, self.ref_arg_data['uuid'], cd.artifact) assert call_data['ref_url'] == ref_log_url
def test_import_exported(self): yaml_output = check.export_YAML(self.cd) cds = check.import_YAML(yaml_output) assert len(cds) == 1 assert isinstance(cds[0], check.CheckDetail) cd = cds[0] assert cd.item == self.cd.item assert cd.outcome == self.cd.outcome assert cd.note == self.cd.note assert cd.report_type == self.cd.report_type assert cd.keyvals == self.keyvals assert cd.checkname == self.checkname assert cd.artifact == self.artifact assert cd._internal == self._internal
def test_minimal(self): '''Test that empty CheckDetail values don't produce empty YAML lines, for example 'note:' should not be present if there's no CheckDetail.note''' cd = check.CheckDetail('foo') yaml_output = check.export_YAML(cd) yaml_obj = yaml.safe_load(yaml_output)['results'][0] # the output should look like this: # # results: # - item: XXX # outcome: XXX # # ('outcome:' is technically not a mandatory line, but with CheckDetail # export we produce it every time) assert len(yaml_obj) == 2
def test_multi(self): '''Test export with multiple item sections''' cd2 = check.CheckDetail(item='foobar-1.2-3.fc20', outcome='FAILED', note='dependency error', report_type=check.ReportType.BODHI_UPDATE) cd3 = check.CheckDetail(item='f20-updates', outcome='INFO', note='2 stale updates', report_type=check.ReportType.KOJI_TAG) yaml_output = check.export_YAML([self.cd, cd2, cd3]) yaml_obj = yaml.safe_load(yaml_output) assert len(yaml_obj['results']) == 3 assert yaml_obj['results'][0]['item'] == self.cd.item assert yaml_obj['results'][1]['item'] == cd2.item assert yaml_obj['results'][2]['item'] == cd3.item
def test_config_reporting_disabled(self): """Checks config option that disables reporting.""" conf = config.get_config() conf.report_to_resultsdb = False yaml = self.test_rdb.process(self.ref_input, self.ref_arg_data) cds = check.import_YAML(yaml) my_cd = check.import_YAML(check.export_YAML(self.cd)) # return value should be the same YAML assert len(cds) == 1 assert cds[0].__dict__ == my_cd[0].__dict__ # no call should have been made assert len(self.stub_rdb.mock_calls) == 0 config._config = None
def test_invalid_keyvals(self): '''Test export with keyvals containing reserved keys.''' cd = deepcopy(self.cd) for key in check.RESERVED_KEYS: cd.keyvals[key] = 'foo' yaml_output = check.export_YAML(cd) yaml_obj = yaml.load(yaml_output, Loader=yaml.SafeLoader)['results'][0] assert yaml_obj['item'] == self.item assert yaml_obj['outcome'] == self.outcome assert yaml_obj['note'] == self.note assert yaml_obj['type'] == self.report_type assert yaml_obj['foo'] == self.keyvals['foo'] assert yaml_obj['moo'] == self.keyvals['moo'] assert yaml_obj['checkname'] == self.checkname assert yaml_obj['_internal'] == self._internal
def test_single_yaml(self): '''Test export with a single item section.''' yaml_output = check.export_YAML(self.cd) yaml_obj = yaml.safe_load(yaml_output) assert type(yaml_obj) is dict assert type(yaml_obj['results']) is list yaml_obj = yaml_obj['results'][0] assert yaml_obj['item'] == self.item assert yaml_obj['outcome'] == self.outcome assert yaml_obj['note'] == self.note assert yaml_obj['type'] == self.report_type assert yaml_obj['foo'] == self.keyvals['foo'] assert yaml_obj['moo'] == self.keyvals['moo'] assert yaml_obj['checkname'] == self.checkname assert yaml_obj['_internal'] == self._internal
def test_keyval_parse(self, monkeypatch): test_args = mock.MagicMock() test_args.__dict__ = { 'file': 'somefile', 'report_type': 'beer', 'item': 'punkipa', 'keyval': ['hop=Simcoe'], 'outcome': 'PASSED' } stub_parser = mock.MagicMock() stub_parser.parse_args = mock.MagicMock(return_value=test_args) stub_get_argparser = mock.MagicMock(return_value=stub_parser) monkeypatch.setattr(checkb_result, 'get_argparser', stub_get_argparser) stub_file = mock.MagicMock() stub_file.write = save_output with mock.patch(builtin_open, mock.MagicMock(return_value=stub_file), create=True): checkb_result.main() assert OUTPUT == export_YAML(CheckDetail(**vars(test_args)))
def process(self, params, arg_data): # checking if reporting is enabled is done after importing yaml which # serves as validation of input results if 'file' in params and 'results' in params: raise CheckbDirectiveError( "Either `file` or `results` can be used, not both.") try: if params.get('file', None): with open(params['file']) as resultfile: params['results'] = resultfile.read() check_details = check.import_YAML(params['results']) log.debug("YAML output parsed OK.") except (CheckbValueError, IOError) as e: raise CheckbDirectiveError("Failed to load results: %s" % e) for detail in check_details: if not (detail.item and detail.report_type and detail.checkname): raise CheckbDirectiveError( "The resultsdb directive requires 'item', 'type' and " "'checkname' to be present in the YAML data.") conf = config.get_config() if not conf.report_to_resultsdb: log.info( "Reporting to ResultsDB is disabled. Once enabled, the " "following would get reported:\n%s", params['results']) return check.export_YAML(check_details) artifactsdir_url = '%s/%s' % (self.artifacts_baseurl, arg_data['uuid']) # for now, we're creating the resultsdb group at reporting time group_data = self.create_resultsdb_group(uuid=arg_data['uuid']) log.info('Posting %s results to ResultsDB...' % len(check_details)) for detail in check_details: checkname = detail.checkname # find out if the task is allowed to post results into the namespace if config.get_config().profile == config.ProfileName.PRODUCTION: self.check_namespace(checkname, arg_data) self.ensure_testcase_exists(checkname) result_log_url = artifactsdir_url if detail.artifact: artifact_path = self.get_artifact_path( arg_data['artifactsdir'], detail.artifact) if artifact_path: result_log_url = "%s/%s" % (artifactsdir_url, artifact_path) try: result = self.resultsdb.create_result(outcome=detail.outcome, testcase=checkname, groups=[group_data], note=detail.note or None, ref_url=result_log_url, item=detail.item, type=detail.report_type, **detail.keyvals) log.debug('Result saved in ResultsDB:\n%s', pprint.pformat(result)) detail._internal['resultsdb_result_id'] = result['id'] except resultsdb_api.ResultsDBapiException as e: log.error(e) log.error("Failed to store to ResultsDB: `%s` `%s` `%s`", detail.item, checkname, detail.outcome) return check.export_YAML(check_details)
def test_invalid_missing_item(self): '''Test invalid input parameters''' with pytest.raises(exc.CheckbValueError): self.cd.item = None check.export_YAML(self.cd)
def update_input(self): self.yaml = check.export_YAML(self.cd) self.ref_input = {'results': self.yaml}