def main():
parser = get_argparser()
args = vars(parser.parse_args())
output_file = args.pop('file')
# parse additional key=value pairs
keyvals = {}
for var in args['keyval']:
key, value = var.split('=', 1)
keyvals[key] = value
args['keyvals'] = keyvals
args.pop('keyval')
detail = CheckDetail(**args)
if os.path.isfile(output_file):
# if output file exists, parse its content and append new detail to it
f = open(output_file, 'r+')
yaml = f.read()
details = import_YAML(yaml) if yaml else []
details.append(detail)
f.seek(0)
f.write(export_YAML(details))
f.close()
else:
# if output file doesn't exist, create it
f = open(output_file, 'w')
f.write(export_YAML([detail]))
f.close()
def process(self, params, arg_data):
for param in ['item', 'type', 'checkname', 'file']:
if param not in params.keys():
raise CheckbDirectiveError('Mandatory parameter missing: %s' %
param)
item = params['item']
itemtype = params['type']
checkname = params['checkname']
xunitfile = params['file']
aggregation = params.get('aggregation', 'allpass')
if aggregation not in self.aggregations:
raise CheckbDirectiveError(
"Aggregation '%s' is not one of: %s" %
(aggregation, ', '.join(self.aggregations)))
with open(xunitfile) as xmlfile:
testsuite, testresult = xunitparser.parse(xmlfile)
if aggregation == 'none':
details = []
for tc in testsuite:
outcome = 'PASSED' if tc.good else 'FAILED'
details.append(
CheckDetail(item=item,
checkname="%s.%s" % (checkname, tc.methodname),
report_type=itemtype,
outcome=outcome,
artifact=xunitfile))
return export_YAML(details)
elif aggregation == 'allpass':
passed = len([tc for tc in testsuite if tc.good])
failed = len([tc for tc in testsuite if not tc.good])
final_outcome = 'PASSED' if failed == 0 else 'FAILED'
note = CheckDetail.create_multi_item_summary(['PASSED'] * passed +
['FAILED'] * failed)
return export_YAML(
CheckDetail(item=item,
report_type=itemtype,
outcome=final_outcome,
checkname=checkname,
note=note,
artifact=xunitfile))
else:
assert False, "This should never happen, aggregation is %r" % aggregation
def setup(self, tmpdir):
self.artifactsdir = tmpdir.mkdir('artifacts')
self.report_filename = 'report.html'
self.cd = check.CheckDetail(
item='foo_bar',
report_type=check.ReportType.KOJI_BUILD,
outcome='NEEDS_INSPECTION',
note='foo_bar note',
output=["foo\nbar"],
keyvals={
"foo": "moo1",
"bar": "moo2"
},
artifact='artifact.log',
)
self.yaml = check.export_YAML(self.cd)
self.ref_params = {
'results': self.yaml,
'artifactsdir': self.artifactsdir.strpath,
}
self.ref_arg_data = {
'resultsdb_job_id': 1,
'checkname': 'test_resultsdb_report',
'jobid': 'all/123',
'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701',
}
self.rd = create_report_directive.CreateReportDirective()
def test_custom_report_type(self):
'''Test that user can specify a custom CheckDetail.report_type'''
self.cd.report_type = 'My Custom Type'
yaml_output = check.export_YAML(self.cd)
yaml_obj = yaml.safe_load(yaml_output)['results'][0]
assert yaml_obj['type'] == self.cd.report_type
def test_nonempty_run(self, monkeypatch):
test_args = mock.MagicMock()
test_args.__dict__ = {
'file': 'somefile',
'report_type': 'beer',
'item': 'punkipa',
'keyval': [],
'outcome': 'PASSED'
}
stub_parser = mock.MagicMock()
stub_parser.parse_args = mock.MagicMock(return_value=test_args)
stub_get_argparser = mock.MagicMock(return_value=stub_parser)
monkeypatch.setattr(checkb_result, 'get_argparser', stub_get_argparser)
stub_file = mock.MagicMock()
stub_file.write = save_output
stub_file.read = lambda: FILE
monkeypatch.setattr(os.path, 'isfile',
mock.MagicMock(return_value=True))
with mock.patch(builtin_open,
mock.MagicMock(return_value=stub_file),
create=True):
checkb_result.main()
assert OUTPUT == export_YAML(
import_YAML(FILE) + [CheckDetail(**vars(test_args))])
def setup(self, monkeypatch):
'''Run this before every test invocation'''
self.cd = check.CheckDetail(
item='foo_bar',
report_type=check.ReportType.KOJI_BUILD,
outcome='NEEDS_INSPECTION',
note='foo_bar note',
output=["foo\nbar"],
keyvals={"foo": "moo1", "bar": "moo2"},
checkname='qa.test_resultsdb_report',
)
self.yaml = check.export_YAML(self.cd)
self.ref_input = {'results': self.yaml}
self.ref_arg_data = {
'resultsdb_job_id': 1,
'jobid': 'all/123',
'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701',
'artifactsdir': '/some/directory/',
'task': '/taskdir',
'item': 'firefox-45.0.2-1.fc23'
}
self.ref_resultdata = {u'id': 1234}
self.ref_jobid = 1234
self.ref_uuid = 'c25237a4-b6b3-11e4-b98a-3c970e018701'
self.ref_refurl = u'http://example.com/%s' % self.cd.checkname
self.ref_jobdata = {u'end_time': None,
u'href': u'http://127.0.0.1/api/v2.0/jobs/%d' % self.ref_jobid,
u'id': self.ref_jobid,
u'name': self.cd.checkname,
u'ref_url': self.ref_refurl,
u'results': [],
u'results_count': 0,
u'start_time': None,
u'status': u'SCHEDULED'}
self.stub_rdb = mock.Mock(**{
'get_testcase.return_value': {},
'create_job.return_value': self.ref_jobdata,
'create_result.return_value': self.ref_resultdata,
})
self.test_rdb = resultsdb_directive.ResultsdbDirective(self.stub_rdb)
# while it appears useless, this actually sets config in several tests
monkeypatch.setattr(config, '_config', None)
self.conf = config.get_config()
self.conf.report_to_resultsdb = True
monkeypatch.setattr(configparser, 'ConfigParser', StubConfigParser)
def test_report_artifact_in_log_url(self, monkeypatch):
"""Checks whether artifact is correctly mapped to log_url"""
cd = check.CheckDetail(item='foo_bar',
report_type=check.ReportType.KOJI_BUILD,
outcome='NEEDS_INSPECTION',
note='foo_bar note',
output=["foo\nbar"],
keyvals={
"foo": "moo1",
"bar": "moo2"
},
checkname='qa.test_resultsdb_report',
artifact='digest/logs/logfile.log')
monkeypatch.setattr(self.test_rdb, 'get_artifact_path',
lambda *x: 'digest/logs/logfile.log')
yaml = check.export_YAML(cd)
ref_input = {'results': yaml}
self.test_rdb.process(ref_input, self.ref_arg_data)
# Given the input data, the resultsdb should be called once, and only
# once, calling "create_result".
# This assert failing either means that more calls were added in the
# source code, or that a bug is present, and "create_result" is
# called multiple times.
# we expect rdb to be called 4 times: create job, update to RUNNING,
# check for testcase, report result and complete job
assert len(self.stub_rdb.mock_calls) == 2
# Select the first call of "create_result" method.
# This could be written as self.stub_rdb.calls()[0] at the moment, but
# this is more future-proof, and accidental addition of resultsdb
# calls is handled by the previous assert.
call = [
call for call in self.stub_rdb.mock_calls
if call[0] == 'create_result'
][0]
# Select the keyword arguments of that call
call_data = call[2]
# the log url depends on the arg_data, so construct it here
ref_log_url = '%s/%s/%s' %\
(self.conf.artifacts_baseurl, self.ref_arg_data['uuid'], cd.artifact)
assert call_data['ref_url'] == ref_log_url
def test_import_exported(self):
yaml_output = check.export_YAML(self.cd)
cds = check.import_YAML(yaml_output)
assert len(cds) == 1
assert isinstance(cds[0], check.CheckDetail)
cd = cds[0]
assert cd.item == self.cd.item
assert cd.outcome == self.cd.outcome
assert cd.note == self.cd.note
assert cd.report_type == self.cd.report_type
assert cd.keyvals == self.keyvals
assert cd.checkname == self.checkname
assert cd.artifact == self.artifact
assert cd._internal == self._internal
def test_minimal(self):
'''Test that empty CheckDetail values don't produce empty YAML lines,
for example 'note:' should not be present if there's no
CheckDetail.note'''
cd = check.CheckDetail('foo')
yaml_output = check.export_YAML(cd)
yaml_obj = yaml.safe_load(yaml_output)['results'][0]
# the output should look like this:
#
# results:
# - item: XXX
# outcome: XXX
#
# ('outcome:' is technically not a mandatory line, but with CheckDetail
# export we produce it every time)
assert len(yaml_obj) == 2
def test_multi(self):
'''Test export with multiple item sections'''
cd2 = check.CheckDetail(item='foobar-1.2-3.fc20',
outcome='FAILED',
note='dependency error',
report_type=check.ReportType.BODHI_UPDATE)
cd3 = check.CheckDetail(item='f20-updates',
outcome='INFO',
note='2 stale updates',
report_type=check.ReportType.KOJI_TAG)
yaml_output = check.export_YAML([self.cd, cd2, cd3])
yaml_obj = yaml.safe_load(yaml_output)
assert len(yaml_obj['results']) == 3
assert yaml_obj['results'][0]['item'] == self.cd.item
assert yaml_obj['results'][1]['item'] == cd2.item
assert yaml_obj['results'][2]['item'] == cd3.item
def test_config_reporting_disabled(self):
"""Checks config option that disables reporting."""
conf = config.get_config()
conf.report_to_resultsdb = False
yaml = self.test_rdb.process(self.ref_input, self.ref_arg_data)
cds = check.import_YAML(yaml)
my_cd = check.import_YAML(check.export_YAML(self.cd))
# return value should be the same YAML
assert len(cds) == 1
assert cds[0].__dict__ == my_cd[0].__dict__
# no call should have been made
assert len(self.stub_rdb.mock_calls) == 0
config._config = None
def test_invalid_keyvals(self):
'''Test export with keyvals containing reserved keys.'''
cd = deepcopy(self.cd)
for key in check.RESERVED_KEYS:
cd.keyvals[key] = 'foo'
yaml_output = check.export_YAML(cd)
yaml_obj = yaml.load(yaml_output, Loader=yaml.SafeLoader)['results'][0]
assert yaml_obj['item'] == self.item
assert yaml_obj['outcome'] == self.outcome
assert yaml_obj['note'] == self.note
assert yaml_obj['type'] == self.report_type
assert yaml_obj['foo'] == self.keyvals['foo']
assert yaml_obj['moo'] == self.keyvals['moo']
assert yaml_obj['checkname'] == self.checkname
assert yaml_obj['_internal'] == self._internal
def test_single_yaml(self):
'''Test export with a single item section.'''
yaml_output = check.export_YAML(self.cd)
yaml_obj = yaml.safe_load(yaml_output)
assert type(yaml_obj) is dict
assert type(yaml_obj['results']) is list
yaml_obj = yaml_obj['results'][0]
assert yaml_obj['item'] == self.item
assert yaml_obj['outcome'] == self.outcome
assert yaml_obj['note'] == self.note
assert yaml_obj['type'] == self.report_type
assert yaml_obj['foo'] == self.keyvals['foo']
assert yaml_obj['moo'] == self.keyvals['moo']
assert yaml_obj['checkname'] == self.checkname
assert yaml_obj['_internal'] == self._internal
def test_keyval_parse(self, monkeypatch):
test_args = mock.MagicMock()
test_args.__dict__ = {
'file': 'somefile',
'report_type': 'beer',
'item': 'punkipa',
'keyval': ['hop=Simcoe'],
'outcome': 'PASSED'
}
stub_parser = mock.MagicMock()
stub_parser.parse_args = mock.MagicMock(return_value=test_args)
stub_get_argparser = mock.MagicMock(return_value=stub_parser)
monkeypatch.setattr(checkb_result, 'get_argparser', stub_get_argparser)
stub_file = mock.MagicMock()
stub_file.write = save_output
with mock.patch(builtin_open,
mock.MagicMock(return_value=stub_file),
create=True):
checkb_result.main()
assert OUTPUT == export_YAML(CheckDetail(**vars(test_args)))
def process(self, params, arg_data):
# checking if reporting is enabled is done after importing yaml which
# serves as validation of input results
if 'file' in params and 'results' in params:
raise CheckbDirectiveError(
"Either `file` or `results` can be used, not both.")
try:
if params.get('file', None):
with open(params['file']) as resultfile:
params['results'] = resultfile.read()
check_details = check.import_YAML(params['results'])
log.debug("YAML output parsed OK.")
except (CheckbValueError, IOError) as e:
raise CheckbDirectiveError("Failed to load results: %s" % e)
for detail in check_details:
if not (detail.item and detail.report_type and detail.checkname):
raise CheckbDirectiveError(
"The resultsdb directive requires 'item', 'type' and "
"'checkname' to be present in the YAML data.")
conf = config.get_config()
if not conf.report_to_resultsdb:
log.info(
"Reporting to ResultsDB is disabled. Once enabled, the "
"following would get reported:\n%s", params['results'])
return check.export_YAML(check_details)
artifactsdir_url = '%s/%s' % (self.artifacts_baseurl, arg_data['uuid'])
# for now, we're creating the resultsdb group at reporting time
group_data = self.create_resultsdb_group(uuid=arg_data['uuid'])
log.info('Posting %s results to ResultsDB...' % len(check_details))
for detail in check_details:
checkname = detail.checkname
# find out if the task is allowed to post results into the namespace
if config.get_config().profile == config.ProfileName.PRODUCTION:
self.check_namespace(checkname, arg_data)
self.ensure_testcase_exists(checkname)
result_log_url = artifactsdir_url
if detail.artifact:
artifact_path = self.get_artifact_path(
arg_data['artifactsdir'], detail.artifact)
if artifact_path:
result_log_url = "%s/%s" % (artifactsdir_url,
artifact_path)
try:
result = self.resultsdb.create_result(outcome=detail.outcome,
testcase=checkname,
groups=[group_data],
note=detail.note or None,
ref_url=result_log_url,
item=detail.item,
type=detail.report_type,
**detail.keyvals)
log.debug('Result saved in ResultsDB:\n%s',
pprint.pformat(result))
detail._internal['resultsdb_result_id'] = result['id']
except resultsdb_api.ResultsDBapiException as e:
log.error(e)
log.error("Failed to store to ResultsDB: `%s` `%s` `%s`",
detail.item, checkname, detail.outcome)
return check.export_YAML(check_details)
def test_invalid_missing_item(self):
'''Test invalid input parameters'''
with pytest.raises(exc.CheckbValueError):
self.cd.item = None
check.export_YAML(self.cd)
def update_input(self):
self.yaml = check.export_YAML(self.cd)
self.ref_input = {'results': self.yaml}
