class SubmitCommandTest(unittest.TestCase):
testing_server = 'http://localhost:%s' % settings.DEFAULT_SQUAD_PORT
testing_token = '193cd8bb41ab9217714515954e8724f651ef8601'
def setUp(self):
self.squad = Squad()
SquadApi.configure(url=self.testing_server, token=self.testing_token)
def manage_submit(self,
results=None,
result_name=None,
result_value=None,
metrics=None,
metadata=None,
attachments=None,
logs=None,
environment=None):
argv = [
'./manage.py', '--squad-host', self.testing_server,
'--squad-token', self.testing_token, 'submit', '--group',
'my_group', '--project', 'my_project', '--build', 'my_build6',
'--environment', 'test_submit_env'
]
if logs:
argv += ['--logs', logs]
if results:
argv += ['--results', results]
if metrics:
argv += ['--metrics', metrics]
if metadata:
argv += ['--metadata', metadata]
if attachments:
argv += ['--attachments', attachments]
if result_name:
argv += ['--result-name', result_name]
if result_value:
argv += ['--result-value', result_value]
proc = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
proc.ok = False
try:
out, err = proc.communicate()
proc.ok = (proc.returncode == 0)
except sp.TimeoutExpired:
self.logger.error('Running "%s" time out after %i seconds!' %
' '.join(argv))
proc.kill()
out, err = proc.communicate()
proc.out = out.decode('utf-8')
proc.err = err.decode('utf-8')
return proc
def test_submit_empty(self):
proc = self.manage_submit()
self.assertFalse(proc.ok)
self.assertIn(
'At least one of --result-name, --results, --metrics is required',
proc.err)
def test_submit_single_test(self):
proc = self.manage_submit(result_name='single-test',
result_value='pass')
self.assertTrue(proc.ok)
self.assertIn('1 tests', proc.err)
test = first(self.squad.tests(name='single-test'))
self.assertEqual('single-test', test.name)
self.assertEqual('pass', test.status)
def test_submit_invalid_result_value(self):
proc = self.manage_submit(result_name='single-invalid-test',
result_value='not-valid')
self.assertFalse(proc.ok)
self.assertIn("result-value: invalid choice: 'not-valid'", proc.err)
def test_submit_results_json(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results.json')
self.assertTrue(proc.ok)
self.assertIn('2 tests', proc.err)
test = first(self.squad.tests(name='json-test-1'))
self.assertEqual('json-test-1', test.name)
self.assertEqual('pass', test.status)
test = first(self.squad.tests(name='json-test-2'))
self.assertEqual('json-test-2', test.name)
self.assertEqual('fail', test.status)
self.assertEqual('json-test-2 log', test.log)
def test_submit_results_malformed_json(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results_malformed.json')
self.assertFalse(proc.ok)
self.assertIn('Failed parsing file', proc.err)
def test_submit_results_yaml(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results.yaml')
self.assertTrue(proc.ok)
self.assertIn('2 tests', proc.err)
test = first(self.squad.tests(name='yaml-test-1'))
self.assertEqual('yaml-test-1', test.name)
self.assertEqual('pass', test.status)
test = first(self.squad.tests(name='yaml-test-2'))
self.assertEqual('yaml-test-2', test.name)
self.assertEqual('fail', test.status)
self.assertEqual('yaml-test-2 log', test.log)
def test_submit_results_malformed_yaml(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results_malformed.yaml')
self.assertFalse(proc.ok)
self.assertIn('Failed parsing file', proc.err)
def test_submit_single_metric(self):
proc = self.manage_submit(
metrics='tests/submit_results/sample_metrics.json')
self.assertTrue(proc.ok)
self.assertIn('1 metrics', proc.err)
def test_submit_everything(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results.json',
metrics='tests/submit_results/sample_metrics.json',
metadata='tests/submit_results/sample_metadata.json',
logs='tests/submit_results/sample_log.log')
self.assertTrue(proc.ok)
self.assertIn('2 tests, 1 metrics', proc.err)
testrun = first(self.squad.testruns(job_id='jsonmetadatajobid1'))
self.assertEqual('jsonmetadatajobid1', testrun.job_id)
self.assertEqual(2, len(testrun.tests()))
self.assertEqual(1, len(testrun.metrics()))
class SquadTest(unittest.TestCase):
def setUp(self):
self.squad = Squad()
def test_groups(self):
groups = self.squad.groups()
self.assertTrue(True, len(groups))
def test_not_found_groups(self):
groups = self.squad.groups(name__startswith='no group with this name')
self.assertEqual(0, len(groups))
def test_groups_with_count(self):
all_groups = self.squad.groups(count=ALL)
self.assertEqual(2, len(all_groups))
one_groups = self.squad.groups(count=1)
self.assertEqual(1, len(one_groups))
def test_not_found_group(self):
not_found_group = self.squad.group('this-group-does-not-really-exist')
self.assertEqual(None, not_found_group)
def test_group(self):
group = self.squad.group('my_group')
self.assertTrue(group is not None)
def test_projects(self):
projects = self.squad.projects()
self.assertTrue(True, len(projects))
def test_builds(self):
builds = self.squad.builds()
self.assertTrue(True, len(builds))
def test_testjobs(self):
testjobs = self.squad.testjobs()
self.assertTrue(True, len(testjobs))
def test_testruns(self):
testruns = self.squad.testruns()
self.assertTrue(True, len(testruns))
def test_tests(self):
tests = self.squad.tests()
self.assertTrue(True, len(tests))
def test_suites(self):
suites = self.squad.suites()
self.assertTrue(True, len(suites))
def test_environments(self):
environments = self.squad.environments()
self.assertTrue(True, len(environments))
def test_backends(self):
backends = self.squad.backends()
self.assertTrue(True, len(backends))
def test_emailtemplates(self):
emailtemplates = self.squad.emailtemplates()
self.assertTrue(True, len(emailtemplates))
def test_knownissues(self):
knownissues = self.squad.knownissues()
self.assertTrue(True, len(knownissues))
def test_suitemetadata(self):
suitemetadata = self.squad.suitemetadata()
self.assertTrue(True, len(suitemetadata))
def test_annotations(self):
annotations = self.squad.annotations()
self.assertTrue(True, len(annotations))
def test_metricthresholds(self):
metricthresholds = self.squad.metricthresholds()
self.assertTrue(True, len(metricthresholds))
def test_reports(self):
reports = self.squad.reports()
self.assertTrue(True, len(reports))
class SubmitCommandTest(unittest.TestCase):
testing_server = 'http://localhost:%s' % settings.DEFAULT_SQUAD_PORT
testing_token = '193cd8bb41ab9217714515954e8724f651ef8601'
def setUp(self):
self.squad = Squad()
SquadApi.configure(url=self.testing_server, token=self.testing_token)
def manage_submit(self,
results=None,
results_layout=None,
result_name=None,
result_value=None,
metrics=None,
metadata=None,
attachments=None,
logs=None,
environment=None):
argv = [
'./manage.py', '--squad-host', self.testing_server,
'--squad-token', self.testing_token, 'submit', '--group',
'my_group', '--project', 'my_project', '--build', 'my_build6',
'--environment', 'test_submit_env'
]
if logs:
argv += ['--logs', logs]
if results:
argv += ['--results', results]
if results_layout:
argv += ['--results-layout', results_layout]
if metrics:
argv += ['--metrics', metrics]
if metadata:
argv += ['--metadata', metadata]
if attachments:
argv += ['--attachments', attachments]
if result_name:
argv += ['--result-name', result_name]
if result_value:
argv += ['--result-value', result_value]
proc = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE)
proc.ok = False
try:
out, err = proc.communicate()
proc.ok = (proc.returncode == 0)
except sp.TimeoutExpired:
self.logger.error('Running "%s" time out after %i seconds!' %
' '.join(argv))
proc.kill()
out, err = proc.communicate()
proc.out = out.decode('utf-8')
proc.err = err.decode('utf-8')
return proc
def test_submit_empty(self):
proc = self.manage_submit()
self.assertFalse(proc.ok)
self.assertIn(
'At least one of --result-name, --results, --metrics is required',
proc.err)
def test_submit_single_test(self):
proc = self.manage_submit(result_name='single-test',
result_value='pass')
self.assertTrue(proc.ok)
self.assertIn('1 tests', proc.err)
test = first(self.squad.tests(name='single-test'))
self.assertEqual('single-test', test.name)
self.assertEqual('pass', test.status)
def test_submit_invalid_result_value(self):
proc = self.manage_submit(result_name='single-invalid-test',
result_value='not-valid')
self.assertFalse(proc.ok)
self.assertIn("result-value: invalid choice: 'not-valid'", proc.err)
def test_submit_results_json(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results.json')
self.assertTrue(proc.ok)
self.assertIn('2 tests', proc.err)
test = first(self.squad.tests(name='json-test-1'))
self.assertEqual('json-test-1', test.name)
self.assertEqual('pass', test.status)
test = first(self.squad.tests(name='json-test-2'))
self.assertEqual('json-test-2', test.name)
self.assertEqual('fail', test.status)
self.assertEqual('json-test-2 log', test.log)
def test_submit_results_malformed_json(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results_malformed.json')
self.assertFalse(proc.ok)
self.assertIn('Failed parsing file', proc.err)
def test_submit_results_tuxbuild_json(self):
proc = self.manage_submit(
results='tests/data/submit/tuxbuild/build.json',
results_layout='tuxbuild_json')
self.assertTrue(proc.ok, msg=proc.err)
self.assertIn("Submitting 2 tests", proc.err)
test = first(self.squad.tests(name="gcc-9-defconfig-b9979cfa"))
self.assertEqual("build/gcc-9-defconfig-b9979cfa", test.name)
self.assertEqual("pass", test.status)
test = first(self.squad.tests(name="gcc-9-defconfig-5b09568e"))
self.assertEqual("build/gcc-9-defconfig-5b09568e", test.name)
self.assertEqual("fail", test.status)
def test_submit_results_tuxbuild_buildset_json(self):
proc = self.manage_submit(
results='tests/data/submit/tuxbuild/buildset.json',
results_layout='tuxbuild_json')
self.assertIn("Submitting 3 tests", proc.err)
test = first(self.squad.tests(name="gcc-8-allnoconfig"))
self.assertEqual("build/gcc-8-allnoconfig", test.name)
self.assertEqual("pass", test.status)
test = first(self.squad.tests(name="gcc-8-tinyconfig"))
self.assertEqual("build/gcc-8-tinyconfig", test.name)
self.assertEqual("pass", test.status)
test = first(self.squad.tests(name="gcc-8-x86_64_defconfig"))
self.assertEqual("build/gcc-8-x86_64_defconfig", test.name)
self.assertEqual("pass", test.status)
def test_submit_results_tuxbuild_json_malformed(self):
proc = self.manage_submit(
results='tests/data/submit/tuxbuild/malformed.json',
results_layout='tuxbuild_json')
self.assertFalse(proc.ok, msg=proc.err)
self.assertIn("Failed to load json", proc.err)
def test_submit_results_tuxbuild_json_missing(self):
proc = self.manage_submit(
results="tests/data/submit/tuxbuild/missing.json",
results_layout="tuxbuild_json")
self.assertFalse(proc.ok)
self.assertIn(
"Requested file tests/data/submit/tuxbuild/missing.json doesn't exist",
proc.err)
def test_submit_results_tuxbuild_json_results_opt_missing(self):
proc = self.manage_submit(results_layout="tuxbuild_json")
self.assertFalse(proc.ok)
self.assertIn(
"At least one of --result-name, --results, --metrics is required",
proc.err)
def test_submit_results_tuxbuild_json_layout_arg_bad(self):
proc = self.manage_submit(
results="tests/data/submit/tuxbuild/build.json",
results_layout="bad_layout")
self.assertFalse(proc.ok)
self.assertIn(
"argument --results-layout: invalid choice: 'bad_layout'",
proc.err)
def test_submit_results_tuxbuild_json_empty_kconfig(self):
proc = self.manage_submit(
results="tests/data/submit/tuxbuild/empty_kconfig.json",
results_layout="tuxbuild_json")
self.assertFalse(proc.ok)
self.assertIn(
"Failed to load tuxbuild json due to a missing kconfig value: list index out of range",
proc.err)
def test_submit_results_tuxbuild_json_missing_kconfig(self):
proc = self.manage_submit(
results="tests/data/submit/tuxbuild/missing_kconfig.json",
results_layout="tuxbuild_json")
self.assertFalse(proc.ok)
self.assertIn(
"Failed to load tuxbuild json due to a missing variable: 'kconfig'",
proc.err)
def test_submit_results_yaml(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results.yaml')
self.assertTrue(proc.ok)
self.assertIn('2 tests', proc.err)
test = first(self.squad.tests(name='yaml-test-1'))
self.assertEqual('yaml-test-1', test.name)
self.assertEqual('pass', test.status)
test = first(self.squad.tests(name='yaml-test-2'))
self.assertEqual('yaml-test-2', test.name)
self.assertEqual('fail', test.status)
self.assertEqual('yaml-test-2 log', test.log)
def test_submit_results_malformed_yaml(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results_malformed.yaml')
self.assertFalse(proc.ok)
self.assertIn('Failed parsing file', proc.err)
def test_submit_results_bad_extension(self):
p = "tests/submit_results/sample_results.txt"
proc = self.manage_submit(results=p)
self.assertFalse(proc.ok)
self.assertIn(
'File "%s" does not have a JSON or YAML file extension' % p,
proc.err)
def test_submit_single_metric(self):
proc = self.manage_submit(
metrics='tests/submit_results/sample_metrics.json')
self.assertTrue(proc.ok)
self.assertIn('1 metrics', proc.err)
def test_submit_everything(self):
proc = self.manage_submit(
results='tests/submit_results/sample_results.json',
metrics='tests/submit_results/sample_metrics.json',
metadata='tests/submit_results/sample_metadata.json',
logs='tests/submit_results/sample_log.log')
self.assertTrue(proc.ok, msg=proc.err)
self.assertIn('2 tests, 1 metrics', proc.err)
testrun = first(self.squad.testruns(job_id='jsonmetadatajobid1'))
self.assertEqual('jsonmetadatajobid1', testrun.job_id)
self.assertEqual(2, len(testrun.tests()))
self.assertEqual(1, len(testrun.metrics()))
