class SubmitCommandTest(unittest.TestCase): testing_server = 'http://localhost:%s' % settings.DEFAULT_SQUAD_PORT testing_token = '193cd8bb41ab9217714515954e8724f651ef8601' def setUp(self): self.squad = Squad() SquadApi.configure(url=self.testing_server, token=self.testing_token) def manage_submit(self, results=None, result_name=None, result_value=None, metrics=None, metadata=None, attachments=None, logs=None, environment=None): argv = [ './manage.py', '--squad-host', self.testing_server, '--squad-token', self.testing_token, 'submit', '--group', 'my_group', '--project', 'my_project', '--build', 'my_build6', '--environment', 'test_submit_env' ] if logs: argv += ['--logs', logs] if results: argv += ['--results', results] if metrics: argv += ['--metrics', metrics] if metadata: argv += ['--metadata', metadata] if attachments: argv += ['--attachments', attachments] if result_name: argv += ['--result-name', result_name] if result_value: argv += ['--result-value', result_value] proc = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE) proc.ok = False try: out, err = proc.communicate() proc.ok = (proc.returncode == 0) except sp.TimeoutExpired: self.logger.error('Running "%s" time out after %i seconds!' % ' '.join(argv)) proc.kill() out, err = proc.communicate() proc.out = out.decode('utf-8') proc.err = err.decode('utf-8') return proc def test_submit_empty(self): proc = self.manage_submit() self.assertFalse(proc.ok) self.assertIn( 'At least one of --result-name, --results, --metrics is required', proc.err) def test_submit_single_test(self): proc = self.manage_submit(result_name='single-test', result_value='pass') self.assertTrue(proc.ok) self.assertIn('1 tests', proc.err) test = first(self.squad.tests(name='single-test')) self.assertEqual('single-test', test.name) self.assertEqual('pass', test.status) def test_submit_invalid_result_value(self): proc = self.manage_submit(result_name='single-invalid-test', result_value='not-valid') self.assertFalse(proc.ok) self.assertIn("result-value: invalid choice: 'not-valid'", proc.err) def test_submit_results_json(self): proc = self.manage_submit( results='tests/submit_results/sample_results.json') self.assertTrue(proc.ok) self.assertIn('2 tests', proc.err) test = first(self.squad.tests(name='json-test-1')) self.assertEqual('json-test-1', test.name) self.assertEqual('pass', test.status) test = first(self.squad.tests(name='json-test-2')) self.assertEqual('json-test-2', test.name) self.assertEqual('fail', test.status) self.assertEqual('json-test-2 log', test.log) def test_submit_results_malformed_json(self): proc = self.manage_submit( results='tests/submit_results/sample_results_malformed.json') self.assertFalse(proc.ok) self.assertIn('Failed parsing file', proc.err) def test_submit_results_yaml(self): proc = self.manage_submit( results='tests/submit_results/sample_results.yaml') self.assertTrue(proc.ok) self.assertIn('2 tests', proc.err) test = first(self.squad.tests(name='yaml-test-1')) self.assertEqual('yaml-test-1', test.name) self.assertEqual('pass', test.status) test = first(self.squad.tests(name='yaml-test-2')) self.assertEqual('yaml-test-2', test.name) self.assertEqual('fail', test.status) self.assertEqual('yaml-test-2 log', test.log) def test_submit_results_malformed_yaml(self): proc = self.manage_submit( results='tests/submit_results/sample_results_malformed.yaml') self.assertFalse(proc.ok) self.assertIn('Failed parsing file', proc.err) def test_submit_single_metric(self): proc = self.manage_submit( metrics='tests/submit_results/sample_metrics.json') self.assertTrue(proc.ok) self.assertIn('1 metrics', proc.err) def test_submit_everything(self): proc = self.manage_submit( results='tests/submit_results/sample_results.json', metrics='tests/submit_results/sample_metrics.json', metadata='tests/submit_results/sample_metadata.json', logs='tests/submit_results/sample_log.log') self.assertTrue(proc.ok) self.assertIn('2 tests, 1 metrics', proc.err) testrun = first(self.squad.testruns(job_id='jsonmetadatajobid1')) self.assertEqual('jsonmetadatajobid1', testrun.job_id) self.assertEqual(2, len(testrun.tests())) self.assertEqual(1, len(testrun.metrics()))
class SquadTest(unittest.TestCase): def setUp(self): self.squad = Squad() def test_groups(self): groups = self.squad.groups() self.assertTrue(True, len(groups)) def test_not_found_groups(self): groups = self.squad.groups(name__startswith='no group with this name') self.assertEqual(0, len(groups)) def test_groups_with_count(self): all_groups = self.squad.groups(count=ALL) self.assertEqual(2, len(all_groups)) one_groups = self.squad.groups(count=1) self.assertEqual(1, len(one_groups)) def test_not_found_group(self): not_found_group = self.squad.group('this-group-does-not-really-exist') self.assertEqual(None, not_found_group) def test_group(self): group = self.squad.group('my_group') self.assertTrue(group is not None) def test_projects(self): projects = self.squad.projects() self.assertTrue(True, len(projects)) def test_builds(self): builds = self.squad.builds() self.assertTrue(True, len(builds)) def test_testjobs(self): testjobs = self.squad.testjobs() self.assertTrue(True, len(testjobs)) def test_testruns(self): testruns = self.squad.testruns() self.assertTrue(True, len(testruns)) def test_tests(self): tests = self.squad.tests() self.assertTrue(True, len(tests)) def test_suites(self): suites = self.squad.suites() self.assertTrue(True, len(suites)) def test_environments(self): environments = self.squad.environments() self.assertTrue(True, len(environments)) def test_backends(self): backends = self.squad.backends() self.assertTrue(True, len(backends)) def test_emailtemplates(self): emailtemplates = self.squad.emailtemplates() self.assertTrue(True, len(emailtemplates)) def test_knownissues(self): knownissues = self.squad.knownissues() self.assertTrue(True, len(knownissues)) def test_suitemetadata(self): suitemetadata = self.squad.suitemetadata() self.assertTrue(True, len(suitemetadata)) def test_annotations(self): annotations = self.squad.annotations() self.assertTrue(True, len(annotations)) def test_metricthresholds(self): metricthresholds = self.squad.metricthresholds() self.assertTrue(True, len(metricthresholds)) def test_reports(self): reports = self.squad.reports() self.assertTrue(True, len(reports))
class SubmitCommandTest(unittest.TestCase): testing_server = 'http://localhost:%s' % settings.DEFAULT_SQUAD_PORT testing_token = '193cd8bb41ab9217714515954e8724f651ef8601' def setUp(self): self.squad = Squad() SquadApi.configure(url=self.testing_server, token=self.testing_token) def manage_submit(self, results=None, results_layout=None, result_name=None, result_value=None, metrics=None, metadata=None, attachments=None, logs=None, environment=None): argv = [ './manage.py', '--squad-host', self.testing_server, '--squad-token', self.testing_token, 'submit', '--group', 'my_group', '--project', 'my_project', '--build', 'my_build6', '--environment', 'test_submit_env' ] if logs: argv += ['--logs', logs] if results: argv += ['--results', results] if results_layout: argv += ['--results-layout', results_layout] if metrics: argv += ['--metrics', metrics] if metadata: argv += ['--metadata', metadata] if attachments: argv += ['--attachments', attachments] if result_name: argv += ['--result-name', result_name] if result_value: argv += ['--result-value', result_value] proc = sp.Popen(argv, stdout=sp.PIPE, stderr=sp.PIPE) proc.ok = False try: out, err = proc.communicate() proc.ok = (proc.returncode == 0) except sp.TimeoutExpired: self.logger.error('Running "%s" time out after %i seconds!' % ' '.join(argv)) proc.kill() out, err = proc.communicate() proc.out = out.decode('utf-8') proc.err = err.decode('utf-8') return proc def test_submit_empty(self): proc = self.manage_submit() self.assertFalse(proc.ok) self.assertIn( 'At least one of --result-name, --results, --metrics is required', proc.err) def test_submit_single_test(self): proc = self.manage_submit(result_name='single-test', result_value='pass') self.assertTrue(proc.ok) self.assertIn('1 tests', proc.err) test = first(self.squad.tests(name='single-test')) self.assertEqual('single-test', test.name) self.assertEqual('pass', test.status) def test_submit_invalid_result_value(self): proc = self.manage_submit(result_name='single-invalid-test', result_value='not-valid') self.assertFalse(proc.ok) self.assertIn("result-value: invalid choice: 'not-valid'", proc.err) def test_submit_results_json(self): proc = self.manage_submit( results='tests/submit_results/sample_results.json') self.assertTrue(proc.ok) self.assertIn('2 tests', proc.err) test = first(self.squad.tests(name='json-test-1')) self.assertEqual('json-test-1', test.name) self.assertEqual('pass', test.status) test = first(self.squad.tests(name='json-test-2')) self.assertEqual('json-test-2', test.name) self.assertEqual('fail', test.status) self.assertEqual('json-test-2 log', test.log) def test_submit_results_malformed_json(self): proc = self.manage_submit( results='tests/submit_results/sample_results_malformed.json') self.assertFalse(proc.ok) self.assertIn('Failed parsing file', proc.err) def test_submit_results_tuxbuild_json(self): proc = self.manage_submit( results='tests/data/submit/tuxbuild/build.json', results_layout='tuxbuild_json') self.assertTrue(proc.ok, msg=proc.err) self.assertIn("Submitting 2 tests", proc.err) test = first(self.squad.tests(name="gcc-9-defconfig-b9979cfa")) self.assertEqual("build/gcc-9-defconfig-b9979cfa", test.name) self.assertEqual("pass", test.status) test = first(self.squad.tests(name="gcc-9-defconfig-5b09568e")) self.assertEqual("build/gcc-9-defconfig-5b09568e", test.name) self.assertEqual("fail", test.status) def test_submit_results_tuxbuild_buildset_json(self): proc = self.manage_submit( results='tests/data/submit/tuxbuild/buildset.json', results_layout='tuxbuild_json') self.assertIn("Submitting 3 tests", proc.err) test = first(self.squad.tests(name="gcc-8-allnoconfig")) self.assertEqual("build/gcc-8-allnoconfig", test.name) self.assertEqual("pass", test.status) test = first(self.squad.tests(name="gcc-8-tinyconfig")) self.assertEqual("build/gcc-8-tinyconfig", test.name) self.assertEqual("pass", test.status) test = first(self.squad.tests(name="gcc-8-x86_64_defconfig")) self.assertEqual("build/gcc-8-x86_64_defconfig", test.name) self.assertEqual("pass", test.status) def test_submit_results_tuxbuild_json_malformed(self): proc = self.manage_submit( results='tests/data/submit/tuxbuild/malformed.json', results_layout='tuxbuild_json') self.assertFalse(proc.ok, msg=proc.err) self.assertIn("Failed to load json", proc.err) def test_submit_results_tuxbuild_json_missing(self): proc = self.manage_submit( results="tests/data/submit/tuxbuild/missing.json", results_layout="tuxbuild_json") self.assertFalse(proc.ok) self.assertIn( "Requested file tests/data/submit/tuxbuild/missing.json doesn't exist", proc.err) def test_submit_results_tuxbuild_json_results_opt_missing(self): proc = self.manage_submit(results_layout="tuxbuild_json") self.assertFalse(proc.ok) self.assertIn( "At least one of --result-name, --results, --metrics is required", proc.err) def test_submit_results_tuxbuild_json_layout_arg_bad(self): proc = self.manage_submit( results="tests/data/submit/tuxbuild/build.json", results_layout="bad_layout") self.assertFalse(proc.ok) self.assertIn( "argument --results-layout: invalid choice: 'bad_layout'", proc.err) def test_submit_results_tuxbuild_json_empty_kconfig(self): proc = self.manage_submit( results="tests/data/submit/tuxbuild/empty_kconfig.json", results_layout="tuxbuild_json") self.assertFalse(proc.ok) self.assertIn( "Failed to load tuxbuild json due to a missing kconfig value: list index out of range", proc.err) def test_submit_results_tuxbuild_json_missing_kconfig(self): proc = self.manage_submit( results="tests/data/submit/tuxbuild/missing_kconfig.json", results_layout="tuxbuild_json") self.assertFalse(proc.ok) self.assertIn( "Failed to load tuxbuild json due to a missing variable: 'kconfig'", proc.err) def test_submit_results_yaml(self): proc = self.manage_submit( results='tests/submit_results/sample_results.yaml') self.assertTrue(proc.ok) self.assertIn('2 tests', proc.err) test = first(self.squad.tests(name='yaml-test-1')) self.assertEqual('yaml-test-1', test.name) self.assertEqual('pass', test.status) test = first(self.squad.tests(name='yaml-test-2')) self.assertEqual('yaml-test-2', test.name) self.assertEqual('fail', test.status) self.assertEqual('yaml-test-2 log', test.log) def test_submit_results_malformed_yaml(self): proc = self.manage_submit( results='tests/submit_results/sample_results_malformed.yaml') self.assertFalse(proc.ok) self.assertIn('Failed parsing file', proc.err) def test_submit_results_bad_extension(self): p = "tests/submit_results/sample_results.txt" proc = self.manage_submit(results=p) self.assertFalse(proc.ok) self.assertIn( 'File "%s" does not have a JSON or YAML file extension' % p, proc.err) def test_submit_single_metric(self): proc = self.manage_submit( metrics='tests/submit_results/sample_metrics.json') self.assertTrue(proc.ok) self.assertIn('1 metrics', proc.err) def test_submit_everything(self): proc = self.manage_submit( results='tests/submit_results/sample_results.json', metrics='tests/submit_results/sample_metrics.json', metadata='tests/submit_results/sample_metadata.json', logs='tests/submit_results/sample_log.log') self.assertTrue(proc.ok, msg=proc.err) self.assertIn('2 tests, 1 metrics', proc.err) testrun = first(self.squad.testruns(job_id='jsonmetadatajobid1')) self.assertEqual('jsonmetadatajobid1', testrun.job_id) self.assertEqual(2, len(testrun.tests())) self.assertEqual(1, len(testrun.metrics()))