def process(self, fp, artifact): test_list = self.get_tests(fp) manager = TestResultManager(self.step, artifact) manager.save(test_list) return test_list
def process(self, fp, artifact): target_name = self._get_target_name(artifact) target, _ = get_or_create(BazelTarget, where={ 'step_id': self.step.id, 'job_id': self.step.job.id, 'name': target_name, 'result_source': ResultSource.from_self, }) test_suites = self.get_test_suites(fp) tests = self.aggregate_tests_from_suites(test_suites) manager = TestResultManager(self.step, artifact) manager.save(tests) # update target metadata # TODO handle multiple files per target, i.e. sharding and running multiple times target.status = Status.finished target.result = aggregate_result([t.result for t in tests]) duration = 0 for t in test_suites: if t.duration is None: duration = None break duration += t.duration target.duration = duration target.date_created = min([t.date_created for t in test_suites]) db.session.add(target) db.session.commit() return tests
def test_bad_duration(self): from changes.models.test import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) artifact = self.create_artifact(jobstep, 'junit.xml') results = [ TestResult(step=jobstep, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=2147483647 * 2, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('sample content') }]), ] manager = TestResultManager(jobstep, artifact) with mock.patch.object(logger, 'warning') as warn: manager.save(results) assert warn.called testcase_list = TestCase.query.all() assert len(testcase_list) == 1 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[ 0].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[0].result == Result.failed assert testcase_list[0].message == 'collection failed' assert testcase_list[0].duration == 0
def test_bad_duration(self): from changes.models.test import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) artifact = self.create_artifact(jobstep, 'junit.xml') results = [ TestResult( step=jobstep, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=2147483647 * 2, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('sample content')}]), ] manager = TestResultManager(jobstep, artifact) with mock.patch.object(logger, 'warning') as warn: manager.save(results) assert warn.called testcase_list = TestCase.query.all() assert len(testcase_list) == 1 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[0].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[0].result == Result.failed assert testcase_list[0].message == 'collection failed' assert testcase_list[0].duration == 0
def test_simple(self): from changes.models import TestCase, TestGroup build = self.create_build(self.project) job = self.create_job(build) suite = TestSuite(name='foobar', job=job, project=self.project) db.session.add(suite) results = [ TestResult( job=job, suite=suite, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=156, ), TestResult( job=job, suite=suite, name='test_foo', package='tests.changes.handlers.test_coverage', result=Result.passed, message='foobar failed', duration=12, ), ] manager = TestResultManager(job) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.package) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.project_id == self.project.id assert test.suite_id == suite.id assert testcase_list[0].name == 'test_foo' assert testcase_list[0].package == 'tests.changes.handlers.test_coverage' assert testcase_list[0].result == Result.passed assert testcase_list[0].message == 'foobar failed' assert testcase_list[0].duration == 12 assert len(testcase_list[0].groups) == 1 assert testcase_list[1].name == 'test_bar' assert testcase_list[1].package == 'tests.changes.handlers.test_xunit' assert testcase_list[1].result == Result.failed assert testcase_list[1].message == 'collection failed' assert testcase_list[1].duration == 156 group_list = sorted(TestGroup.query.all(), key=lambda x: x.name) assert len(group_list) == 4 for group in group_list: assert group.job_id == job.id assert group.project_id == self.project.id assert group.suite_id == suite.id assert group_list[0].name == 'tests.changes.handlers.test_coverage' assert group_list[0].duration == 12 assert group_list[0].num_tests == 1 assert group_list[0].num_failed == 0 assert group_list[0].result == Result.passed assert group_list[0].num_leaves == 1 assert list(group_list[0].testcases) == [] assert group_list[1].name == 'tests.changes.handlers.test_coverage.test_foo' assert group_list[1].duration == 12 assert group_list[1].num_tests == 1 assert group_list[1].num_failed == 0 assert group_list[1].result == Result.passed assert group_list[1].num_leaves == 0 assert list(group_list[1].testcases) == [testcase_list[0]] assert group_list[2].name == 'tests.changes.handlers.test_xunit' assert group_list[2].duration == 156 assert group_list[2].num_tests == 1 assert group_list[2].num_failed == 1 assert group_list[2].result == Result.failed assert group_list[2].num_leaves == 1 assert list(group_list[2].testcases) == [] assert group_list[3].name == 'tests.changes.handlers.test_xunit.test_bar' assert group_list[3].duration == 156 assert group_list[3].num_tests == 1 assert group_list[3].num_failed == 1 assert group_list[3].result == Result.failed assert group_list[3].num_leaves == 0 assert list(group_list[3].testcases) == [testcase_list[1]]
def test_simple(self): from changes.models import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) results = [ TestResult(step=jobstep, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=156, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('sample content') }]), TestResult( step=jobstep, name='test_foo', package='tests.changes.handlers.test_coverage', result=Result.passed, message='foobar failed', duration=12, reruns=1, ), ] manager = TestResultManager(jobstep) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[ 0].name == 'tests.changes.handlers.test_coverage.test_foo' assert testcase_list[0].result == Result.passed assert testcase_list[0].message == 'foobar failed' assert testcase_list[0].duration == 12 assert testcase_list[0].reruns == 1 assert testcase_list[ 1].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[1].result == Result.failed assert testcase_list[1].message == 'collection failed' assert testcase_list[1].duration == 156 assert testcase_list[1].reruns is 0 testartifacts = testcase_list[1].artifacts assert len(testartifacts) == 1 assert testartifacts[0].file.get_file().read() == 'sample content' teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_failures', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 1
def test_simple(self): from changes.models import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) results = [ TestResult( step=jobstep, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=156, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('sample content')}]), TestResult( step=jobstep, name='test_foo', package='tests.changes.handlers.test_coverage', result=Result.passed, message='foobar failed', duration=12, reruns=1, ), ] manager = TestResultManager(jobstep) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[0].name == 'tests.changes.handlers.test_coverage.test_foo' assert testcase_list[0].result == Result.passed assert testcase_list[0].message == 'foobar failed' assert testcase_list[0].duration == 12 assert testcase_list[0].reruns == 1 assert testcase_list[1].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[1].result == Result.failed assert testcase_list[1].message == 'collection failed' assert testcase_list[1].duration == 156 assert testcase_list[1].reruns is 0 testartifacts = testcase_list[1].artifacts assert len(testartifacts) == 1 assert testartifacts[0].file.get_file().read() == 'sample content' teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_failures', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 1
def test_duplicate_tests_in_different_result_lists(self): from changes.models.test import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase, label='STEP1') artifact = self.create_artifact(jobstep, 'junit.xml') results = [ TestResult( step=jobstep, name='test_foo', package='project.tests', result=Result.passed, duration=12, reruns=0, artifacts=[{ 'name': 'one_artifact', 'type': 'text', 'base64': b64encode('first artifact')}] ), TestResult( step=jobstep, name='test_bar', package='project.tests', result=Result.passed, duration=13, reruns=0, ), ] manager = TestResultManager(jobstep, artifact) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[0].name == 'project.tests.test_bar' assert testcase_list[0].result == Result.passed assert testcase_list[0].message is None assert testcase_list[0].duration == 13 assert testcase_list[0].reruns == 0 assert len(testcase_list[0].artifacts) == 0 assert testcase_list[1].name == 'project.tests.test_foo' assert testcase_list[1].result == Result.passed assert testcase_list[1].message is None assert testcase_list[1].duration == 12 assert testcase_list[1].reruns == 0 testartifacts = testcase_list[1].artifacts assert len(testartifacts) == 1 a1 = testartifacts[0].file.get_file().read() assert a1 == 'first artifact' assert _stat(jobstep, 'test_count') == 2 assert _stat(jobstep, 'test_failures') == 0 assert _stat(jobstep, 'test_duration') == 25 assert _stat(jobstep, 'test_rerun_count') == 0 jobstep2 = self.create_jobstep(jobphase, label='STEP2') artifact2 = self.create_artifact(jobstep2, 'junit.xml') results = [ TestResult( step=jobstep2, name='test_foo', package='project.tests', result=Result.passed, duration=11, reruns=0, artifacts=[{ 'name': 'another_artifact', 'type': 'text', 'base64': b64encode('second artifact')}] ), TestResult( step=jobstep2, name='test_baz', package='project.tests', result=Result.passed, duration=18, reruns=2, ), ] manager = TestResultManager(jobstep2, artifact2) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 3 for test in testcase_list: assert test.job_id == job.id assert test.project_id == project.id assert testcase_list[0].step_id == jobstep.id assert testcase_list[0].name == 'project.tests.test_bar' assert testcase_list[0].result == Result.passed assert testcase_list[0].message is None assert testcase_list[0].duration == 13 assert testcase_list[0].reruns == 0 assert testcase_list[1].step_id == jobstep2.id assert testcase_list[1].name == 'project.tests.test_baz' assert testcase_list[1].result == Result.passed assert testcase_list[1].message is None assert testcase_list[1].duration == 18 assert testcase_list[1].reruns == 2 assert testcase_list[2].step_id == jobstep.id assert testcase_list[2].name == 'project.tests.test_foo' assert testcase_list[2].result == Result.failed assert testcase_list[2].message.startswith('Error: Duplicate Test') assert testcase_list[2].message.endswith('\nSTEP1\nSTEP2\n') assert testcase_list[2].duration == 12 assert testcase_list[2].reruns == 0 testartifacts = testcase_list[2].artifacts assert len(testartifacts) == 2 a1 = testartifacts[0].file.get_file().read() a2 = testartifacts[1].file.get_file().read() assert {a1, a2} == {'first artifact', 'second artifact'} # Stats for original step are unharmed: assert _stat(jobstep, 'test_count') == 2 assert _stat(jobstep, 'test_failures') == 1 assert _stat(jobstep, 'test_duration') == 25 assert _stat(jobstep, 'test_rerun_count') == 0 # Stats for new step: assert _stat(jobstep2, 'test_count') == 1 assert _stat(jobstep2, 'test_failures') == 0 assert _stat(jobstep2, 'test_duration') == 18 assert _stat(jobstep2, 'test_rerun_count') == 1 failures = FailureReason.query.filter_by(step_id=jobstep.id).all() assert len(failures) == 0 failures = FailureReason.query.filter_by(step_id=jobstep2.id).all() assert len(failures) == 1 assert failures[0].reason == 'duplicate_test_name'
def test_duplicate_tests_in_same_result_list(self): from changes.models.test import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase, label='STEP1') artifact = self.create_artifact(jobstep, 'junit.xml') results = [ TestResult( step=jobstep, name='test_foo', package='project.tests', result=Result.passed, duration=12, reruns=0, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('first artifact')}], message_offsets=[('system-out', 123, 10)], ), TestResult( step=jobstep, name='test_bar', package='project.tests', result=Result.passed, duration=13, reruns=0, ), TestResult( step=jobstep, name='test_foo', package='project.tests', result=Result.passed, duration=11, reruns=0, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('second artifact')}], message_offsets=[('system-err', 555, 25)], ), ] manager = TestResultManager(jobstep, artifact) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[0].name == 'project.tests.test_bar' assert testcase_list[0].result == Result.passed assert testcase_list[0].message is None assert testcase_list[0].duration == 13 assert testcase_list[0].reruns == 0 assert len(testcase_list[0].artifacts) == 0 assert len(testcase_list[0].messages) == 0 assert testcase_list[1].name == 'project.tests.test_foo' assert testcase_list[1].result == Result.failed assert testcase_list[1].message.startswith('Error: Duplicate Test') assert testcase_list[1].message.endswith('\nSTEP1\n') assert testcase_list[1].duration == 12 assert testcase_list[1].reruns == 0 testartifacts = testcase_list[1].artifacts assert len(testartifacts) == 2 a1 = testartifacts[0].file.get_file().read() a2 = testartifacts[1].file.get_file().read() assert {a1, a2} == {'first artifact', 'second artifact'} testmessages = testcase_list[1].messages assert len(testmessages) == 2 assert testmessages[0].artifact == artifact assert testmessages[0].label == 'system-out' assert testmessages[0].start_offset == 123 assert testmessages[0].length == 10 assert testmessages[1].artifact == artifact assert testmessages[1].label == 'system-err' assert testmessages[1].start_offset == 555 assert testmessages[1].length == 25 assert _stat(jobstep, 'test_count') == 2 assert _stat(jobstep, 'test_failures') == 1 assert _stat(jobstep, 'test_duration') == 25 assert _stat(jobstep, 'test_rerun_count') == 0 failures = FailureReason.query.filter_by(step_id=jobstep.id).all() assert len(failures) == 1 assert failures[0].reason == 'duplicate_test_name'
def test_simple(self): from changes.models import TestCase build = self.create_build(self.project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) suite = TestSuite(name='foobar', job=job, project=self.project) db.session.add(suite) db.session.commit() results = [ TestResult( step=jobstep, suite=suite, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=156, ), TestResult( step=jobstep, suite=suite, name='test_foo', package='tests.changes.handlers.test_coverage', result=Result.passed, message='foobar failed', duration=12, reruns=1, ), ] manager = TestResultManager(jobstep) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == self.project.id assert test.suite_id == suite.id assert testcase_list[0].name == 'tests.changes.handlers.test_coverage.test_foo' assert testcase_list[0].result == Result.passed assert testcase_list[0].message == 'foobar failed' assert testcase_list[0].duration == 12 assert testcase_list[0].reruns == 1 assert testcase_list[1].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[1].result == Result.failed assert testcase_list[1].message == 'collection failed' assert testcase_list[1].duration == 156 assert testcase_list[1].reruns is 0 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == job.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == build.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == job.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == job.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 1 job2 = self.create_job(build) jobphase2 = self.create_jobphase(job2) jobstep2 = self.create_jobstep(jobphase2) results2 = [ TestResult( step=jobstep2, name='test_bar', package='tests.changes.handlers.test_bar', result=Result.failed, message='collection failed', duration=156, reruns=2, ), ] manager = TestResultManager(jobstep2) manager.save(results2) teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 3 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == job2.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == build.id, )[0] assert teststat.value == 324 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == job2.id, )[0] assert teststat.value == 156 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == job2.id, )[0] assert teststat.value == 1
def test_duplicate_tests_in_different_result_lists(self): from changes.models.test import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase, label='STEP1') artifact = self.create_artifact(jobstep, 'junit.xml') results = [ TestResult(step=jobstep, name='test_foo', package='project.tests', result=Result.passed, duration=12, reruns=0, artifacts=[{ 'name': 'one_artifact', 'type': 'text', 'base64': b64encode('first artifact') }]), TestResult( step=jobstep, name='test_bar', package='project.tests', result=Result.passed, duration=13, reruns=0, ), ] manager = TestResultManager(jobstep, artifact) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[0].name == 'project.tests.test_bar' assert testcase_list[0].result == Result.passed assert testcase_list[0].message is None assert testcase_list[0].duration == 13 assert testcase_list[0].reruns == 0 assert len(testcase_list[0].artifacts) == 0 assert testcase_list[1].name == 'project.tests.test_foo' assert testcase_list[1].result == Result.passed assert testcase_list[1].message is None assert testcase_list[1].duration == 12 assert testcase_list[1].reruns == 0 testartifacts = testcase_list[1].artifacts assert len(testartifacts) == 1 a1 = testartifacts[0].file.get_file().read() assert a1 == 'first artifact' assert _stat(jobstep, 'test_count') == 2 assert _stat(jobstep, 'test_failures') == 0 assert _stat(jobstep, 'test_duration') == 25 assert _stat(jobstep, 'test_rerun_count') == 0 jobstep2 = self.create_jobstep(jobphase, label='STEP2') artifact2 = self.create_artifact(jobstep2, 'junit.xml') results = [ TestResult(step=jobstep2, name='test_foo', package='project.tests', result=Result.passed, duration=11, reruns=0, artifacts=[{ 'name': 'another_artifact', 'type': 'text', 'base64': b64encode('second artifact') }]), TestResult( step=jobstep2, name='test_baz', package='project.tests', result=Result.passed, duration=18, reruns=2, ), ] manager = TestResultManager(jobstep2, artifact2) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 3 for test in testcase_list: assert test.job_id == job.id assert test.project_id == project.id assert testcase_list[0].step_id == jobstep.id assert testcase_list[0].name == 'project.tests.test_bar' assert testcase_list[0].result == Result.passed assert testcase_list[0].message is None assert testcase_list[0].duration == 13 assert testcase_list[0].reruns == 0 assert testcase_list[1].step_id == jobstep2.id assert testcase_list[1].name == 'project.tests.test_baz' assert testcase_list[1].result == Result.passed assert testcase_list[1].message is None assert testcase_list[1].duration == 18 assert testcase_list[1].reruns == 2 assert testcase_list[2].step_id == jobstep.id assert testcase_list[2].name == 'project.tests.test_foo' assert testcase_list[2].result == Result.failed assert testcase_list[2].message.startswith('Error: Duplicate Test') assert testcase_list[2].message.endswith('\nSTEP1\nSTEP2\n') assert testcase_list[2].duration == 12 assert testcase_list[2].reruns == 0 testartifacts = testcase_list[2].artifacts assert len(testartifacts) == 2 a1 = testartifacts[0].file.get_file().read() a2 = testartifacts[1].file.get_file().read() assert {a1, a2} == {'first artifact', 'second artifact'} # Stats for original step are unharmed: assert _stat(jobstep, 'test_count') == 2 assert _stat(jobstep, 'test_failures') == 1 assert _stat(jobstep, 'test_duration') == 25 assert _stat(jobstep, 'test_rerun_count') == 0 # Stats for new step: assert _stat(jobstep2, 'test_count') == 1 assert _stat(jobstep2, 'test_failures') == 0 assert _stat(jobstep2, 'test_duration') == 18 assert _stat(jobstep2, 'test_rerun_count') == 1 failures = FailureReason.query.filter_by(step_id=jobstep.id).all() assert len(failures) == 0 failures = FailureReason.query.filter_by(step_id=jobstep2.id).all() assert len(failures) == 1 assert failures[0].reason == 'duplicate_test_name'
def test_duplicate_tests_in_same_result_list(self): from changes.models.test import TestCase project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase, label='STEP1') artifact = self.create_artifact(jobstep, 'junit.xml') results = [ TestResult( step=jobstep, name='test_foo', package='project.tests', result=Result.passed, duration=12, reruns=0, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('first artifact') }], message_offsets=[('system-out', 123, 10)], ), TestResult( step=jobstep, name='test_bar', package='project.tests', result=Result.passed, duration=13, reruns=0, ), TestResult( step=jobstep, name='test_foo', package='project.tests', result=Result.passed, duration=11, reruns=0, artifacts=[{ 'name': 'artifact_name', 'type': 'text', 'base64': b64encode('second artifact') }], message_offsets=[('system-err', 555, 25)], ), ] manager = TestResultManager(jobstep, artifact) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == project.id assert testcase_list[0].name == 'project.tests.test_bar' assert testcase_list[0].result == Result.passed assert testcase_list[0].message is None assert testcase_list[0].duration == 13 assert testcase_list[0].reruns == 0 assert len(testcase_list[0].artifacts) == 0 assert len(testcase_list[0].messages) == 0 assert testcase_list[1].name == 'project.tests.test_foo' assert testcase_list[1].result == Result.failed assert testcase_list[1].message.startswith('Error: Duplicate Test') assert testcase_list[1].message.endswith('\nSTEP1\n') assert testcase_list[1].duration == 12 assert testcase_list[1].reruns == 0 testartifacts = testcase_list[1].artifacts assert len(testartifacts) == 2 a1 = testartifacts[0].file.get_file().read() a2 = testartifacts[1].file.get_file().read() assert {a1, a2} == {'first artifact', 'second artifact'} testmessages = testcase_list[1].messages assert len(testmessages) == 2 assert testmessages[0].artifact == artifact assert testmessages[0].label == 'system-out' assert testmessages[0].start_offset == 123 assert testmessages[0].length == 10 assert testmessages[1].artifact == artifact assert testmessages[1].label == 'system-err' assert testmessages[1].start_offset == 555 assert testmessages[1].length == 25 assert _stat(jobstep, 'test_count') == 2 assert _stat(jobstep, 'test_failures') == 1 assert _stat(jobstep, 'test_duration') == 25 assert _stat(jobstep, 'test_rerun_count') == 0 failures = FailureReason.query.filter_by(step_id=jobstep.id).all() assert len(failures) == 1 assert failures[0].reason == 'duplicate_test_name'
def test_simple(self): from changes.models import TestCase, TestGroup build = self.create_build(self.project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) suite = TestSuite(name='foobar', job=job, project=self.project) db.session.add(suite) db.session.commit() results = [ TestResult( step=jobstep, suite=suite, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=156, ), TestResult( step=jobstep, suite=suite, name='test_foo', package='tests.changes.handlers.test_coverage', result=Result.passed, message='foobar failed', duration=12, reruns=1, ), ] manager = TestResultManager(jobstep) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == self.project.id assert test.suite_id == suite.id assert testcase_list[0].name == 'tests.changes.handlers.test_coverage.test_foo' assert testcase_list[0].result == Result.passed assert testcase_list[0].message == 'foobar failed' assert testcase_list[0].duration == 12 assert testcase_list[0].reruns == 1 assert len(testcase_list[0].groups) == 1 assert testcase_list[1].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[1].result == Result.failed assert testcase_list[1].message == 'collection failed' assert testcase_list[1].duration == 156 assert testcase_list[1].reruns is None group_list = sorted(TestGroup.query.all(), key=lambda x: x.name) assert len(group_list) == 4 for group in group_list: assert group.job_id == job.id assert group.project_id == self.project.id assert group.suite_id == suite.id assert group_list[0].name == 'tests.changes.handlers.test_coverage' assert group_list[0].duration == 12 assert group_list[0].num_tests == 1 assert group_list[0].num_failed == 0 assert group_list[0].result == Result.passed assert group_list[0].num_leaves == 1 assert list(group_list[0].testcases) == [] assert group_list[1].name == 'tests.changes.handlers.test_coverage.test_foo' assert group_list[1].duration == 12 assert group_list[1].num_tests == 1 assert group_list[1].num_failed == 0 assert group_list[1].result == Result.passed assert group_list[1].num_leaves == 0 assert list(group_list[1].testcases) == [testcase_list[0]] assert group_list[2].name == 'tests.changes.handlers.test_xunit' assert group_list[2].duration == 156 assert group_list[2].num_tests == 1 assert group_list[2].num_failed == 1 assert group_list[2].result == Result.failed assert group_list[2].num_leaves == 1 assert list(group_list[2].testcases) == [] assert group_list[3].name == 'tests.changes.handlers.test_xunit.test_bar' assert group_list[3].duration == 156 assert group_list[3].num_tests == 1 assert group_list[3].num_failed == 1 assert group_list[3].result == Result.failed assert group_list[3].num_leaves == 0 assert list(group_list[3].testcases) == [testcase_list[1]] teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == job.id, )[0] assert teststat.value == 2 # agg_groups = sorted(AggregateTestGroup.query.all(), key=lambda x: x.name) # assert len(agg_groups) == 4 # for agg in agg_groups: # # assert agg.last_job == build # assert agg.first_job_id == job.id # assert agg.project_id == self.project.id # assert agg_groups[0].name == 'tests.changes.handlers.test_coverage' # assert agg_groups[1].name == 'tests.changes.handlers.test_coverage.test_foo' # assert agg_groups[2].name == 'tests.changes.handlers.test_xunit' # assert agg_groups[3].name == 'tests.changes.handlers.test_xunit.test_bar' job2 = self.create_job(build) jobphase2 = self.create_jobphase(job2) jobstep2 = self.create_jobstep(jobphase2) results2 = [ TestResult( step=jobstep2, name='test_bar', package='tests.changes.handlers.test_bar', result=Result.failed, message='collection failed', duration=156, ), ] manager = TestResultManager(jobstep2) manager.save(results2) teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 3 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == job2.id, )[0] assert teststat.value == 1
def test_simple(self): from changes.models import TestCase build = self.create_build(self.project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) suite = TestSuite(name='foobar', job=job, project=self.project) db.session.add(suite) db.session.commit() results = [ TestResult( step=jobstep, suite=suite, name='test_bar', package='tests.changes.handlers.test_xunit', result=Result.failed, message='collection failed', duration=156, ), TestResult( step=jobstep, suite=suite, name='test_foo', package='tests.changes.handlers.test_coverage', result=Result.passed, message='foobar failed', duration=12, reruns=1, ), ] manager = TestResultManager(jobstep) manager.save(results) testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name) assert len(testcase_list) == 2 for test in testcase_list: assert test.job_id == job.id assert test.step_id == jobstep.id assert test.project_id == self.project.id assert test.suite_id == suite.id assert testcase_list[ 0].name == 'tests.changes.handlers.test_coverage.test_foo' assert testcase_list[0].result == Result.passed assert testcase_list[0].message == 'foobar failed' assert testcase_list[0].duration == 12 assert testcase_list[0].reruns == 1 assert testcase_list[ 1].name == 'tests.changes.handlers.test_xunit.test_bar' assert testcase_list[1].result == Result.failed assert testcase_list[1].message == 'collection failed' assert testcase_list[1].duration == 156 assert testcase_list[1].reruns is 0 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == job.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == build.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == job.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 168 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == job.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == jobstep.id, )[0] assert teststat.value == 1 job2 = self.create_job(build) jobphase2 = self.create_jobphase(job2) jobstep2 = self.create_jobstep(jobphase2) results2 = [ TestResult( step=jobstep2, name='test_bar', package='tests.changes.handlers.test_bar', result=Result.failed, message='collection failed', duration=156, reruns=2, ), ] manager = TestResultManager(jobstep2) manager.save(results2) teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 3 teststat = ItemStat.query.filter( ItemStat.name == 'test_count', ItemStat.item_id == job2.id, )[0] assert teststat.value == 1 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == build.id, )[0] assert teststat.value == 324 teststat = ItemStat.query.filter( ItemStat.name == 'test_duration', ItemStat.item_id == job2.id, )[0] assert teststat.value == 156 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == build.id, )[0] assert teststat.value == 2 teststat = ItemStat.query.filter( ItemStat.name == 'test_rerun_count', ItemStat.item_id == job2.id, )[0] assert teststat.value == 1