def process(self, fp, artifact):
test_list = self.get_tests(fp)
manager = TestResultManager(self.step, artifact)
manager.save(test_list)
return test_list
def process(self, fp, artifact):
target_name = self._get_target_name(artifact)
target, _ = get_or_create(BazelTarget,
where={
'step_id': self.step.id,
'job_id': self.step.job.id,
'name': target_name,
'result_source': ResultSource.from_self,
})
test_suites = self.get_test_suites(fp)
tests = self.aggregate_tests_from_suites(test_suites)
manager = TestResultManager(self.step, artifact)
manager.save(tests)
# update target metadata
# TODO handle multiple files per target, i.e. sharding and running multiple times
target.status = Status.finished
target.result = aggregate_result([t.result for t in tests])
duration = 0
for t in test_suites:
if t.duration is None:
duration = None
break
duration += t.duration
target.duration = duration
target.date_created = min([t.date_created for t in test_suites])
db.session.add(target)
db.session.commit()
return tests
def process(self, fp, artifact):
target_name = self._get_target_name(artifact)
target, _ = get_or_create(BazelTarget, where={
'step_id': self.step.id,
'job_id': self.step.job.id,
'name': target_name,
'result_source': ResultSource.from_self,
})
test_suites = self.get_test_suites(fp)
tests = self.aggregate_tests_from_suites(test_suites)
manager = TestResultManager(self.step, artifact)
manager.save(tests)
# update target metadata
# TODO handle multiple files per target, i.e. sharding and running multiple times
target.status = Status.finished
target.result = aggregate_result([t.result for t in tests])
duration = 0
for t in test_suites:
if t.duration is None:
duration = None
break
duration += t.duration
target.duration = duration
target.date_created = min([t.date_created for t in test_suites])
db.session.add(target)
db.session.commit()
return tests
def test_bad_duration(self):
from changes.models.test import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
artifact = self.create_artifact(jobstep, 'junit.xml')
results = [
TestResult(step=jobstep,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=2147483647 * 2,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('sample content')
}]),
]
manager = TestResultManager(jobstep, artifact)
with mock.patch.object(logger, 'warning') as warn:
manager.save(results)
assert warn.called
testcase_list = TestCase.query.all()
assert len(testcase_list) == 1
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[
0].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[0].result == Result.failed
assert testcase_list[0].message == 'collection failed'
assert testcase_list[0].duration == 0
def test_bad_duration(self):
from changes.models.test import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
artifact = self.create_artifact(jobstep, 'junit.xml')
results = [
TestResult(
step=jobstep,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=2147483647 * 2,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('sample content')}]),
]
manager = TestResultManager(jobstep, artifact)
with mock.patch.object(logger, 'warning') as warn:
manager.save(results)
assert warn.called
testcase_list = TestCase.query.all()
assert len(testcase_list) == 1
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[0].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[0].result == Result.failed
assert testcase_list[0].message == 'collection failed'
assert testcase_list[0].duration == 0
def test_simple(self):
from changes.models import TestCase, TestGroup
build = self.create_build(self.project)
job = self.create_job(build)
suite = TestSuite(name='foobar', job=job, project=self.project)
db.session.add(suite)
results = [
TestResult(
job=job,
suite=suite,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
),
TestResult(
job=job,
suite=suite,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
),
]
manager = TestResultManager(job)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.package)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.project_id == self.project.id
assert test.suite_id == suite.id
assert testcase_list[0].name == 'test_foo'
assert testcase_list[0].package == 'tests.changes.handlers.test_coverage'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert len(testcase_list[0].groups) == 1
assert testcase_list[1].name == 'test_bar'
assert testcase_list[1].package == 'tests.changes.handlers.test_xunit'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
group_list = sorted(TestGroup.query.all(), key=lambda x: x.name)
assert len(group_list) == 4
for group in group_list:
assert group.job_id == job.id
assert group.project_id == self.project.id
assert group.suite_id == suite.id
assert group_list[0].name == 'tests.changes.handlers.test_coverage'
assert group_list[0].duration == 12
assert group_list[0].num_tests == 1
assert group_list[0].num_failed == 0
assert group_list[0].result == Result.passed
assert group_list[0].num_leaves == 1
assert list(group_list[0].testcases) == []
assert group_list[1].name == 'tests.changes.handlers.test_coverage.test_foo'
assert group_list[1].duration == 12
assert group_list[1].num_tests == 1
assert group_list[1].num_failed == 0
assert group_list[1].result == Result.passed
assert group_list[1].num_leaves == 0
assert list(group_list[1].testcases) == [testcase_list[0]]
assert group_list[2].name == 'tests.changes.handlers.test_xunit'
assert group_list[2].duration == 156
assert group_list[2].num_tests == 1
assert group_list[2].num_failed == 1
assert group_list[2].result == Result.failed
assert group_list[2].num_leaves == 1
assert list(group_list[2].testcases) == []
assert group_list[3].name == 'tests.changes.handlers.test_xunit.test_bar'
assert group_list[3].duration == 156
assert group_list[3].num_tests == 1
assert group_list[3].num_failed == 1
assert group_list[3].result == Result.failed
assert group_list[3].num_leaves == 0
assert list(group_list[3].testcases) == [testcase_list[1]]
def test_simple(self):
from changes.models import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
results = [
TestResult(step=jobstep,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('sample content')
}]),
TestResult(
step=jobstep,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
reruns=1,
),
]
manager = TestResultManager(jobstep)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[
0].name == 'tests.changes.handlers.test_coverage.test_foo'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert testcase_list[0].reruns == 1
assert testcase_list[
1].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
assert testcase_list[1].reruns is 0
testartifacts = testcase_list[1].artifacts
assert len(testartifacts) == 1
assert testartifacts[0].file.get_file().read() == 'sample content'
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_failures',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 1
def test_simple(self):
from changes.models import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
results = [
TestResult(
step=jobstep,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('sample content')}]),
TestResult(
step=jobstep,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
reruns=1,
),
]
manager = TestResultManager(jobstep)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[0].name == 'tests.changes.handlers.test_coverage.test_foo'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert testcase_list[0].reruns == 1
assert testcase_list[1].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
assert testcase_list[1].reruns is 0
testartifacts = testcase_list[1].artifacts
assert len(testartifacts) == 1
assert testartifacts[0].file.get_file().read() == 'sample content'
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_failures',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 1
def test_duplicate_tests_in_different_result_lists(self):
from changes.models.test import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase, label='STEP1')
artifact = self.create_artifact(jobstep, 'junit.xml')
results = [
TestResult(
step=jobstep,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=12,
reruns=0,
artifacts=[{
'name': 'one_artifact',
'type': 'text',
'base64': b64encode('first artifact')}]
),
TestResult(
step=jobstep,
name='test_bar',
package='project.tests',
result=Result.passed,
duration=13,
reruns=0,
),
]
manager = TestResultManager(jobstep, artifact)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[0].name == 'project.tests.test_bar'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message is None
assert testcase_list[0].duration == 13
assert testcase_list[0].reruns == 0
assert len(testcase_list[0].artifacts) == 0
assert testcase_list[1].name == 'project.tests.test_foo'
assert testcase_list[1].result == Result.passed
assert testcase_list[1].message is None
assert testcase_list[1].duration == 12
assert testcase_list[1].reruns == 0
testartifacts = testcase_list[1].artifacts
assert len(testartifacts) == 1
a1 = testartifacts[0].file.get_file().read()
assert a1 == 'first artifact'
assert _stat(jobstep, 'test_count') == 2
assert _stat(jobstep, 'test_failures') == 0
assert _stat(jobstep, 'test_duration') == 25
assert _stat(jobstep, 'test_rerun_count') == 0
jobstep2 = self.create_jobstep(jobphase, label='STEP2')
artifact2 = self.create_artifact(jobstep2, 'junit.xml')
results = [
TestResult(
step=jobstep2,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=11,
reruns=0,
artifacts=[{
'name': 'another_artifact',
'type': 'text',
'base64': b64encode('second artifact')}]
),
TestResult(
step=jobstep2,
name='test_baz',
package='project.tests',
result=Result.passed,
duration=18,
reruns=2,
),
]
manager = TestResultManager(jobstep2, artifact2)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 3
for test in testcase_list:
assert test.job_id == job.id
assert test.project_id == project.id
assert testcase_list[0].step_id == jobstep.id
assert testcase_list[0].name == 'project.tests.test_bar'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message is None
assert testcase_list[0].duration == 13
assert testcase_list[0].reruns == 0
assert testcase_list[1].step_id == jobstep2.id
assert testcase_list[1].name == 'project.tests.test_baz'
assert testcase_list[1].result == Result.passed
assert testcase_list[1].message is None
assert testcase_list[1].duration == 18
assert testcase_list[1].reruns == 2
assert testcase_list[2].step_id == jobstep.id
assert testcase_list[2].name == 'project.tests.test_foo'
assert testcase_list[2].result == Result.failed
assert testcase_list[2].message.startswith('Error: Duplicate Test')
assert testcase_list[2].message.endswith('\nSTEP1\nSTEP2\n')
assert testcase_list[2].duration == 12
assert testcase_list[2].reruns == 0
testartifacts = testcase_list[2].artifacts
assert len(testartifacts) == 2
a1 = testartifacts[0].file.get_file().read()
a2 = testartifacts[1].file.get_file().read()
assert {a1, a2} == {'first artifact', 'second artifact'}
# Stats for original step are unharmed:
assert _stat(jobstep, 'test_count') == 2
assert _stat(jobstep, 'test_failures') == 1
assert _stat(jobstep, 'test_duration') == 25
assert _stat(jobstep, 'test_rerun_count') == 0
# Stats for new step:
assert _stat(jobstep2, 'test_count') == 1
assert _stat(jobstep2, 'test_failures') == 0
assert _stat(jobstep2, 'test_duration') == 18
assert _stat(jobstep2, 'test_rerun_count') == 1
failures = FailureReason.query.filter_by(step_id=jobstep.id).all()
assert len(failures) == 0
failures = FailureReason.query.filter_by(step_id=jobstep2.id).all()
assert len(failures) == 1
assert failures[0].reason == 'duplicate_test_name'
def test_duplicate_tests_in_same_result_list(self):
from changes.models.test import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase, label='STEP1')
artifact = self.create_artifact(jobstep, 'junit.xml')
results = [
TestResult(
step=jobstep,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=12,
reruns=0,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('first artifact')}],
message_offsets=[('system-out', 123, 10)],
),
TestResult(
step=jobstep,
name='test_bar',
package='project.tests',
result=Result.passed,
duration=13,
reruns=0,
),
TestResult(
step=jobstep,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=11,
reruns=0,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('second artifact')}],
message_offsets=[('system-err', 555, 25)],
),
]
manager = TestResultManager(jobstep, artifact)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[0].name == 'project.tests.test_bar'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message is None
assert testcase_list[0].duration == 13
assert testcase_list[0].reruns == 0
assert len(testcase_list[0].artifacts) == 0
assert len(testcase_list[0].messages) == 0
assert testcase_list[1].name == 'project.tests.test_foo'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message.startswith('Error: Duplicate Test')
assert testcase_list[1].message.endswith('\nSTEP1\n')
assert testcase_list[1].duration == 12
assert testcase_list[1].reruns == 0
testartifacts = testcase_list[1].artifacts
assert len(testartifacts) == 2
a1 = testartifacts[0].file.get_file().read()
a2 = testartifacts[1].file.get_file().read()
assert {a1, a2} == {'first artifact', 'second artifact'}
testmessages = testcase_list[1].messages
assert len(testmessages) == 2
assert testmessages[0].artifact == artifact
assert testmessages[0].label == 'system-out'
assert testmessages[0].start_offset == 123
assert testmessages[0].length == 10
assert testmessages[1].artifact == artifact
assert testmessages[1].label == 'system-err'
assert testmessages[1].start_offset == 555
assert testmessages[1].length == 25
assert _stat(jobstep, 'test_count') == 2
assert _stat(jobstep, 'test_failures') == 1
assert _stat(jobstep, 'test_duration') == 25
assert _stat(jobstep, 'test_rerun_count') == 0
failures = FailureReason.query.filter_by(step_id=jobstep.id).all()
assert len(failures) == 1
assert failures[0].reason == 'duplicate_test_name'
def test_simple(self):
from changes.models import TestCase
build = self.create_build(self.project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
suite = TestSuite(name='foobar', job=job, project=self.project)
db.session.add(suite)
db.session.commit()
results = [
TestResult(
step=jobstep,
suite=suite,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
),
TestResult(
step=jobstep,
suite=suite,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
reruns=1,
),
]
manager = TestResultManager(jobstep)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == self.project.id
assert test.suite_id == suite.id
assert testcase_list[0].name == 'tests.changes.handlers.test_coverage.test_foo'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert testcase_list[0].reruns == 1
assert testcase_list[1].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
assert testcase_list[1].reruns is 0
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 1
job2 = self.create_job(build)
jobphase2 = self.create_jobphase(job2)
jobstep2 = self.create_jobstep(jobphase2)
results2 = [
TestResult(
step=jobstep2,
name='test_bar',
package='tests.changes.handlers.test_bar',
result=Result.failed,
message='collection failed',
duration=156,
reruns=2,
),
]
manager = TestResultManager(jobstep2)
manager.save(results2)
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 3
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 324
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 156
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 1
def test_duplicate_tests_in_different_result_lists(self):
from changes.models.test import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase, label='STEP1')
artifact = self.create_artifact(jobstep, 'junit.xml')
results = [
TestResult(step=jobstep,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=12,
reruns=0,
artifacts=[{
'name': 'one_artifact',
'type': 'text',
'base64': b64encode('first artifact')
}]),
TestResult(
step=jobstep,
name='test_bar',
package='project.tests',
result=Result.passed,
duration=13,
reruns=0,
),
]
manager = TestResultManager(jobstep, artifact)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[0].name == 'project.tests.test_bar'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message is None
assert testcase_list[0].duration == 13
assert testcase_list[0].reruns == 0
assert len(testcase_list[0].artifacts) == 0
assert testcase_list[1].name == 'project.tests.test_foo'
assert testcase_list[1].result == Result.passed
assert testcase_list[1].message is None
assert testcase_list[1].duration == 12
assert testcase_list[1].reruns == 0
testartifacts = testcase_list[1].artifacts
assert len(testartifacts) == 1
a1 = testartifacts[0].file.get_file().read()
assert a1 == 'first artifact'
assert _stat(jobstep, 'test_count') == 2
assert _stat(jobstep, 'test_failures') == 0
assert _stat(jobstep, 'test_duration') == 25
assert _stat(jobstep, 'test_rerun_count') == 0
jobstep2 = self.create_jobstep(jobphase, label='STEP2')
artifact2 = self.create_artifact(jobstep2, 'junit.xml')
results = [
TestResult(step=jobstep2,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=11,
reruns=0,
artifacts=[{
'name': 'another_artifact',
'type': 'text',
'base64': b64encode('second artifact')
}]),
TestResult(
step=jobstep2,
name='test_baz',
package='project.tests',
result=Result.passed,
duration=18,
reruns=2,
),
]
manager = TestResultManager(jobstep2, artifact2)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 3
for test in testcase_list:
assert test.job_id == job.id
assert test.project_id == project.id
assert testcase_list[0].step_id == jobstep.id
assert testcase_list[0].name == 'project.tests.test_bar'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message is None
assert testcase_list[0].duration == 13
assert testcase_list[0].reruns == 0
assert testcase_list[1].step_id == jobstep2.id
assert testcase_list[1].name == 'project.tests.test_baz'
assert testcase_list[1].result == Result.passed
assert testcase_list[1].message is None
assert testcase_list[1].duration == 18
assert testcase_list[1].reruns == 2
assert testcase_list[2].step_id == jobstep.id
assert testcase_list[2].name == 'project.tests.test_foo'
assert testcase_list[2].result == Result.failed
assert testcase_list[2].message.startswith('Error: Duplicate Test')
assert testcase_list[2].message.endswith('\nSTEP1\nSTEP2\n')
assert testcase_list[2].duration == 12
assert testcase_list[2].reruns == 0
testartifacts = testcase_list[2].artifacts
assert len(testartifacts) == 2
a1 = testartifacts[0].file.get_file().read()
a2 = testartifacts[1].file.get_file().read()
assert {a1, a2} == {'first artifact', 'second artifact'}
# Stats for original step are unharmed:
assert _stat(jobstep, 'test_count') == 2
assert _stat(jobstep, 'test_failures') == 1
assert _stat(jobstep, 'test_duration') == 25
assert _stat(jobstep, 'test_rerun_count') == 0
# Stats for new step:
assert _stat(jobstep2, 'test_count') == 1
assert _stat(jobstep2, 'test_failures') == 0
assert _stat(jobstep2, 'test_duration') == 18
assert _stat(jobstep2, 'test_rerun_count') == 1
failures = FailureReason.query.filter_by(step_id=jobstep.id).all()
assert len(failures) == 0
failures = FailureReason.query.filter_by(step_id=jobstep2.id).all()
assert len(failures) == 1
assert failures[0].reason == 'duplicate_test_name'
def test_duplicate_tests_in_same_result_list(self):
from changes.models.test import TestCase
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase, label='STEP1')
artifact = self.create_artifact(jobstep, 'junit.xml')
results = [
TestResult(
step=jobstep,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=12,
reruns=0,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('first artifact')
}],
message_offsets=[('system-out', 123, 10)],
),
TestResult(
step=jobstep,
name='test_bar',
package='project.tests',
result=Result.passed,
duration=13,
reruns=0,
),
TestResult(
step=jobstep,
name='test_foo',
package='project.tests',
result=Result.passed,
duration=11,
reruns=0,
artifacts=[{
'name': 'artifact_name',
'type': 'text',
'base64': b64encode('second artifact')
}],
message_offsets=[('system-err', 555, 25)],
),
]
manager = TestResultManager(jobstep, artifact)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == project.id
assert testcase_list[0].name == 'project.tests.test_bar'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message is None
assert testcase_list[0].duration == 13
assert testcase_list[0].reruns == 0
assert len(testcase_list[0].artifacts) == 0
assert len(testcase_list[0].messages) == 0
assert testcase_list[1].name == 'project.tests.test_foo'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message.startswith('Error: Duplicate Test')
assert testcase_list[1].message.endswith('\nSTEP1\n')
assert testcase_list[1].duration == 12
assert testcase_list[1].reruns == 0
testartifacts = testcase_list[1].artifacts
assert len(testartifacts) == 2
a1 = testartifacts[0].file.get_file().read()
a2 = testartifacts[1].file.get_file().read()
assert {a1, a2} == {'first artifact', 'second artifact'}
testmessages = testcase_list[1].messages
assert len(testmessages) == 2
assert testmessages[0].artifact == artifact
assert testmessages[0].label == 'system-out'
assert testmessages[0].start_offset == 123
assert testmessages[0].length == 10
assert testmessages[1].artifact == artifact
assert testmessages[1].label == 'system-err'
assert testmessages[1].start_offset == 555
assert testmessages[1].length == 25
assert _stat(jobstep, 'test_count') == 2
assert _stat(jobstep, 'test_failures') == 1
assert _stat(jobstep, 'test_duration') == 25
assert _stat(jobstep, 'test_rerun_count') == 0
failures = FailureReason.query.filter_by(step_id=jobstep.id).all()
assert len(failures) == 1
assert failures[0].reason == 'duplicate_test_name'
def test_simple(self):
from changes.models import TestCase, TestGroup
build = self.create_build(self.project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
suite = TestSuite(name='foobar', job=job, project=self.project)
db.session.add(suite)
db.session.commit()
results = [
TestResult(
step=jobstep,
suite=suite,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
),
TestResult(
step=jobstep,
suite=suite,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
reruns=1,
),
]
manager = TestResultManager(jobstep)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == self.project.id
assert test.suite_id == suite.id
assert testcase_list[0].name == 'tests.changes.handlers.test_coverage.test_foo'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert testcase_list[0].reruns == 1
assert len(testcase_list[0].groups) == 1
assert testcase_list[1].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
assert testcase_list[1].reruns is None
group_list = sorted(TestGroup.query.all(), key=lambda x: x.name)
assert len(group_list) == 4
for group in group_list:
assert group.job_id == job.id
assert group.project_id == self.project.id
assert group.suite_id == suite.id
assert group_list[0].name == 'tests.changes.handlers.test_coverage'
assert group_list[0].duration == 12
assert group_list[0].num_tests == 1
assert group_list[0].num_failed == 0
assert group_list[0].result == Result.passed
assert group_list[0].num_leaves == 1
assert list(group_list[0].testcases) == []
assert group_list[1].name == 'tests.changes.handlers.test_coverage.test_foo'
assert group_list[1].duration == 12
assert group_list[1].num_tests == 1
assert group_list[1].num_failed == 0
assert group_list[1].result == Result.passed
assert group_list[1].num_leaves == 0
assert list(group_list[1].testcases) == [testcase_list[0]]
assert group_list[2].name == 'tests.changes.handlers.test_xunit'
assert group_list[2].duration == 156
assert group_list[2].num_tests == 1
assert group_list[2].num_failed == 1
assert group_list[2].result == Result.failed
assert group_list[2].num_leaves == 1
assert list(group_list[2].testcases) == []
assert group_list[3].name == 'tests.changes.handlers.test_xunit.test_bar'
assert group_list[3].duration == 156
assert group_list[3].num_tests == 1
assert group_list[3].num_failed == 1
assert group_list[3].result == Result.failed
assert group_list[3].num_leaves == 0
assert list(group_list[3].testcases) == [testcase_list[1]]
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 2
# agg_groups = sorted(AggregateTestGroup.query.all(), key=lambda x: x.name)
# assert len(agg_groups) == 4
# for agg in agg_groups:
# # assert agg.last_job == build
# assert agg.first_job_id == job.id
# assert agg.project_id == self.project.id
# assert agg_groups[0].name == 'tests.changes.handlers.test_coverage'
# assert agg_groups[1].name == 'tests.changes.handlers.test_coverage.test_foo'
# assert agg_groups[2].name == 'tests.changes.handlers.test_xunit'
# assert agg_groups[3].name == 'tests.changes.handlers.test_xunit.test_bar'
job2 = self.create_job(build)
jobphase2 = self.create_jobphase(job2)
jobstep2 = self.create_jobstep(jobphase2)
results2 = [
TestResult(
step=jobstep2,
name='test_bar',
package='tests.changes.handlers.test_bar',
result=Result.failed,
message='collection failed',
duration=156,
),
]
manager = TestResultManager(jobstep2)
manager.save(results2)
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 3
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 1
def test_simple(self):
from changes.models import TestCase
build = self.create_build(self.project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase)
suite = TestSuite(name='foobar', job=job, project=self.project)
db.session.add(suite)
db.session.commit()
results = [
TestResult(
step=jobstep,
suite=suite,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
),
TestResult(
step=jobstep,
suite=suite,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
reruns=1,
),
]
manager = TestResultManager(jobstep)
manager.save(results)
testcase_list = sorted(TestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == job.id
assert test.step_id == jobstep.id
assert test.project_id == self.project.id
assert test.suite_id == suite.id
assert testcase_list[
0].name == 'tests.changes.handlers.test_coverage.test_foo'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert testcase_list[0].reruns == 1
assert testcase_list[
1].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
assert testcase_list[1].reruns is 0
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 168
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == job.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == jobstep.id,
)[0]
assert teststat.value == 1
job2 = self.create_job(build)
jobphase2 = self.create_jobphase(job2)
jobstep2 = self.create_jobstep(jobphase2)
results2 = [
TestResult(
step=jobstep2,
name='test_bar',
package='tests.changes.handlers.test_bar',
result=Result.failed,
message='collection failed',
duration=156,
reruns=2,
),
]
manager = TestResultManager(jobstep2)
manager.save(results2)
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 3
teststat = ItemStat.query.filter(
ItemStat.name == 'test_count',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 324
teststat = ItemStat.query.filter(
ItemStat.name == 'test_duration',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 156
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == build.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'test_rerun_count',
ItemStat.item_id == job2.id,
)[0]
assert teststat.value == 1
