def test_build_tests_list(
client,
db_session,
default_login,
default_repo,
default_revision,
default_build,
default_repo_access,
):
job1 = factories.JobFactory(build=default_build)
job2 = factories.JobFactory(build=default_build)
db_session.add(job1)
db_session.add(job2)
testcase1 = factories.TestCaseFactory(job=job1, name="bar", passed=True)
testcase2 = factories.TestCaseFactory(job=job2, name="foo", passed=True)
testcase3 = factories.TestCaseFactory(job=job2, name="bar", failed=True)
db_session.add(testcase1)
db_session.add(testcase2)
db_session.add(testcase3)
resp = client.get("/api/repos/{}/revisions/{}/tests".format(
default_repo.get_full_name(), default_revision.sha))
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
def test_build_tests_list(client, db_session, default_login, default_repo,
default_build, default_repo_access):
job1 = factories.JobFactory(build=default_build, )
job2 = factories.JobFactory(build=default_build, )
db_session.add(job1)
db_session.add(job2)
testcase1 = factories.TestCaseFactory(
job=job1,
name='bar',
)
testcase2 = factories.TestCaseFactory(
job=job2,
name='foo',
)
db_session.add(testcase1)
db_session.add(testcase2)
resp = client.get('/api/repos/{}/builds/{}/tests'.format(
default_repo.get_full_name(), default_build.number))
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert data[0]['id'] == str(testcase1.id)
assert data[1]['id'] == str(testcase2.id)
def test_test_stats(mocker, db_session, default_source):
auth.set_current_tenant(
auth.Tenant(repository_ids=[default_source.repository_id]))
build = factories.BuildFactory(source=default_source, in_progress=True)
db_session.add(build)
job = factories.JobFactory(build=build, passed=True)
db_session.add(job)
job2 = factories.JobFactory(build=build, passed=True)
db_session.add(job2)
db_session.add(
factories.TestCaseFactory(
job=job,
name='foo',
failed=True,
duration=8,
))
db_session.add(
factories.TestCaseFactory(
job=job,
name='bar',
passed=True,
duration=2,
))
db_session.add(
factories.TestCaseFactory(
job=job2,
name='bar',
failed=True,
duration=2,
))
aggregate_build_stats_for_job(job.id)
aggregate_build_stats_for_job(job2.id)
aggregate_build_stats(build.id)
build_stats = {
i.name: i.value
for i in ItemStat.query.filter(ItemStat.item_id == build.id, )
}
assert build_stats['tests.count'] == 3
assert build_stats['tests.count_unique'] == 2
assert build_stats['tests.failures'] == 2
assert build_stats['tests.failures_unique'] == 2
assert build_stats['tests.duration'] == 12
job_stats = {
i.name: i.value
for i in ItemStat.query.filter(ItemStat.item_id == job.id, )
}
assert job_stats['tests.count'] == 2
assert job_stats['tests.failures'] == 1
assert job_stats['tests.duration'] == 10
def test_build_tests_list(client, db_session, default_login, default_repo,
default_build, default_repo_access):
job1 = factories.JobFactory(build=default_build, )
job2 = factories.JobFactory(build=default_build, )
db_session.add(job1)
db_session.add(job2)
testcase1 = factories.TestCaseFactory(
job=job1,
name='bar',
passed=True,
)
testcase2 = factories.TestCaseFactory(
job=job2,
name='foo',
passed=True,
)
testcase3 = factories.TestCaseFactory(
job=job2,
name='bar',
failed=True,
)
db_session.add(testcase1)
db_session.add(testcase2)
db_session.add(testcase3)
resp = client.get('/api/repos/{}/builds/{}/tests'.format(
default_repo.get_full_name(), default_build.number))
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert data[0]['name'] == 'bar'
assert data[0]['result'] == 'failed'
assert data[0]['runs'] == [{
'id': str(testcase3.id),
'job_id': str(job2.id),
'result': 'failed',
'duration': testcase3.duration,
}, {
'id': str(testcase1.id),
'job_id': str(job1.id),
'result': 'passed',
'duration': testcase1.duration,
}]
assert data[1]['name'] == 'foo'
assert data[1]['result'] == 'passed'
assert data[1]['runs'] == [{
'id': str(testcase2.id),
'job_id': str(job2.id),
'result': 'passed',
'duration': testcase2.duration,
}]
def test_build_tests_list(
client,
db_session,
default_login,
default_repo,
default_build,
default_repo_access,
mock_vcs_server,
):
job1 = factories.JobFactory(build=default_build)
job2 = factories.JobFactory(build=default_build)
db_session.add(job1)
db_session.add(job2)
testcase1 = factories.TestCaseFactory(job=job1, name="bar", passed=True)
testcase2 = factories.TestCaseFactory(job=job2, name="foo", passed=True)
testcase3 = factories.TestCaseFactory(job=job2, name="bar", failed=True)
db_session.add(testcase1)
db_session.add(testcase2)
db_session.add(testcase3)
resp = client.get("/api/repos/{}/builds/{}/tests".format(
default_repo.get_full_name(), default_build.number))
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert data[0]["name"] == "bar"
assert data[0]["result"] == "failed"
assert data[0]["runs"] == [
{
"id": str(testcase3.id),
"job_id": str(job2.id),
"result": "failed",
"duration": testcase3.duration,
},
{
"id": str(testcase1.id),
"job_id": str(job1.id),
"result": "passed",
"duration": testcase1.duration,
},
]
assert data[1]["name"] == "foo"
assert data[1]["result"] == "passed"
assert data[1]["runs"] == [{
"id": str(testcase2.id),
"job_id": str(job2.id),
"result": "passed",
"duration": testcase2.duration,
}]
def test_failing_tests_duplicate_reason(mocker, db_session, default_revision,
default_tenant):
build = factories.BuildFactory(revision=default_revision, in_progress=True)
db_session.add(build)
job = factories.JobFactory(build=build, passed=True)
db_session.add(job)
factories.TestCaseFactory(job=job, failed=True)
db_session.add(
FailureReason(
build_id=build.id,
reason=FailureReason.Reason.failing_tests,
job_id=job.id,
repository_id=job.repository_id,
))
aggregate_build_stats_for_job(job.id)
assert job.result == Result.failed
reasons = list(FailureReason.query.filter(FailureReason.job_id == job.id))
assert len(reasons) == 1
assert reasons[0].reason == FailureReason.Reason.failing_tests
def test_repository_tests_history_by_build(
client,
default_login,
default_build,
default_testcase,
default_repo,
default_repo_access,
default_revision,
):
build2 = factories.BuildFactory(revision=default_revision, finished=True)
job2 = factories.JobFactory(build=build2)
factories.TestCaseFactory(job=job2,
name=default_testcase.name,
failed=True)
build3 = factories.BuildFactory(revision=default_revision, finished=True)
job3 = factories.JobFactory(build=build3)
testcase2 = factories.TestCaseFactory(job=job3,
passed=True,
name=default_testcase.name + "2")
build4 = factories.BuildFactory(revision=default_revision, finished=True)
job4 = factories.JobFactory(build=build4)
factories.TestCaseFactory(job=job4,
name=default_testcase.name,
passed=True)
resp = client.get("/api/repos/{}/tests-by-build?results=3".format(
default_repo.get_full_name()))
assert resp.status_code == 200
data = resp.json()
assert data["tests"] == [
{
"name": default_testcase.name,
"hash": default_testcase.hash,
"results": ["passed", None, "failed"],
},
{
"name": testcase2.name,
"hash": testcase2.hash,
"results": [None, "passed", None],
},
]
assert len(data["builds"]) == 3
assert data["builds"][0]["id"] == str(build4.id)
assert data["builds"][1]["id"] == str(build3.id)
assert data["builds"][2]["id"] == str(build2.id)
def test_test_stats(mocker, db_session, default_revision, default_tenant):
build = factories.BuildFactory(revision=default_revision, in_progress=True)
db_session.add(build)
job = factories.JobFactory(build=build, passed=True)
db_session.add(job)
job2 = factories.JobFactory(build=build, passed=True)
db_session.add(job2)
db_session.add(
factories.TestCaseFactory(job=job, name="foo", failed=True,
duration=8))
db_session.add(
factories.TestCaseFactory(job=job, name="bar", passed=True,
duration=2))
db_session.add(
factories.TestCaseFactory(job=job2,
name="bar",
failed=True,
duration=2))
aggregate_build_stats_for_job(job.id)
aggregate_build_stats_for_job(job2.id)
aggregate_build_stats(build.id)
build_stats = {
i.name: i.value
for i in ItemStat.query.filter(ItemStat.item_id == build.id)
}
assert build_stats["tests.count"] == 3
assert build_stats["tests.count_unique"] == 2
assert build_stats["tests.failures"] == 2
assert build_stats["tests.failures_unique"] == 2
assert build_stats["tests.duration"] == 12
job_stats = {
i.name: i.value
for i in ItemStat.query.filter(ItemStat.item_id == job.id)
}
assert job_stats["tests.count"] == 2
assert job_stats["tests.failures"] == 1
assert job_stats["tests.duration"] == 10
def test_test_details(client, default_login, default_org, default_project,
default_build, default_job, default_repo_access):
testcase = factories.TestCaseFactory(
job=default_job,
failed=True,
)
resp = client.get('/api/projects/{}/{}/builds/{}/tests/{}'.format(
default_org.name, default_project.name, default_build.number,
testcase.name))
assert resp.status_code == 200
data = resp.json()
assert data['id'] == str(testcase.id)
assert data['message'] == str(testcase.message)
def test_failing_tests(mocker, db_session, default_source):
auth.set_current_tenant(
auth.Tenant(repository_ids=[default_source.repository_id]))
build = factories.BuildFactory(source=default_source, in_progress=True)
db_session.add(build)
job = factories.JobFactory(build=build, passed=True)
db_session.add(job)
factories.TestCaseFactory(job=job, failed=True)
aggregate_build_stats_for_job(job.id)
assert job.result == Result.failed
reasons = list(FailureReason.query.filter(FailureReason.job_id == job.id))
assert len(reasons) == 1
assert reasons[0].reason == FailureReason.Code.failing_tests
def test_simple(
client,
db_session,
default_login,
default_repo,
default_repo_access,
default_revision,
):
# finished build
build = factories.BuildFactory(revision=default_revision, passed=True)
db_session.add(build)
# an unfinished build which shouldn't be used
factories.BuildFactory(revision=default_revision,
status=Status.in_progress)
db_session.add(build)
# a couple of needed jobs that split the tests
job1 = factories.JobFactory(build=build,
status=Status.finished,
result=Result.passed)
db_session.add(job1)
job2 = factories.JobFactory(build=build,
status=Status.finished,
result=Result.passed)
db_session.add(job2)
# and finally our testcases
test1 = factories.TestCaseFactory(job=job1, name="foo.bar", duration=50)
db_session.add(test1)
test2 = factories.TestCaseFactory(job=job1, name="foo.baz", duration=70)
db_session.add(test2)
test3 = factories.TestCaseFactory(job=job2, name="blah.blah", duration=10)
db_session.add(test3)
db_session.commit()
path = "/api/repos/{}/test-tree".format(default_repo.get_full_name())
resp = client.get(path)
assert resp.status_code == 200
data = resp.json()
assert len(data["entries"]) == 2
assert data["entries"][0]["name"] == "foo"
assert data["entries"][0]["path"] == "foo"
assert data["entries"][0]["numTests"] == 2
assert data["entries"][0]["totalDuration"] == 120
assert data["entries"][1]["name"] == "blah.blah"
assert data["entries"][1]["path"] == "blah.blah"
assert data["entries"][1]["numTests"] == 1
assert data["entries"][1]["totalDuration"] == 10
assert len(data["trail"]) == 0
resp = client.get("{}?parent=foo".format(path))
assert resp.status_code == 200
data = resp.json()
assert len(data["entries"]) == 2
assert data["entries"][0]["name"] == "baz"
assert data["entries"][0]["path"] == "foo.baz"
assert data["entries"][0]["numTests"] == 1
assert data["entries"][0]["totalDuration"] == 70
assert data["entries"][1]["name"] == "bar"
assert data["entries"][1]["path"] == "foo.bar"
assert data["entries"][1]["numTests"] == 1
assert data["entries"][1]["totalDuration"] == 50
assert len(data["trail"]) == 1
assert data["trail"][0] == {"name": "foo", "path": "foo"}
def default_testcase(default_job):
return factories.TestCaseFactory(
job=default_job,
passed=True,
)
def test_simple(
client, db_session, default_login, default_repo, default_repo_access, default_revision
):
source = factories.SourceFactory(
revision=default_revision,
)
# finished build
build = factories.BuildFactory(
source=source,
passed=True,
)
db_session.add(build)
# an unfinished build which shouldn't be used
factories.BuildFactory(
source=source,
status=Status.in_progress,
)
db_session.add(build)
# a couple of needed jobs that split the tests
job1 = factories.JobFactory(
build=build,
status=Status.finished,
result=Result.passed,
)
db_session.add(job1)
job2 = factories.JobFactory(
build=build,
status=Status.finished,
result=Result.passed,
)
db_session.add(job2)
# and finally our testcases
test1 = factories.TestCaseFactory(
job=job1,
name='foo.bar',
duration=50,
)
db_session.add(test1)
test2 = factories.TestCaseFactory(
job=job1,
name='foo.baz',
duration=70,
)
db_session.add(test2)
test3 = factories.TestCaseFactory(
job=job2,
name='blah.blah',
duration=10,
)
db_session.add(test3)
db_session.commit()
path = '/api/repos/{}/{}/test-tree'.format(default_repo.owner_name, default_repo.name)
resp = client.get(path)
assert resp.status_code == 200
data = resp.json()
assert len(data['entries']) == 2
assert data['entries'][0]['name'] == 'foo'
assert data['entries'][0]['path'] == 'foo'
assert data['entries'][0]['numTests'] == 2
assert data['entries'][0]['totalDuration'] == 120
assert data['entries'][1]['name'] == 'blah.blah'
assert data['entries'][1]['path'] == 'blah.blah'
assert data['entries'][1]['numTests'] == 1
assert data['entries'][1]['totalDuration'] == 10
assert len(data['trail']) == 0
resp = client.get('{}?parent=foo'.format(path))
assert resp.status_code == 200
data = resp.json()
assert len(data['entries']) == 2
assert data['entries'][0]['name'] == 'baz'
assert data['entries'][0]['path'] == 'foo.baz'
assert data['entries'][0]['numTests'] == 1
assert data['entries'][0]['totalDuration'] == 70
assert data['entries'][1]['name'] == 'bar'
assert data['entries'][1]['path'] == 'foo.bar'
assert data['entries'][1]['numTests'] == 1
assert data['entries'][1]['totalDuration'] == 50
assert len(data['trail']) == 1
assert data['trail'][0] == {
'name': 'foo',
'path': 'foo',
}
def test_failure_origin(default_repo):
auth.set_current_tenant(
auth.Tenant(access={default_repo.id: Permission.read}))
new_revision = factories.RevisionFactory(repository=default_repo)
new_source = factories.SourceFactory(revision=new_revision)
new_build = factories.BuildFactory(source=new_source, failed=True)
new_job = factories.JobFactory(build=new_build, failed=True)
new_testcase = factories.TestCaseFactory(job=new_job, failed=True)
new_testcase2 = factories.TestCaseFactory(job=new_job, passed=True)
old_revision = factories.RevisionFactory(
repository=default_repo,
date_created=new_revision.date_created - timedelta(hours=1),
)
old_source = factories.SourceFactory(revision=old_revision,
date_created=new_source.date_created -
timedelta(hours=1))
old_build = factories.BuildFactory(
source=old_source,
failed=True,
date_created=new_build.date_created - timedelta(hours=1),
)
old_job = factories.JobFactory(
build=old_build,
failed=True,
date_created=new_job.date_created - timedelta(hours=1),
)
factories.TestCaseFactory(job=old_job, failed=True, name=new_testcase.name)
oldold_revision = factories.RevisionFactory(
repository=default_repo,
date_created=old_revision.date_created - timedelta(hours=1),
)
oldold_source = factories.SourceFactory(
revision=oldold_revision,
date_created=old_source.date_created - timedelta(hours=1),
)
factories.BuildFactory(
source=oldold_source,
passed=True,
date_created=old_build.date_created - timedelta(hours=1),
)
schema = AggregateTestCaseSummarySchema(many=True,
strict=True,
context={"build": new_build})
result = schema.dump([
AggregateTestCase(
hash=new_testcase.hash,
name=new_testcase.name,
runs=[[
str(new_testcase.id),
str(new_testcase.job_id),
new_testcase.duration,
int(new_testcase.result),
]],
),
AggregateTestCase(
hash=new_testcase2.hash,
name=new_testcase2.name,
runs=[[
str(new_testcase2.id),
str(new_testcase2.job_id),
new_testcase2.duration,
int(new_testcase2.result),
]],
),
]).data
assert len(result) == 2
assert result[0]["hash"] == new_testcase.hash
assert result[0]["name"] == new_testcase.name
assert result[0]["origin_build"]["id"] == str(old_build.id)
assert result[1]["hash"] == new_testcase2.hash
assert result[1]["name"] == new_testcase2.name
assert result[1]["origin_build"] is None
