def test_build_tests_list( client, db_session, default_login, default_repo, default_revision, default_build, default_repo_access, ): job1 = factories.JobFactory(build=default_build) job2 = factories.JobFactory(build=default_build) db_session.add(job1) db_session.add(job2) testcase1 = factories.TestCaseFactory(job=job1, name="bar", passed=True) testcase2 = factories.TestCaseFactory(job=job2, name="foo", passed=True) testcase3 = factories.TestCaseFactory(job=job2, name="bar", failed=True) db_session.add(testcase1) db_session.add(testcase2) db_session.add(testcase3) resp = client.get("/api/repos/{}/revisions/{}/tests".format( default_repo.get_full_name(), default_revision.sha)) assert resp.status_code == 200 data = resp.json() assert len(data) == 2
def test_build_tests_list(client, db_session, default_login, default_repo, default_build, default_repo_access): job1 = factories.JobFactory(build=default_build, ) job2 = factories.JobFactory(build=default_build, ) db_session.add(job1) db_session.add(job2) testcase1 = factories.TestCaseFactory( job=job1, name='bar', ) testcase2 = factories.TestCaseFactory( job=job2, name='foo', ) db_session.add(testcase1) db_session.add(testcase2) resp = client.get('/api/repos/{}/builds/{}/tests'.format( default_repo.get_full_name(), default_build.number)) assert resp.status_code == 200 data = resp.json() assert len(data) == 2 assert data[0]['id'] == str(testcase1.id) assert data[1]['id'] == str(testcase2.id)
def test_test_stats(mocker, db_session, default_source): auth.set_current_tenant( auth.Tenant(repository_ids=[default_source.repository_id])) build = factories.BuildFactory(source=default_source, in_progress=True) db_session.add(build) job = factories.JobFactory(build=build, passed=True) db_session.add(job) job2 = factories.JobFactory(build=build, passed=True) db_session.add(job2) db_session.add( factories.TestCaseFactory( job=job, name='foo', failed=True, duration=8, )) db_session.add( factories.TestCaseFactory( job=job, name='bar', passed=True, duration=2, )) db_session.add( factories.TestCaseFactory( job=job2, name='bar', failed=True, duration=2, )) aggregate_build_stats_for_job(job.id) aggregate_build_stats_for_job(job2.id) aggregate_build_stats(build.id) build_stats = { i.name: i.value for i in ItemStat.query.filter(ItemStat.item_id == build.id, ) } assert build_stats['tests.count'] == 3 assert build_stats['tests.count_unique'] == 2 assert build_stats['tests.failures'] == 2 assert build_stats['tests.failures_unique'] == 2 assert build_stats['tests.duration'] == 12 job_stats = { i.name: i.value for i in ItemStat.query.filter(ItemStat.item_id == job.id, ) } assert job_stats['tests.count'] == 2 assert job_stats['tests.failures'] == 1 assert job_stats['tests.duration'] == 10
def test_build_tests_list(client, db_session, default_login, default_repo, default_build, default_repo_access): job1 = factories.JobFactory(build=default_build, ) job2 = factories.JobFactory(build=default_build, ) db_session.add(job1) db_session.add(job2) testcase1 = factories.TestCaseFactory( job=job1, name='bar', passed=True, ) testcase2 = factories.TestCaseFactory( job=job2, name='foo', passed=True, ) testcase3 = factories.TestCaseFactory( job=job2, name='bar', failed=True, ) db_session.add(testcase1) db_session.add(testcase2) db_session.add(testcase3) resp = client.get('/api/repos/{}/builds/{}/tests'.format( default_repo.get_full_name(), default_build.number)) assert resp.status_code == 200 data = resp.json() assert len(data) == 2 assert data[0]['name'] == 'bar' assert data[0]['result'] == 'failed' assert data[0]['runs'] == [{ 'id': str(testcase3.id), 'job_id': str(job2.id), 'result': 'failed', 'duration': testcase3.duration, }, { 'id': str(testcase1.id), 'job_id': str(job1.id), 'result': 'passed', 'duration': testcase1.duration, }] assert data[1]['name'] == 'foo' assert data[1]['result'] == 'passed' assert data[1]['runs'] == [{ 'id': str(testcase2.id), 'job_id': str(job2.id), 'result': 'passed', 'duration': testcase2.duration, }]
def test_build_tests_list( client, db_session, default_login, default_repo, default_build, default_repo_access, mock_vcs_server, ): job1 = factories.JobFactory(build=default_build) job2 = factories.JobFactory(build=default_build) db_session.add(job1) db_session.add(job2) testcase1 = factories.TestCaseFactory(job=job1, name="bar", passed=True) testcase2 = factories.TestCaseFactory(job=job2, name="foo", passed=True) testcase3 = factories.TestCaseFactory(job=job2, name="bar", failed=True) db_session.add(testcase1) db_session.add(testcase2) db_session.add(testcase3) resp = client.get("/api/repos/{}/builds/{}/tests".format( default_repo.get_full_name(), default_build.number)) assert resp.status_code == 200 data = resp.json() assert len(data) == 2 assert data[0]["name"] == "bar" assert data[0]["result"] == "failed" assert data[0]["runs"] == [ { "id": str(testcase3.id), "job_id": str(job2.id), "result": "failed", "duration": testcase3.duration, }, { "id": str(testcase1.id), "job_id": str(job1.id), "result": "passed", "duration": testcase1.duration, }, ] assert data[1]["name"] == "foo" assert data[1]["result"] == "passed" assert data[1]["runs"] == [{ "id": str(testcase2.id), "job_id": str(job2.id), "result": "passed", "duration": testcase2.duration, }]
def test_failing_tests_duplicate_reason(mocker, db_session, default_revision, default_tenant): build = factories.BuildFactory(revision=default_revision, in_progress=True) db_session.add(build) job = factories.JobFactory(build=build, passed=True) db_session.add(job) factories.TestCaseFactory(job=job, failed=True) db_session.add( FailureReason( build_id=build.id, reason=FailureReason.Reason.failing_tests, job_id=job.id, repository_id=job.repository_id, )) aggregate_build_stats_for_job(job.id) assert job.result == Result.failed reasons = list(FailureReason.query.filter(FailureReason.job_id == job.id)) assert len(reasons) == 1 assert reasons[0].reason == FailureReason.Reason.failing_tests
def test_repository_tests_history_by_build( client, default_login, default_build, default_testcase, default_repo, default_repo_access, default_revision, ): build2 = factories.BuildFactory(revision=default_revision, finished=True) job2 = factories.JobFactory(build=build2) factories.TestCaseFactory(job=job2, name=default_testcase.name, failed=True) build3 = factories.BuildFactory(revision=default_revision, finished=True) job3 = factories.JobFactory(build=build3) testcase2 = factories.TestCaseFactory(job=job3, passed=True, name=default_testcase.name + "2") build4 = factories.BuildFactory(revision=default_revision, finished=True) job4 = factories.JobFactory(build=build4) factories.TestCaseFactory(job=job4, name=default_testcase.name, passed=True) resp = client.get("/api/repos/{}/tests-by-build?results=3".format( default_repo.get_full_name())) assert resp.status_code == 200 data = resp.json() assert data["tests"] == [ { "name": default_testcase.name, "hash": default_testcase.hash, "results": ["passed", None, "failed"], }, { "name": testcase2.name, "hash": testcase2.hash, "results": [None, "passed", None], }, ] assert len(data["builds"]) == 3 assert data["builds"][0]["id"] == str(build4.id) assert data["builds"][1]["id"] == str(build3.id) assert data["builds"][2]["id"] == str(build2.id)
def test_test_stats(mocker, db_session, default_revision, default_tenant): build = factories.BuildFactory(revision=default_revision, in_progress=True) db_session.add(build) job = factories.JobFactory(build=build, passed=True) db_session.add(job) job2 = factories.JobFactory(build=build, passed=True) db_session.add(job2) db_session.add( factories.TestCaseFactory(job=job, name="foo", failed=True, duration=8)) db_session.add( factories.TestCaseFactory(job=job, name="bar", passed=True, duration=2)) db_session.add( factories.TestCaseFactory(job=job2, name="bar", failed=True, duration=2)) aggregate_build_stats_for_job(job.id) aggregate_build_stats_for_job(job2.id) aggregate_build_stats(build.id) build_stats = { i.name: i.value for i in ItemStat.query.filter(ItemStat.item_id == build.id) } assert build_stats["tests.count"] == 3 assert build_stats["tests.count_unique"] == 2 assert build_stats["tests.failures"] == 2 assert build_stats["tests.failures_unique"] == 2 assert build_stats["tests.duration"] == 12 job_stats = { i.name: i.value for i in ItemStat.query.filter(ItemStat.item_id == job.id) } assert job_stats["tests.count"] == 2 assert job_stats["tests.failures"] == 1 assert job_stats["tests.duration"] == 10
def test_test_details(client, default_login, default_org, default_project, default_build, default_job, default_repo_access): testcase = factories.TestCaseFactory( job=default_job, failed=True, ) resp = client.get('/api/projects/{}/{}/builds/{}/tests/{}'.format( default_org.name, default_project.name, default_build.number, testcase.name)) assert resp.status_code == 200 data = resp.json() assert data['id'] == str(testcase.id) assert data['message'] == str(testcase.message)
def test_failing_tests(mocker, db_session, default_source): auth.set_current_tenant( auth.Tenant(repository_ids=[default_source.repository_id])) build = factories.BuildFactory(source=default_source, in_progress=True) db_session.add(build) job = factories.JobFactory(build=build, passed=True) db_session.add(job) factories.TestCaseFactory(job=job, failed=True) aggregate_build_stats_for_job(job.id) assert job.result == Result.failed reasons = list(FailureReason.query.filter(FailureReason.job_id == job.id)) assert len(reasons) == 1 assert reasons[0].reason == FailureReason.Code.failing_tests
def test_simple( client, db_session, default_login, default_repo, default_repo_access, default_revision, ): # finished build build = factories.BuildFactory(revision=default_revision, passed=True) db_session.add(build) # an unfinished build which shouldn't be used factories.BuildFactory(revision=default_revision, status=Status.in_progress) db_session.add(build) # a couple of needed jobs that split the tests job1 = factories.JobFactory(build=build, status=Status.finished, result=Result.passed) db_session.add(job1) job2 = factories.JobFactory(build=build, status=Status.finished, result=Result.passed) db_session.add(job2) # and finally our testcases test1 = factories.TestCaseFactory(job=job1, name="foo.bar", duration=50) db_session.add(test1) test2 = factories.TestCaseFactory(job=job1, name="foo.baz", duration=70) db_session.add(test2) test3 = factories.TestCaseFactory(job=job2, name="blah.blah", duration=10) db_session.add(test3) db_session.commit() path = "/api/repos/{}/test-tree".format(default_repo.get_full_name()) resp = client.get(path) assert resp.status_code == 200 data = resp.json() assert len(data["entries"]) == 2 assert data["entries"][0]["name"] == "foo" assert data["entries"][0]["path"] == "foo" assert data["entries"][0]["numTests"] == 2 assert data["entries"][0]["totalDuration"] == 120 assert data["entries"][1]["name"] == "blah.blah" assert data["entries"][1]["path"] == "blah.blah" assert data["entries"][1]["numTests"] == 1 assert data["entries"][1]["totalDuration"] == 10 assert len(data["trail"]) == 0 resp = client.get("{}?parent=foo".format(path)) assert resp.status_code == 200 data = resp.json() assert len(data["entries"]) == 2 assert data["entries"][0]["name"] == "baz" assert data["entries"][0]["path"] == "foo.baz" assert data["entries"][0]["numTests"] == 1 assert data["entries"][0]["totalDuration"] == 70 assert data["entries"][1]["name"] == "bar" assert data["entries"][1]["path"] == "foo.bar" assert data["entries"][1]["numTests"] == 1 assert data["entries"][1]["totalDuration"] == 50 assert len(data["trail"]) == 1 assert data["trail"][0] == {"name": "foo", "path": "foo"}
def default_testcase(default_job): return factories.TestCaseFactory( job=default_job, passed=True, )
def test_simple( client, db_session, default_login, default_repo, default_repo_access, default_revision ): source = factories.SourceFactory( revision=default_revision, ) # finished build build = factories.BuildFactory( source=source, passed=True, ) db_session.add(build) # an unfinished build which shouldn't be used factories.BuildFactory( source=source, status=Status.in_progress, ) db_session.add(build) # a couple of needed jobs that split the tests job1 = factories.JobFactory( build=build, status=Status.finished, result=Result.passed, ) db_session.add(job1) job2 = factories.JobFactory( build=build, status=Status.finished, result=Result.passed, ) db_session.add(job2) # and finally our testcases test1 = factories.TestCaseFactory( job=job1, name='foo.bar', duration=50, ) db_session.add(test1) test2 = factories.TestCaseFactory( job=job1, name='foo.baz', duration=70, ) db_session.add(test2) test3 = factories.TestCaseFactory( job=job2, name='blah.blah', duration=10, ) db_session.add(test3) db_session.commit() path = '/api/repos/{}/{}/test-tree'.format(default_repo.owner_name, default_repo.name) resp = client.get(path) assert resp.status_code == 200 data = resp.json() assert len(data['entries']) == 2 assert data['entries'][0]['name'] == 'foo' assert data['entries'][0]['path'] == 'foo' assert data['entries'][0]['numTests'] == 2 assert data['entries'][0]['totalDuration'] == 120 assert data['entries'][1]['name'] == 'blah.blah' assert data['entries'][1]['path'] == 'blah.blah' assert data['entries'][1]['numTests'] == 1 assert data['entries'][1]['totalDuration'] == 10 assert len(data['trail']) == 0 resp = client.get('{}?parent=foo'.format(path)) assert resp.status_code == 200 data = resp.json() assert len(data['entries']) == 2 assert data['entries'][0]['name'] == 'baz' assert data['entries'][0]['path'] == 'foo.baz' assert data['entries'][0]['numTests'] == 1 assert data['entries'][0]['totalDuration'] == 70 assert data['entries'][1]['name'] == 'bar' assert data['entries'][1]['path'] == 'foo.bar' assert data['entries'][1]['numTests'] == 1 assert data['entries'][1]['totalDuration'] == 50 assert len(data['trail']) == 1 assert data['trail'][0] == { 'name': 'foo', 'path': 'foo', }
def test_failure_origin(default_repo): auth.set_current_tenant( auth.Tenant(access={default_repo.id: Permission.read})) new_revision = factories.RevisionFactory(repository=default_repo) new_source = factories.SourceFactory(revision=new_revision) new_build = factories.BuildFactory(source=new_source, failed=True) new_job = factories.JobFactory(build=new_build, failed=True) new_testcase = factories.TestCaseFactory(job=new_job, failed=True) new_testcase2 = factories.TestCaseFactory(job=new_job, passed=True) old_revision = factories.RevisionFactory( repository=default_repo, date_created=new_revision.date_created - timedelta(hours=1), ) old_source = factories.SourceFactory(revision=old_revision, date_created=new_source.date_created - timedelta(hours=1)) old_build = factories.BuildFactory( source=old_source, failed=True, date_created=new_build.date_created - timedelta(hours=1), ) old_job = factories.JobFactory( build=old_build, failed=True, date_created=new_job.date_created - timedelta(hours=1), ) factories.TestCaseFactory(job=old_job, failed=True, name=new_testcase.name) oldold_revision = factories.RevisionFactory( repository=default_repo, date_created=old_revision.date_created - timedelta(hours=1), ) oldold_source = factories.SourceFactory( revision=oldold_revision, date_created=old_source.date_created - timedelta(hours=1), ) factories.BuildFactory( source=oldold_source, passed=True, date_created=old_build.date_created - timedelta(hours=1), ) schema = AggregateTestCaseSummarySchema(many=True, strict=True, context={"build": new_build}) result = schema.dump([ AggregateTestCase( hash=new_testcase.hash, name=new_testcase.name, runs=[[ str(new_testcase.id), str(new_testcase.job_id), new_testcase.duration, int(new_testcase.result), ]], ), AggregateTestCase( hash=new_testcase2.hash, name=new_testcase2.name, runs=[[ str(new_testcase2.id), str(new_testcase2.job_id), new_testcase2.duration, int(new_testcase2.result), ]], ), ]).data assert len(result) == 2 assert result[0]["hash"] == new_testcase.hash assert result[0]["name"] == new_testcase.name assert result[0]["origin_build"]["id"] == str(old_build.id) assert result[1]["hash"] == new_testcase2.hash assert result[1]["name"] == new_testcase2.name assert result[1]["origin_build"] is None