def test_multi_phase(self):
project = self.create_project()
plan = self.create_plan()
option = ItemOption(
item_id=plan.id,
name='build.expect-tests',
value='1',
)
db.session.add(option)
db.session.commit()
build = self.create_build(project=project)
job = self.create_job(build=build)
jobplan = self.create_job_plan(job, plan)
jobphase = self.create_jobphase(
job=job,
label='setup',
# it's important that the date_created here is actually newer
# than the second phase
date_created=datetime(2013, 9, 19, 22, 17, 24),
date_started=datetime(2013, 9, 19, 22, 15, 24),
)
jobphase2 = self.create_jobphase(
job=job,
label='test',
date_created=datetime(2013, 9, 19, 22, 16, 24),
date_started=datetime(2013, 9, 19, 22, 16, 24),
)
jobstep = self.create_jobstep(jobphase)
jobstep2 = self.create_jobstep(jobphase2)
assert not is_missing_tests(jobstep, jobplan)
assert is_missing_tests(jobstep2, jobplan)
testcase = TestCase(
project_id=project.id,
job_id=job.id,
step_id=jobstep.id,
name='test',
)
db.session.add(testcase)
db.session.commit()
assert not is_missing_tests(jobstep, jobplan)
assert is_missing_tests(jobstep2, jobplan)
testcase = TestCase(
project_id=project.id,
job_id=job.id,
step_id=jobstep2.id,
name='test2',
)
db.session.add(testcase)
db.session.commit()
assert not is_missing_tests(jobstep2, jobplan)
def test_single_phase(self):
project = self.create_project()
plan = self.create_plan()
option = ItemOption(
item_id=plan.id,
name='build.expect-tests',
value='0',
)
db.session.add(option)
db.session.commit()
build = self.create_build(project=project)
job = self.create_job(build=build)
jobplan = self.create_job_plan(job, plan)
jobphase = self.create_jobphase(
job=job,
date_started=datetime(2013, 9, 19, 22, 15, 24),
)
jobstep = self.create_jobstep(jobphase)
jobstep2 = self.create_jobstep(jobphase)
assert not is_missing_tests(jobstep, jobplan)
jobplan.data['snapshot']['options'][option.name] = '1'
db.session.add(jobplan)
db.session.commit()
assert is_missing_tests(jobstep, jobplan)
testcase = TestCase(
project_id=project.id,
job_id=job.id,
step_id=jobstep2.id,
name='test',
)
db.session.add(testcase)
db.session.commit()
assert is_missing_tests(jobstep, jobplan)
testcase = TestCase(
project_id=project.id,
job_id=job.id,
step_id=jobstep.id,
name='test2',
)
db.session.add(testcase)
db.session.commit()
assert not is_missing_tests(jobstep, jobplan)
def save(self, test_list):
step = self.step
job = step.job
project = job.project
# agg_groups_by_id = {}
# create all test cases
for test in test_list:
testcase = TestCase(job=job,
step=step,
name_sha=test.name_sha,
project=project,
suite=test.suite,
name=test.name,
duration=test.duration,
message=test.message,
result=test.result,
date_created=test.date_created,
reruns=test.reruns)
db.session.add(testcase)
db.session.commit()
self._record_test_counts(test_list)
self._record_test_duration(test_list)
self._record_test_rerun_counts(test_list)
def save(self, test_list):
if not test_list:
return
step = self.step
job = step.job
project = job.project
# agg_groups_by_id = {}
# create all test cases
for test in test_list:
testcase = TestCase(job=job,
step=step,
name_sha=test.name_sha,
project=project,
name=test.name,
duration=test.duration,
message=test.message,
result=test.result,
date_created=test.date_created,
reruns=test.reruns)
db.session.add(testcase)
db.session.commit()
try:
self._record_test_counts(test_list)
self._record_test_failures(test_list)
self._record_test_duration(test_list)
self._record_test_rerun_counts(test_list)
except Exception:
logger.exception('Failed to record aggregate test statistics')
def _normalize_test_segments(cls, test_name):
sep = TestCase(name=test_name).sep
segments = test_name.split(sep)
# kill the file extension
if sep is '/' and '.' in segments[-1]:
segments[-1] = segments[-1].rsplit('.', 1)[0]
return tuple(segments)
def test_simple(self):
project = self.create_project()
previous_build = self.create_build(project)
previous_job = self.create_job(previous_build)
changed_test = TestCase(
job=previous_job,
project=previous_job.project,
name='unchanged test',
)
removed_test = TestCase(
job=previous_job,
project=previous_job.project,
name='removed test',
)
db.session.add(removed_test)
db.session.add(changed_test)
current_build = self.create_build(project)
current_job = self.create_job(current_build)
added_test = TestCase(
job=current_job,
project=current_job.project,
name='added test',
)
db.session.add(added_test)
db.session.add(
TestCase(
job=current_job,
project=current_job.project,
name='unchanged test',
))
db.session.commit()
results = find_changed_tests(current_build, previous_build)
assert results['total'] == 2
assert ('-', removed_test) in results['changes']
assert ('+', added_test) in results['changes']
assert len(results['changes']) == 2
def create_test(self, job, **kwargs):
kwargs.setdefault('name', uuid4().hex)
case = TestCase(job=job,
project=job.project,
project_id=job.project_id,
job_id=job.id,
**kwargs)
db.session.add(case)
db.session.commit()
return case
def get_test_stats(cls, project_slug):
response = api_client.get(
'/projects/{project}/'.format(project=project_slug))
last_build = response['lastPassingBuild']
if not last_build:
# use a failing build if no build has passed yet
last_build = response['lastBuild']
if not last_build:
return {}, 0
# XXX(dcramer): ideally this would be abstracted via an API
job_list = db.session.query(Job.id).filter(
Job.build_id == last_build['id'], )
test_durations = dict(
db.session.query(TestCase.name, TestCase.duration).filter(
TestCase.job_id.in_(job_list), ))
test_names = []
total_count, total_duration = 0, 0
for test in test_durations:
test_names.append(test)
total_duration += test_durations[test]
total_count += 1
test_stats = {}
if test_names:
sep = TestCase(name=test_names[0]).sep
tree = build_flat_tree(test_names, sep=sep)
for group_name, group_tests in tree.iteritems():
segments = cls._normalize_test_segments(group_name)
test_stats[segments] = sum(test_durations[t]
for t in group_tests)
# the build report can contain different test suites so this isnt the
# most accurate
if total_duration > 0:
avg_test_time = int(total_duration / total_count)
else:
avg_test_time = 0
return test_stats, avg_test_time
def get(self, project_id):
project = Project.get(project_id)
if not project:
return '', 404
args = self.parser.parse_args()
latest_job = Job.query.join(
Source, Source.id == Job.source_id,
).filter(
Source.patch_id == None, # NOQA
Job.project_id == project.id,
Job.result == Result.passed,
Job.status == Status.finished,
).order_by(
Job.date_created.desc(),
).limit(1).first()
if not latest_job:
return self.respond([])
# use the most recent test
test_list = db.session.query(
TestCase.name, TestCase.duration
).filter(
TestCase.project_id == project_id,
TestCase.job_id == latest_job.id,
)
if args.parent:
test_list = test_list.filter(
TestCase.name.startswith(args.parent),
)
test_list = list(test_list)
if test_list:
sep = TestCase(name=test_list[0][0]).sep
groups = build_tree(
[t[0] for t in test_list],
sep=sep,
min_children=2,
parent=args.parent,
)
results = []
for group in groups:
num_tests = 0
total_duration = 0
for name, duration in test_list:
if name == group or name.startswith(group + sep):
num_tests += 1
total_duration += duration
if args.parent:
name = group[len(args.parent) + len(sep):]
else:
name = group
data = {
'name': name,
'path': group,
'totalDuration': total_duration,
'numTests': num_tests,
}
results.append(data)
results.sort(key=lambda x: x['totalDuration'], reverse=True)
trail = []
context = []
if args.parent:
for chunk in args.parent.split(sep):
context.append(chunk)
trail.append({
'path': sep.join(context),
'name': chunk,
})
else:
results = []
trail = []
data = {
'groups': results,
'trail': trail,
}
return self.respond(data, serialize=False)
def name_sha(self):
return TestCase.calculate_name_sha(self.name)
def test_finished(self, get_implementation, queue_delay):
implementation = mock.Mock()
get_implementation.return_value = implementation
def mark_finished(step):
step.status = Status.finished
step.result = Result.failed
implementation.update_step.side_effect = mark_finished
project = self.create_project()
build = self.create_build(project=project)
job = self.create_job(build=build)
plan = self.create_plan()
self.create_step(plan, implementation='test', order=0)
self.create_job_plan(job, plan)
phase = self.create_jobphase(job)
step = self.create_jobstep(phase)
task = self.create_task(
parent_id=job.id,
task_id=step.id,
task_name='sync_job_step',
status=Status.finished,
)
db.session.add(
TestCase(
name='test',
step_id=step.id,
job_id=job.id,
project_id=project.id,
result=Result.failed,
))
db.session.add(
FileCoverage(
job=job,
step=step,
project=job.project,
filename='foo.py',
data='CCCUUUCCCUUNNN',
lines_covered=6,
lines_uncovered=5,
diff_lines_covered=3,
diff_lines_uncovered=2,
))
db.session.commit()
sync_job_step(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=job.id.hex,
)
get_implementation.assert_called_once_with()
implementation.update_step.assert_called_once_with(step=step)
db.session.expire(step)
db.session.expire(task)
step = JobStep.query.get(step.id)
assert step.status == Status.finished
task = Task.query.get(task.id)
assert task.status == Status.finished
assert len(queue_delay.mock_calls) == 0
stat = ItemStat.query.filter(
ItemStat.item_id == step.id,
ItemStat.name == 'tests_missing',
).first()
assert stat.value == 0
stat = ItemStat.query.filter(
ItemStat.item_id == step.id,
ItemStat.name == 'lines_covered',
).first()
assert stat.value == 6
stat = ItemStat.query.filter(
ItemStat.item_id == step.id,
ItemStat.name == 'lines_uncovered',
).first()
assert stat.value == 5
stat = ItemStat.query.filter(
ItemStat.item_id == step.id,
ItemStat.name == 'diff_lines_covered',
).first()
assert stat.value == 3
stat = ItemStat.query.filter(
ItemStat.item_id == step.id,
ItemStat.name == 'diff_lines_uncovered',
).first()
assert stat.value == 2
assert FailureReason.query.filter(
FailureReason.step_id == step.id,
FailureReason.reason == 'test_failures',
)
def name_sha(self):
return TestCase.calculate_name_sha(self.name)
def get(self, project_id):
project = Project.get(project_id)
if not project:
return '', 404
args = self.parser.parse_args()
latest_build = Build.query.join(
Source,
Source.id == Build.source_id,
).filter(
Source.patch_id == None, # NOQA
Build.project_id == project.id,
Build.result == Result.passed,
Build.status == Status.finished,
).order_by(Build.date_created.desc(), ).limit(1).first()
if not latest_build:
return self.respond({})
job_list = db.session.query(Job.id).filter(
Job.build_id == latest_build.id, )
# use the most recent test
test_list = db.session.query(TestCase.name, TestCase.duration).filter(
TestCase.job_id.in_(job_list), )
if args.parent:
test_list = test_list.filter(TestCase.name.startswith(
args.parent), )
test_list = list(test_list)
if test_list:
sep = TestCase(name=test_list[0][0]).sep
groups = build_tree(
[t[0] for t in test_list],
sep=sep,
min_children=2,
parent=args.parent,
)
results = []
for group in groups:
num_tests = 0
total_duration = 0
for name, duration in test_list:
if name == group or name.startswith(group + sep):
num_tests += 1
total_duration += duration
if args.parent:
name = group[len(args.parent) + len(sep):]
else:
name = group
data = {
'name': name,
'path': group,
'totalDuration': total_duration,
'numTests': num_tests,
}
results.append(data)
results.sort(key=lambda x: x['totalDuration'], reverse=True)
trail = []
context = []
if args.parent:
for chunk in args.parent.split(sep):
context.append(chunk)
trail.append({
'path': sep.join(context),
'name': chunk,
})
else:
results = []
trail = []
options = dict((o.name, o.value) for o in ProjectOption.query.filter(
ProjectOption.project_id == project.id,
ProjectOption.name == 'build.test-duration-warning',
))
over_threshold_duration = options.get('build.test-duration-warning')
if over_threshold_duration:
over_threshold_count = TestCase.query.filter(
TestCase.project_id == project_id,
TestCase.job_id.in_(job_list),
TestCase.duration >= over_threshold_duration,
).count()
else:
over_threshold_count = 0
data = {
'groups': results,
'trail': trail,
'overThreshold': {
'count': over_threshold_count,
'duration': over_threshold_duration,
}
}
return self.respond(data, serialize=False)
def get(self, project_id):
project = Project.get(project_id)
if not project:
return '', 404
args = self.parser.parse_args()
latest_job = Job.query.join(
Source,
Source.id == Job.source_id,
).filter(
Source.patch_id == None, # NOQA
Job.project_id == project.id,
Job.result == Result.passed,
Job.status == Status.finished,
).order_by(Job.date_created.desc(), ).limit(1).first()
if not latest_job:
return self.respond([])
# use the most recent test
test_list = db.session.query(TestCase.name, TestCase.duration).filter(
TestCase.project_id == project_id,
TestCase.job_id == latest_job.id,
)
if args.parent:
test_list = test_list.filter(TestCase.name.startswith(
args.parent), )
test_list = list(test_list)
if test_list:
sep = TestCase(name=test_list[0][0]).sep
groups = build_tree(
[t[0] for t in test_list],
sep=sep,
min_children=2,
parent=args.parent,
)
results = []
for group in groups:
num_tests = 0
total_duration = 0
for name, duration in test_list:
if name == group or name.startswith(group + sep):
num_tests += 1
total_duration += duration
if args.parent:
name = group[len(args.parent) + len(sep):]
else:
name = group
data = {
'name': name,
'path': group,
'totalDuration': total_duration,
'numTests': num_tests,
}
results.append(data)
results.sort(key=lambda x: x['totalDuration'], reverse=True)
trail = []
context = []
if args.parent:
for chunk in args.parent.split(sep):
context.append(chunk)
trail.append({
'path': sep.join(context),
'name': chunk,
})
else:
results = []
trail = []
data = {
'groups': results,
'trail': trail,
}
return self.respond(data, serialize=False)
def test_simple(self):
fake_id = uuid4()
project = self.create_project()
build = self.create_build(project, result=Result.failed)
job = self.create_job(build, status=Status.finished)
test = TestCase(
job=job,
project=project,
name='foo',
name_sha='a' * 40,
)
db.session.add(test)
path = '/api/0/builds/{0}/tests/'.format(fake_id.hex)
resp = self.client.get(path)
assert resp.status_code == 404
# test each sort option just to ensure it doesnt straight up fail
path = '/api/0/builds/{0}/tests/?sort=duration'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
assert data[0]['id'] == test.id.hex
path = '/api/0/builds/{0}/tests/?sort=name'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
assert data[0]['id'] == test.id.hex
path = '/api/0/builds/{0}/tests/?sort=retries'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
assert data[0]['id'] == test.id.hex
path = '/api/0/builds/{0}/tests/?per_page='.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
assert data[0]['id'] == test.id.hex
path = '/api/0/builds/{0}/tests/?query=foo'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
assert data[0]['id'] == test.id.hex
path = '/api/0/builds/{0}/tests/?query=bar'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 0
path = '/api/0/builds/{0}/tests/?result='.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
path = '/api/0/builds/{0}/tests/?result=passed'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 0
def get(self, project_id):
project = Project.get(project_id)
if not project:
return '', 404
args = self.parser.parse_args()
latest_build = Build.query.join(
Source, Source.id == Build.source_id,
).filter(
Source.patch_id == None, # NOQA
Build.project_id == project.id,
Build.result == Result.passed,
Build.status == Status.finished,
).order_by(
Build.date_created.desc(),
).limit(1).first()
if not latest_build:
return self.respond({})
job_list = db.session.query(Job.id).filter(
Job.build_id == latest_build.id,
)
# use the most recent test
test_list = db.session.query(
TestCase.name, TestCase.duration
).filter(
TestCase.job_id.in_(job_list),
)
if args.parent:
test_list = test_list.filter(
TestCase.name.startswith(args.parent),
)
test_list = list(test_list)
if test_list:
sep = TestCase(name=test_list[0][0]).sep
groups = build_tree(
[t[0] for t in test_list],
sep=sep,
min_children=2,
parent=args.parent,
)
results = []
for group in groups:
num_tests = 0
total_duration = 0
for name, duration in test_list:
if name == group or name.startswith(group + sep):
num_tests += 1
total_duration += duration
if args.parent:
name = group[len(args.parent) + len(sep):]
else:
name = group
data = {
'name': name,
'path': group,
'totalDuration': total_duration,
'numTests': num_tests,
}
results.append(data)
results.sort(key=lambda x: x['totalDuration'], reverse=True)
trail = []
context = []
if args.parent:
for chunk in args.parent.split(sep):
context.append(chunk)
trail.append({
'path': sep.join(context),
'name': chunk,
})
else:
results = []
trail = []
options = dict(
(o.name, o.value) for o in ProjectOption.query.filter(
ProjectOption.project_id == project.id,
ProjectOption.name == 'build.test-duration-warning',
)
)
over_threshold_duration = options.get('build.test-duration-warning')
if over_threshold_duration:
over_threshold_count = TestCase.query.filter(
TestCase.project_id == project_id,
TestCase.job_id.in_(job_list),
TestCase.duration >= over_threshold_duration,
).count()
else:
over_threshold_count = 0
data = {
'groups': results,
'trail': trail,
'overThreshold': {
'count': over_threshold_count,
'duration': over_threshold_duration,
}
}
return self.respond(data, serialize=False)