def test_get_build_failure_reasons_multiple_failures(self):
project = self.create_project(name='test', slug='project-slug')
build = self.create_build(project,
result=Result.failed,
target='D1',
label='Some sweet diff')
job = self.create_job(build=build, result=Result.failed)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase,
status=Status.finished,
result=Result.failed)
for reason in ('missing_tests', 'timeout', 'aborted'):
db.session.add(
FailureReason(step_id=jobstep.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason=reason))
jobstep2 = self.create_jobstep(jobphase,
status=Status.finished,
result=Result.failed)
for reason in ('timeout', 'insufficient_politeness'):
db.session.add(
FailureReason(step_id=jobstep2.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason=reason))
db.session.commit()
self.assertEquals(
_get_build_failure_reasons(build),
['aborted', 'insufficient_politeness', 'missing_tests', 'timeout'])
def test_simple(self):
project = self.create_project()
previous_build = self.create_build(project,
date_created=datetime(
2013, 9, 19, 22, 15, 23),
status=Status.finished)
build = self.create_build(project,
date_created=datetime(
2013, 9, 19, 22, 15, 24))
job1 = self.create_job(build)
job2 = self.create_job(build)
phase = self.create_jobphase(job1)
step = self.create_jobstep(phase)
db.session.add(
Event(
item_id=build.id,
type='green_build_notification',
))
db.session.add(
ItemStat(
item_id=build.id,
name='test_failures',
value=2,
))
db.session.add(
FailureReason(project_id=project.id,
build_id=build.id,
job_id=job1.id,
step_id=step.id,
reason='test_failures'))
db.session.commit()
path = '/api/0/builds/{0}/'.format(build.id.hex)
resp = self.client.get(path)
assert resp.status_code == 200
data = self.unserialize(resp)
assert data['id'] == build.id.hex
assert len(data['jobs']) == 2
assert data['jobs'][0]['id'] == job1.id.hex
assert data['jobs'][1]['id'] == job2.id.hex
assert data['seenBy'] == []
assert data['testFailures']['total'] == 0
assert data['testFailures']['tests'] == []
assert data['testChanges'] == []
assert len(data['events']) == 1
assert len(data['failures']) == 1
assert data['failures'][0] == {
'id':
'test_failures',
'reason':
'There were <a href="http://example.com/projects/{0}/builds/{1}/tests/?result=failed">2 failing tests</a>.'
.format(
project.slug,
build.id.hex,
),
'count':
1,
}
def test_failed_build(self, post_fn):
URL = "https://analytics.example.com/report?source=changes"
self._set_config_url(URL)
project = self.create_project(name='test', slug='project-slug')
self.assertEquals(post_fn.call_count, 0)
duration = 1234
created = 1424998888
started = created + 10
finished = started + duration
build = self.create_build(project,
result=Result.failed,
target='D1',
label='Some sweet diff',
duration=duration,
date_created=ts_to_datetime(created),
date_started=ts_to_datetime(started),
date_finished=ts_to_datetime(finished))
job = self.create_job(build=build, result=Result.failed)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase,
status=Status.finished,
result=Result.failed)
db.session.add(
FailureReason(step_id=jobstep.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='missing_tests'))
db.session.commit()
with mock.patch(
'changes.listeners.analytics_notifier._get_phabricator_revision_url'
) as mock_get_phab:
mock_get_phab.return_value = 'https://example.com/D1'
with mock.patch(
'changes.listeners.analytics_notifier._get_build_failure_reasons'
) as mock_get_failures:
mock_get_failures.return_value = ['aborted', 'missing_tests']
build_finished_handler(build_id=build.id.hex)
expected_data = {
'build_id': build.id.hex,
'number': 1,
'target': 'D1',
'project_slug': 'project-slug',
'result': 'Failed',
'label': 'Some sweet diff',
'is_commit': True,
'duration': 1234,
'date_created': created,
'date_started': started,
'date_finished': finished,
'phab_revision_url': 'https://example.com/D1',
'failure_reasons': ['aborted', 'missing_tests'],
}
post_fn.assert_called_once_with(URL, [expected_data])
def _add_failure_reason(self):
db.session.add(
FailureReason(step_id=self.step.id,
job_id=self.step.job_id,
build_id=self.step.job.build_id,
project_id=self.step.project_id,
reason='malformed_manifest_json'))
self.step.result = Result.infra_failed
db.session.add(self.step)
db.session.commit()
def test_get_job_failure_reasons_by_jobstep_failures(self):
project = self.create_project(name='test', slug='project-slug')
build = self.create_build(project,
result=Result.failed,
target='D1',
label='Some sweet diff')
job = self.create_job(build=build, result=Result.failed)
jobphase = self.create_jobphase(job)
jobstep_a = self.create_jobstep(jobphase,
status=Status.finished,
result=Result.failed,
label='Step A')
jobstep_b = self.create_jobstep(jobphase,
status=Status.finished,
result=Result.failed,
label='Step B')
db.session.add(
FailureReason(step_id=jobstep_a.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='missing_tests'))
db.session.add(
FailureReason(step_id=jobstep_a.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='aborted'))
db.session.add(
FailureReason(step_id=jobstep_b.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='aborted'))
db.session.commit()
expected_data = defaultdict(list)
expected_data[jobstep_a.id] = ['aborted', 'missing_tests']
expected_data[jobstep_b.id] = ['aborted']
self.assertEquals(_get_job_failure_reasons_by_jobstep(job),
expected_data)
def verify_final_artifacts(self, step, artifacts):
# If the Jenkins run was aborted or timed out, we don't expect a manifest file.
if (step.result != Result.aborted and
not step.data.get('timed_out', False) and
not any(ManifestJsonHandler.can_process(a.name) for a in artifacts)):
db.session.add(FailureReason(
step_id=step.id,
job_id=step.job.id,
build_id=step.job.build_id,
project_id=step.job.project_id,
reason='missing_manifest_json',
))
step.result = Result.infra_failed
db.session.add(step)
db.session.commit()
def test_failure_reasons(self, get_implementation):
# Simulate test type which doesn't interact with artifacts store.
responses.add(responses.GET,
SyncJobStepTest.ARTIFACTSTORE_REQUEST_RE,
body='',
status=404)
implementation = mock.Mock()
get_implementation.return_value = implementation
project = self.create_project()
build = self.create_build(project=project)
job = self.create_job(build=build)
plan = self.create_plan(project)
self.create_step(plan, implementation='test', order=0)
self.create_job_plan(job, plan)
phase = self.create_jobphase(job)
step = self.create_jobstep(phase,
status=Status.finished,
result=Result.passed)
db.session.add(
FailureReason(step_id=step.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='missing_manifest_json'))
db.session.commit()
with mock.patch.object(sync_job_step, 'allow_absent_from_db', True):
sync_job_step(step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=job.id.hex)
assert step.result == Result.infra_failed
def post(self, step_id):
jobstep = JobStep.query.options(joinedload(
'project', innerjoin=True), ).get(step_id)
if jobstep is None:
return '', 404
args = self.post_parser.parse_args()
current_datetime = args.date or datetime.utcnow()
if args.result:
jobstep.result = Result[args.result]
if args.status:
jobstep.status = Status[args.status]
# if we've finished this job, lets ensure we have set date_finished
if jobstep.status == Status.finished and jobstep.date_finished is None:
jobstep.date_finished = current_datetime
elif jobstep.status != Status.finished and jobstep.date_finished:
jobstep.date_finished = None
if jobstep.status != Status.queued and jobstep.date_started is None:
jobstep.date_started = current_datetime
elif jobstep.status == Status.queued and jobstep.date_started:
jobstep.date_started = None
if args.node:
node, _ = get_or_create(Node, where={
'label': args.node,
})
jobstep.node_id = node.id
# we want to guarantee that even if the jobstep seems to succeed, that
# we accurately reflect what we internally would consider a success state
if jobstep.result == Result.passed and jobstep.status == Status.finished:
last_command = Command.query.filter(
Command.jobstep_id == jobstep.id, ).order_by(
Command.order.desc()).first()
if not last_command:
pass
elif last_command.status != Status.finished:
jobstep.result = Result.failed
elif last_command.return_code != 0:
jobstep.result = Result.failed
# are we missing an expansion step? it must happen before reporting
# the result, and would falsely give us a success metric
elif last_command.type.is_collector() and self._is_final_jobphase(
jobstep.phase):
jobstep.result = Result.failed
job = jobstep.job
# TODO(dcramer): we should add a better failure reason
db.session.add(
FailureReason(
step_id=jobstep.id,
job_id=job.id,
build_id=job.build_id,
project_id=job.project_id,
reason='missing_artifact',
))
db.session.add(jobstep)
if db.session.is_modified(jobstep):
db.session.commit()
# TODO(dcramer): this is a little bit hacky, but until we can entirely
# move to push APIs we need a good way to handle the existing sync
job = jobstep.job
sync_job.delay_if_needed(
task_id=job.id.hex,
parent_task_id=job.id.hex,
job_id=job.build_id.hex,
)
return self.respond(jobstep)
def job(build, change=None, **kwargs):
kwargs.setdefault('project', build.project)
kwargs.setdefault('label', get_sentences(1)[0][:128])
kwargs.setdefault('status', Status.finished)
kwargs.setdefault('result', Result.passed)
kwargs.setdefault('duration', random.randint(10000, 100000))
kwargs['source'] = build.source
kwargs['source_id'] = kwargs['source'].id
kwargs['project_id'] = kwargs['project'].id
kwargs['build_id'] = build.id
if change:
kwargs['change_id'] = change.id
job = Job(
build=build,
change=change,
**kwargs
)
db.session.add(job)
node, created = get_or_create(Node, where={
'label': get_sentences(1)[0][:32],
})
if created:
cluster, _ = get_or_create(Cluster, where={
'label': get_sentences(1)[0][:32],
})
clusternode = ClusterNode(cluster=cluster, node=node)
db.session.add(clusternode)
jobplan = JobPlan.build_jobplan(plan(build.project), job)
db.session.add(jobplan)
phase1_setup = JobPhase(
project=job.project, job=job,
date_started=job.date_started,
date_finished=job.date_finished,
status=Status.finished, result=Result.passed, label='Setup',
)
db.session.add(phase1_setup)
phase1_compile = JobPhase(
project=job.project, job=job,
date_started=job.date_started,
date_finished=job.date_finished,
status=Status.finished, result=Result.passed, label='Compile',
)
db.session.add(phase1_compile)
phase1_test = JobPhase(
project=job.project, job=job,
date_started=job.date_started,
date_finished=job.date_finished,
status=kwargs['status'], result=kwargs['result'], label='Test',
)
db.session.add(phase1_test)
step = JobStep(
project=job.project, job=job,
phase=phase1_setup, status=phase1_setup.status, result=phase1_setup.result,
label='Setup', node=node,
)
db.session.add(step)
command = Command(
jobstep=step,
script="echo 1",
label="echo 1",
)
db.session.add(command)
step = JobStep(
project=job.project, job=job,
phase=phase1_compile, status=phase1_compile.status, result=phase1_compile.result,
label='Compile', node=node,
)
db.session.add(step)
command = Command(
jobstep=step,
script="echo 2",
label="echo 2",
)
db.session.add(command)
step = JobStep(
project=job.project, job=job,
phase=phase1_test, status=phase1_test.status, result=phase1_test.result,
label=TEST_STEP_LABELS.next(), node=node,
)
db.session.add(step)
command = Command(
jobstep=step,
script="echo 3",
label="echo 3",
)
db.session.add(command)
step = JobStep(
project=job.project, job=job,
phase=phase1_test, status=phase1_test.status, result=phase1_test.result,
label=TEST_STEP_LABELS.next(), node=node,
)
db.session.add(step)
command = Command(
jobstep=step,
script="echo 4",
label="echo 4",
)
db.session.add(command)
if phase1_test.result == Result.failed:
db.session.add(FailureReason(
reason='test_failures',
build_id=build.id,
job_id=job.id,
step_id=step.id,
project_id=job.project_id
))
return job
def test_failed_job(self, post_fn):
URL = "https://analytics.example.com/report?source=changes_jobstep"
self._set_config_url(build_url=None, jobstep_url=URL)
project = self.create_project(name='test', slug='project-slug')
self.assertEquals(post_fn.call_count, 0)
duration = 1234
created = 1424998888
started = created + 10
finished = started + duration
build = self.create_build(project,
result=Result.failed,
target='D1',
label='Some sweet diff',
duration=duration,
date_created=ts_to_datetime(created),
date_started=ts_to_datetime(started),
date_finished=ts_to_datetime(finished))
job = self.create_job(build=build, result=Result.failed)
jobphase = self.create_jobphase(job)
node = self.create_node()
jobstep = self.create_jobstep(jobphase,
status=Status.finished,
result=Result.failed,
label='Step 1',
date_created=ts_to_datetime(created),
date_started=ts_to_datetime(started),
date_finished=ts_to_datetime(finished),
node_id=node.id)
db.session.add(
FailureReason(step_id=jobstep.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='missing_tests'))
db.session.add(
FailureReason(step_id=jobstep.id,
job_id=job.id,
build_id=build.id,
project_id=project.id,
reason='aborted'))
db.session.commit()
with mock.patch(
'changes.listeners.analytics_notifier._get_job_failure_reasons_by_jobstep'
) as mock_get_failures:
mock_get_failures.return_value = defaultdict(list)
mock_get_failures.return_value[jobstep.id] = [
'aborted', 'missing_tests'
]
job_finished_handler(job_id=job.id.hex)
expected_data = {
'jobstep_id': jobstep.id.hex,
'phase_id': jobphase.id.hex,
'build_id': build.id.hex,
'job_id': job.id.hex,
'result': 'Failed',
'label': 'Step 1',
'data': {},
'date_created': created,
'date_started': started,
'date_finished': finished,
'failure_reasons': ['aborted', 'missing_tests'],
'log_categories': [],
}
post_fn.assert_called_once_with(URL, [expected_data])
json.dumps(post_fn.call_args[0][1])
