def test_aggregate_result():
status_list = [Result.passed, Result.failed, Result.unknown]
assert aggregate_result(status_list) == Result.failed
status_list = [Result.passed, Result.unknown]
assert aggregate_result(status_list) == Result.unknown
status_list = [Result.passed, Result.skipped]
assert aggregate_result(status_list) == Result.passed
def test_aggregate_result():
status_list = [Result.passed, Result.failed, Result.unknown]
assert aggregate_result(status_list) == Result.failed
status_list = [Result.passed, Result.unknown]
assert aggregate_result(status_list) == Result.passed
status_list = [Result.passed, Result.skipped]
assert aggregate_result(status_list) == Result.passed
def _deduplicate_testresults(results):
"""Combine TestResult objects until every package+name is unique.
The traditions surrounding jUnit do not prohibit a single test from
producing two or more <testcase> elements. In fact, py.test itself
will produce two such elements for a single test if the test both
fails and then hits an error during tear-down. To impedance-match
this situation with the Changes constraint of one result per test,
we combine <testcase> elements that belong to the same test.
"""
result_dict = {}
deduped = []
for result in results:
key = (result.package, result.name)
existing_result = result_dict.get(key)
if existing_result is not None:
e, r = existing_result, result
e.duration = _careful(add, e.duration, r.duration)
e.result = aggregate_result((e.result, r.result))
if e.message and r.message:
e.message += '\n\n' + r.message
elif not e.message:
e.message = r.message or ''
e.reruns = _careful(max, e.reruns, r.reruns)
e.artifacts = _careful(add, e.artifacts, r.artifacts)
e.message_offsets = _careful(add, e.message_offsets, r.message_offsets)
else:
result_dict[key] = result
deduped.append(result)
return deduped
def sync_phase(phase):
phase_steps = list(phase.steps)
if phase.date_started is None:
phase.date_started = safe_agg(min,
(s.date_started for s in phase_steps))
db.session.add(phase)
if phase_steps:
if all(s.status == Status.finished for s in phase_steps):
phase.status = Status.finished
phase.date_finished = safe_agg(max, (s.date_finished
for s in phase_steps))
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((s.status for s in phase_steps))
if new_status != Status.finished:
phase.status = new_status
if any(s.result is Result.failed for s in phase_steps):
phase.result = Result.failed
elif phase.status == Status.finished:
phase.result = aggregate_result((s.result for s in phase_steps))
if db.session.is_modified(phase):
phase.date_modified = datetime.utcnow()
db.session.add(phase)
db.session.commit()
def process(self, fp, artifact):
target_name = self._get_target_name(artifact)
target, _ = get_or_create(BazelTarget, where={
'step_id': self.step.id,
'job_id': self.step.job.id,
'name': target_name,
'result_source': ResultSource.from_self,
})
test_suites = self.get_test_suites(fp)
tests = self.aggregate_tests_from_suites(test_suites)
manager = TestResultManager(self.step, artifact)
manager.save(tests)
# update target metadata
# TODO handle multiple files per target, i.e. sharding and running multiple times
target.status = Status.finished
target.result = aggregate_result([t.result for t in tests])
duration = 0
for t in test_suites:
if t.duration is None:
duration = None
break
duration += t.duration
target.duration = duration
target.date_created = min([t.date_created for t in test_suites])
db.session.add(target)
db.session.commit()
return tests
def _deduplicate_testresults(results):
"""Combine TestResult objects until every package+name is unique.
The traditions surrounding jUnit do not prohibit a single test from
producing two or more <testcase> elements. In fact, py.test itself
will produce two such elements for a single test if the test both
fails and then hits an error during tear-down. To impedance-match
this situation with the Changes constraint of one result per test,
we combine <testcase> elements that belong to the same test.
"""
result_dict = {}
deduped = []
for result in results:
key = (result.package, result.name)
existing_result = result_dict.get(key)
if existing_result is not None:
e, r = existing_result, result
e.duration = _careful(add, e.duration, r.duration)
e.result = aggregate_result((e.result, r.result))
e.message += '\n\n' + r.message
e.reruns = _careful(max, e.reruns, r.reruns)
e.artifacts = _careful(add, e.artifacts, r.artifacts)
else:
result_dict[key] = result
deduped.append(result)
return deduped
def sync_phase(phase):
phase_steps = list(phase.steps)
if phase.date_started is None:
phase.date_started = safe_agg(min, (s.date_started for s in phase_steps))
db.session.add(phase)
if phase_steps:
if all(s.status == Status.finished for s in phase_steps):
phase.status = Status.finished
phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps))
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((s.status for s in phase_steps))
if new_status != Status.finished:
phase.status = new_status
if any(s.result is Result.failed for s in phase_steps):
phase.result = Result.failed
elif phase.status == Status.finished:
phase.result = aggregate_result((s.result for s in phase_steps))
if db.session.is_modified(phase):
phase.date_modified = datetime.utcnow()
db.session.add(phase)
db.session.commit()
def process(self, fp, artifact):
target_name = self._get_target_name(artifact)
target, _ = get_or_create(BazelTarget,
where={
'step_id': self.step.id,
'job_id': self.step.job.id,
'name': target_name,
'result_source': ResultSource.from_self,
})
test_suites = self.get_test_suites(fp)
tests = self.aggregate_tests_from_suites(test_suites)
manager = TestResultManager(self.step, artifact)
manager.save(tests)
# update target metadata
# TODO handle multiple files per target, i.e. sharding and running multiple times
target.status = Status.finished
target.result = aggregate_result([t.result for t in tests])
duration = 0
for t in test_suites:
if t.duration is None:
duration = None
break
duration += t.duration
target.duration = duration
target.date_created = min([t.date_created for t in test_suites])
db.session.add(target)
db.session.commit()
return tests
def validate_phase(self, phase):
"""Called when a job phase is ready to be finished.
This is responsible for setting the phases's final result. We verify
that the proper number of steps were created in the second (i.e.
expanded) phase."""
phase.result = aggregate_result([s.result for s in phase.steps] +
[self._validate_shards(phase.steps)])
def validate_phase(self, phase):
"""Called when a job phase is ready to be finished.
This is responsible for setting the phases's final result. We verify
that the proper number of steps were created in the second (i.e.
expanded) phase."""
phase.result = aggregate_result([s.result for s in phase.steps] +
[self._validate_shards(phase.steps)])
def validate(self, job):
"""Called when a job is ready to be finished.
This is responsible for setting the job's final result. The base
implementation simply aggregates the phase results.
Args:
job (Job): The job being finished.
Returns:
None
"""
# TODO(josiah): ideally, we could record a FailureReason.
# Currently FailureReason must be per-step.
job.result = aggregate_result((p.result for p in job.phases))
def validate_phase(self, phase):
"""Called when a job phase is ready to be finished.
This is responsible for setting the phases's final result. The base
implementation simply aggregates the jobstep results.
Args:
phase (JobPhase): The phase being finished.
Returns:
None
"""
# TODO(josiah): ideally, we could record a FailureReason.
# Currently FailureReason must be per-step.
phase.result = aggregate_result((s.result for s in phase.steps))
def validate(self, job):
"""Called when a job is ready to be finished.
This is responsible for setting the job's final result. The base
implementation simply aggregates the phase results.
Args:
job (Job): The job being finished.
Returns:
None
"""
# TODO(josiah): ideally, we could record a FailureReason.
# Currently FailureReason must be per-step.
job.result = aggregate_result((p.result for p in job.phases))
def validate_phase(self, phase):
"""Called when a job phase is ready to be finished.
This is responsible for setting the phases's final result. The base
implementation simply aggregates the jobstep results.
Args:
phase (JobPhase): The phase being finished.
Returns:
None
"""
# TODO(josiah): ideally, we could record a FailureReason.
# Currently FailureReason must be per-step.
phase.result = aggregate_result((s.result for s in phase.current_steps))
def end(self, tag):
if self._is_message:
self.close_message()
if tag == 'testsuites':
pass
elif tag == 'testsuite':
self.test_suites[-1].test_results = _deduplicate_testresults(self.test_suites[-1].test_results)
if self.test_suites[-1].duration is None:
# NOTE: it is inaccurate to just sum up the duration of individual
# tests, because tests may be run in parallel
self.logger.warning('Test suite does not have timing information; (step=%s, build=%s)',
self.step.id.hex, self.step.job.build_id.hex)
if len(self.test_suites[-1].test_results) > 0:
self.test_suites[-1].result = aggregate_result([t.result for t in self.test_suites[-1].test_results])
if self.test_suites[-1].date_created is None:
self.test_suites[-1].date_created = min([t.date_created for t in self.test_suites[-1].test_results])
else:
if self.test_suites[-1].date_created is None:
self.test_suites[-1].date_created = datetime.utcnow()
elif tag == 'testcase':
if self._current_result.result == Result.unknown:
# Default result is passing
self._current_result.result = Result.passed
if self._test_is_quarantined:
if self._current_result.result == Result.passed:
self._current_result.result = Result.quarantined_passed
elif self._current_result.result == Result.failed:
self._current_result.result = Result.quarantined_failed
elif self._current_result.result == Result.skipped:
self._current_result.result = Result.quarantined_skipped
self._test_is_quarantined = None
self.test_suites[-1].test_results.append(self._current_result)
self._current_result = None
elif tag == 'test-artifacts':
pass
elif tag == 'artifact':
pass
def sync_job(job_id):
with RCount('sync_job'):
job = Job.query.get(job_id)
if not job:
return
if job.status == Status.finished:
return
# TODO(dcramer): we make an assumption that there is a single step
jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)
try:
implementation.update(job=job)
except UnrecoverableException:
job.status = Status.finished
job.result = Result.aborted
current_app.logger.exception('Unrecoverable exception syncing %s',
job.id)
all_phases = list(job.phases)
# propagate changes to any phases as they live outside of the
# normalize synchronization routines
sync_job_phases(job, all_phases)
is_finished = sync_job.verify_all_children() == Status.finished
if any(p.status != Status.finished for p in all_phases):
is_finished = False
job.date_started = safe_agg(min,
(j.date_started
for j in all_phases if j.date_started))
if is_finished:
job.date_finished = safe_agg(
max, (j.date_finished for j in all_phases if j.date_finished))
else:
job.date_finished = None
if job.date_started and job.date_finished:
job.duration = int(
(job.date_finished - job.date_started).total_seconds() * 1000)
else:
job.duration = None
# if any phases are marked as failing, fail the build
if any(j.result is Result.failed for j in all_phases):
job.result = Result.failed
# if any test cases were marked as failing, fail the build
elif TestCase.query.filter(TestCase.result == Result.failed,
TestCase.job_id == job.id).first():
job.result = Result.failed
# if we've finished all phases, use the best result available
elif is_finished:
job.result = aggregate_result((j.result for j in all_phases))
else:
job.result = Result.unknown
if is_finished:
job.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in all_phases))
if new_status != Status.finished:
job.status = new_status
elif job.status == Status.finished:
job.status = Status.in_progress
current_app.logger.exception(
'Job incorrectly marked as finished: %s', job.id)
if db.session.is_modified(job):
job.date_modified = datetime.utcnow()
db.session.add(job)
db.session.commit()
if not is_finished:
raise sync_job.NotFinished
try:
aggregate_job_stat(job, 'test_count')
aggregate_job_stat(job, 'test_duration')
aggregate_job_stat(job, 'test_failures')
aggregate_job_stat(job, 'test_rerun_count')
aggregate_job_stat(job, 'tests_missing')
aggregate_job_stat(job, 'lines_covered')
aggregate_job_stat(job, 'lines_uncovered')
aggregate_job_stat(job, 'diff_lines_covered')
aggregate_job_stat(job, 'diff_lines_uncovered')
except Exception:
current_app.logger.exception(
'Failing recording aggregate stats for job %s', job.id)
fire_signal.delay(
signal='job.finished',
kwargs={'job_id': job.id.hex},
)
if jobplan:
queue.delay('update_project_plan_stats',
kwargs={
'project_id': job.project_id.hex,
'plan_id': jobplan.plan_id.hex,
},
countdown=1)
def sync_build(build_id):
"""
Synchronizing the build happens continuously until all jobs have reported in
as finished or have failed/aborted.
This task is responsible for:
- Checking in with jobs
- Aborting/retrying them if they're beyond limits
- Aggregating the results from jobs into the build itself
"""
with RCount('sync_build'):
build = Build.query.get(build_id)
if not build:
return
if build.status == Status.finished:
return
all_jobs = list(Job.query.filter(
Job.build_id == build_id,
))
is_finished = sync_build.verify_all_children() == Status.finished
if any(p.status != Status.finished for p in all_jobs):
is_finished = False
build.date_started = safe_agg(
min, (j.date_started for j in all_jobs if j.date_started))
if is_finished:
build.date_finished = safe_agg(
max, (j.date_finished for j in all_jobs if j.date_finished))
else:
build.date_finished = None
if build.date_started and build.date_finished:
build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000)
else:
build.duration = None
if any(j.result is Result.failed for j in all_jobs):
build.result = Result.failed
elif is_finished:
build.result = aggregate_result((j.result for j in all_jobs))
else:
build.result = Result.unknown
if is_finished:
build.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in all_jobs))
if new_status != Status.finished:
build.status = new_status
if db.session.is_modified(build):
build.date_modified = datetime.utcnow()
db.session.add(build)
db.session.commit()
if not is_finished:
raise sync_build.NotFinished
try:
aggregate_build_stat(build, 'test_count')
aggregate_build_stat(build, 'test_duration')
aggregate_build_stat(build, 'test_failures')
aggregate_build_stat(build, 'test_rerun_count')
aggregate_build_stat(build, 'tests_missing')
aggregate_build_stat(build, 'lines_covered')
aggregate_build_stat(build, 'lines_uncovered')
aggregate_build_stat(build, 'diff_lines_covered')
aggregate_build_stat(build, 'diff_lines_uncovered')
except Exception:
current_app.logger.exception('Failing recording aggregate stats for build %s', build.id)
fire_signal.delay(
signal='build.finished',
kwargs={'build_id': build.id.hex},
)
queue.delay('update_project_stats', kwargs={
'project_id': build.project_id.hex,
}, countdown=1)
def sync_build(build_id):
"""
Synchronizing the build happens continuously until all jobs have reported in
as finished or have failed/aborted.
This task is responsible for:
- Checking in with jobs
- Aborting/retrying them if they're beyond limits
- Aggregating the results from jobs into the build itself
"""
build = Build.query.get(build_id)
if not build:
return
if build.status == Status.finished:
return
all_jobs = list(Job.query.filter(
Job.build_id == build_id,
))
is_finished = sync_build.verify_all_children() == Status.finished
if any(p.status != Status.finished for p in all_jobs):
is_finished = False
prev_started = build.date_started
build.date_started = safe_agg(
min, (j.date_started for j in all_jobs if j.date_started))
# We want to report how long we waited for the build to start once and only once,
# so we do it at the transition from not started to started.
if not prev_started and build.date_started:
queued_time = build.date_started - build.date_created
statsreporter.stats().log_timing('build_start_latency', _timedelta_to_millis(queued_time))
if is_finished:
# If there are no jobs (or no jobs with a finished date) fall back to
# finishing now, since at this point, the build is done executing.
build.date_finished = safe_agg(
max, (j.date_finished for j in all_jobs if j.date_finished), datetime.utcnow())
else:
build.date_finished = None
if build.date_started and build.date_finished:
build.duration = _timedelta_to_millis(build.date_finished - build.date_started)
else:
build.duration = None
if any(j.result is Result.failed for j in all_jobs):
build.result = Result.failed
elif is_finished:
build.result = aggregate_result((j.result for j in all_jobs))
else:
build.result = Result.unknown
if is_finished:
build.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in all_jobs))
if new_status != Status.finished:
build.status = new_status
if is_finished:
build.date_decided = datetime.utcnow()
decided_latency = build.date_decided - build.date_finished
statsreporter.stats().log_timing('build_decided_latency', _timedelta_to_millis(decided_latency))
else:
build.date_decided = None
if db.session.is_modified(build):
build.date_modified = datetime.utcnow()
db.session.add(build)
db.session.commit()
if not is_finished:
raise sync_build.NotFinished
with statsreporter.stats().timer('build_stat_aggregation'):
try:
aggregate_build_stat(build, 'test_count')
aggregate_build_stat(build, 'test_duration')
aggregate_build_stat(build, 'test_failures')
aggregate_build_stat(build, 'test_rerun_count')
aggregate_build_stat(build, 'tests_missing')
aggregate_build_stat(build, 'lines_covered')
aggregate_build_stat(build, 'lines_uncovered')
aggregate_build_stat(build, 'diff_lines_covered')
aggregate_build_stat(build, 'diff_lines_uncovered')
except Exception:
current_app.logger.exception('Failing recording aggregate stats for build %s', build.id)
fire_signal.delay(
signal='build.finished',
kwargs={'build_id': build.id.hex},
)
queue.delay('update_project_stats', kwargs={
'project_id': build.project_id.hex,
}, countdown=1)
def create_or_update_revision_result(revision_sha, project_id, propagation_limit):
"""Given a revision sha and project ID, try to update the revision result
for it. This involves copying results for unaffected Bazel targets from
the latest parent build.
`propagation_limit` is used to control how many times this function will
be called recursively on the revision's children. If it is 0, then this
function only updates the current revision's revision result and does
not do any recursion.
"""
# type: (str, UUID, int) -> None
project = Project.query.get(project_id)
revision = Revision.query.filter(
Revision.sha == revision_sha,
Revision.repository_id == project.repository_id,
).first()
last_finished_build = get_latest_finished_build_for_revision(
revision_sha, project_id)
if not last_finished_build:
return
unaffected_targets = BazelTarget.query.join(
Job, BazelTarget.job_id == Job.id,
).filter(
BazelTarget.result_source == ResultSource.from_parent,
Job.build_id == last_finished_build.id,
).all()
if len(unaffected_targets) > 0 and len(revision.parents) > 0:
# TODO(naphat) there's probably a better way to select parent,
# but that happens rarely enough that it can be punted for now
parent_revision_sha = revision.parents[0]
# TODO(naphat) we should find a better way to select parent builds.
# Even if a parent build is not finished, we can already start to
# take a look at target results, as it may already have results
# for all of our unaffected_targets
# perhaps an optimization is to take the latest build, instead
# of the latest finished build. Finished build are more likely
# to have the complete set of targets we need though. But if
# a finished build is not the latest build, then maybe that
# finished build had an infra failure. Anyways, for simplicity,
# let's stick to finished build for now.
parent_build = get_latest_finished_build_for_revision(
parent_revision_sha, project_id)
if parent_build:
# group unaffected targets by jobs
unaffected_targets_groups = defaultdict(lambda: {})
for target in unaffected_targets:
unaffected_targets_groups[target.job_id][target.name] = target
# process targets in batch, grouped by job id
# almost always, this is going to be a single job - there is
# usually only one autogenerated plan per project.
for job_id, targets_dict in unaffected_targets_groups.iteritems():
jobplan = JobPlan.query.filter(
JobPlan.project_id == project_id,
JobPlan.build_id == last_finished_build.id,
JobPlan.job_id == job_id,
).first()
if not jobplan:
continue
parent_targets = BazelTarget.query.join(
Job, BazelTarget.job_id == Job.id,
).join(
JobPlan, BazelTarget.job_id == JobPlan.job_id,
).filter(
Job.build_id == parent_build.id,
BazelTarget.name.in_(targets_dict),
JobPlan.plan_id == jobplan.plan_id,
)
for parent_target in parent_targets:
targets_dict[parent_target.name].result = parent_target.result
db.session.add(targets_dict[parent_target.name])
else:
logger.info("Revision %s could not find a parent build for parent revision %s.", revision_sha, parent_revision_sha)
revision_result, _ = create_or_update(RevisionResult, where={
'revision_sha': revision_sha,
'project_id': project_id,
}, values={
'build_id': last_finished_build.id,
'result': aggregate_result([last_finished_build.result] + [t.result for t in unaffected_targets]),
})
db.session.commit()
fire_signal.delay(
signal='revision_result.updated',
kwargs={'revision_result_id': revision_result.id.hex},
)
if propagation_limit > 0:
# TODO stop the propagation if nothing changed
for child_revision in get_child_revisions(revision):
create_or_update_revision_result(
child_revision.sha, project_id, propagation_limit=propagation_limit - 1)
def sync_job(job_id):
with RCount('sync_job'):
job = Job.query.get(job_id)
if not job:
return
if job.status == Status.finished:
return
# TODO(dcramer): we make an assumption that there is a single step
jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)
try:
implementation.update(job=job)
except UnrecoverableException:
job.status = Status.finished
job.result = Result.aborted
current_app.logger.exception('Unrecoverable exception syncing %s', job.id)
all_phases = list(job.phases)
# propagate changes to any phases as they live outside of the
# normalize synchronization routines
sync_job_phases(job, all_phases)
is_finished = sync_job.verify_all_children() == Status.finished
if any(p.status != Status.finished for p in all_phases):
is_finished = False
job.date_started = safe_agg(
min, (j.date_started for j in all_phases if j.date_started))
if is_finished:
job.date_finished = safe_agg(
max, (j.date_finished for j in all_phases if j.date_finished))
else:
job.date_finished = None
if job.date_started and job.date_finished:
job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000)
else:
job.duration = None
# if any phases are marked as failing, fail the build
if any(j.result is Result.failed for j in all_phases):
job.result = Result.failed
# if any test cases were marked as failing, fail the build
elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first():
job.result = Result.failed
# if we've finished all phases, use the best result available
elif is_finished:
job.result = aggregate_result((j.result for j in all_phases))
else:
job.result = Result.unknown
if is_finished:
job.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in all_phases))
if new_status != Status.finished:
job.status = new_status
elif job.status == Status.finished:
job.status = Status.in_progress
current_app.logger.exception('Job incorrectly marked as finished: %s', job.id)
if db.session.is_modified(job):
job.date_modified = datetime.utcnow()
db.session.add(job)
db.session.commit()
if not is_finished:
raise sync_job.NotFinished
try:
aggregate_job_stat(job, 'test_count')
aggregate_job_stat(job, 'test_duration')
aggregate_job_stat(job, 'test_failures')
aggregate_job_stat(job, 'test_rerun_count')
aggregate_job_stat(job, 'tests_missing')
aggregate_job_stat(job, 'lines_covered')
aggregate_job_stat(job, 'lines_uncovered')
aggregate_job_stat(job, 'diff_lines_covered')
aggregate_job_stat(job, 'diff_lines_uncovered')
except Exception:
current_app.logger.exception('Failing recording aggregate stats for job %s', job.id)
fire_signal.delay(
signal='job.finished',
kwargs={'job_id': job.id.hex},
)
if jobplan:
queue.delay('update_project_plan_stats', kwargs={
'project_id': job.project_id.hex,
'plan_id': jobplan.plan_id.hex,
}, countdown=1)
def sync_build(build_id):
"""
Synchronizing the build happens continuously until all jobs have reported in
as finished or have failed/aborted.
This task is responsible for:
- Checking in with jobs
- Aborting/retrying them if they're beyond limits
- Aggregating the results from jobs into the build itself
"""
build = Build.query.get(build_id)
if not build:
return
if build.status == Status.finished:
return
all_jobs = list(Job.query.filter(
Job.build_id == build_id,
))
is_finished = sync_build.verify_all_children() == Status.finished
if any(p.status != Status.finished for p in all_jobs):
is_finished = False
build.date_started = safe_agg(
min, (j.date_started for j in all_jobs if j.date_started))
if is_finished:
build.date_finished = safe_agg(
max, (j.date_finished for j in all_jobs if j.date_finished))
else:
build.date_finished = None
if build.date_started and build.date_finished:
build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000)
else:
build.duration = None
if any(j.result is Result.failed for j in all_jobs):
build.result = Result.failed
elif is_finished:
build.result = aggregate_result((j.result for j in all_jobs))
else:
build.result = Result.unknown
if is_finished:
build.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in all_jobs))
if new_status != Status.finished:
build.status = new_status
if db.session.is_modified(build):
build.date_modified = datetime.utcnow()
db.session.add(build)
db.session.commit()
if not is_finished:
raise sync_build.NotFinished
try:
aggregate_build_stat(build, 'test_count')
aggregate_build_stat(build, 'test_duration')
aggregate_build_stat(build, 'test_failures')
aggregate_build_stat(build, 'test_rerun_count')
aggregate_build_stat(build, 'tests_missing')
aggregate_build_stat(build, 'lines_covered')
aggregate_build_stat(build, 'lines_uncovered')
aggregate_build_stat(build, 'diff_lines_covered')
aggregate_build_stat(build, 'diff_lines_uncovered')
except Exception:
current_app.logger.exception('Failing recording aggregate stats for build %s', build.id)
fire_signal.delay(
signal='build.finished',
kwargs={'build_id': build.id.hex},
)
queue.delay('update_project_stats', kwargs={
'project_id': build.project_id.hex,
}, countdown=1)
def create_or_update_revision_result(revision_sha, project_id,
propagation_limit):
"""Given a revision sha and project ID, try to update the revision result
for it. This involves copying results for unaffected Bazel targets from
the latest parent build.
`propagation_limit` is used to control how many times this function will
be called recursively on the revision's children. If it is 0, then this
function only updates the current revision's revision result and does
not do any recursion.
"""
# type: (str, UUID, int) -> None
project = Project.query.get(project_id)
revision = Revision.query.filter(
Revision.sha == revision_sha,
Revision.repository_id == project.repository_id,
).first()
last_finished_build = get_latest_finished_build_for_revision(
revision_sha, project_id)
if not last_finished_build:
return
unaffected_targets = BazelTarget.query.join(
Job,
BazelTarget.job_id == Job.id,
).filter(
BazelTarget.result_source == ResultSource.from_parent,
Job.build_id == last_finished_build.id,
).all()
if len(unaffected_targets) > 0 and len(revision.parents) > 0:
# TODO(naphat) there's probably a better way to select parent,
# but that happens rarely enough that it can be punted for now
parent_revision_sha = revision.parents[0]
# TODO(naphat) we should find a better way to select parent builds.
# Even if a parent build is not finished, we can already start to
# take a look at target results, as it may already have results
# for all of our unaffected_targets
# perhaps an optimization is to take the latest build, instead
# of the latest finished build. Finished build are more likely
# to have the complete set of targets we need though. But if
# a finished build is not the latest build, then maybe that
# finished build had an infra failure. Anyways, for simplicity,
# let's stick to finished build for now.
parent_build = get_latest_finished_build_for_revision(
parent_revision_sha, project_id)
if parent_build:
# group unaffected targets by jobs
unaffected_targets_groups = defaultdict(lambda: {})
for target in unaffected_targets:
unaffected_targets_groups[target.job_id][target.name] = target
# process targets in batch, grouped by job id
# almost always, this is going to be a single job - there is
# usually only one autogenerated plan per project.
for job_id, targets_dict in unaffected_targets_groups.iteritems():
jobplan = JobPlan.query.filter(
JobPlan.project_id == project_id,
JobPlan.build_id == last_finished_build.id,
JobPlan.job_id == job_id,
).first()
if not jobplan:
continue
parent_targets = BazelTarget.query.join(
Job,
BazelTarget.job_id == Job.id,
).join(
JobPlan,
BazelTarget.job_id == JobPlan.job_id,
).filter(
Job.build_id == parent_build.id,
BazelTarget.name.in_(targets_dict),
JobPlan.plan_id == jobplan.plan_id,
)
for parent_target in parent_targets:
targets_dict[
parent_target.name].result = parent_target.result
db.session.add(targets_dict[parent_target.name])
else:
logger.info(
"Revision %s could not find a parent build for parent revision %s.",
revision_sha, parent_revision_sha)
revision_result, _ = create_or_update(
RevisionResult,
where={
'revision_sha': revision_sha,
'project_id': project_id,
},
values={
'build_id':
last_finished_build.id,
'result':
aggregate_result([last_finished_build.result] +
[t.result for t in unaffected_targets]),
})
db.session.commit()
fire_signal.delay(
signal='revision_result.updated',
kwargs={'revision_result_id': revision_result.id.hex},
)
if propagation_limit > 0:
# TODO stop the propagation if nothing changed
for child_revision in get_child_revisions(revision):
create_or_update_revision_result(
child_revision.sha,
project_id,
propagation_limit=propagation_limit - 1)
