def sync_phase(phase): phase_steps = list(phase.steps) if phase.date_started is None: phase.date_started = safe_agg(min, (s.date_started for s in phase_steps)) db.session.add(phase) if phase_steps: if all(s.status == Status.finished for s in phase_steps): phase.status = Status.finished phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps)) else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((s.status for s in phase_steps)) if new_status != Status.finished: phase.status = new_status if any(s.result is Result.failed for s in phase_steps): phase.result = Result.failed elif phase.status == Status.finished: phase.result = aggregate_result((s.result for s in phase_steps)) if db.session.is_modified(phase): phase.date_modified = datetime.utcnow() db.session.add(phase) db.session.commit()
def sync_phase(phase): phase_steps = list(phase.steps) if phase.date_started is None: phase.date_started = safe_agg( min, (s.date_started for s in phase_steps), phase.date_started) db.session.add(phase) if all(s.status == Status.finished for s in phase_steps): phase.status = Status.finished phase.date_finished = safe_agg( max, (s.date_finished for s in phase_steps), phase.date_finished) if any(s.result is Result.failed for s in phase_steps): phase.result = Result.failed else: phase.result = safe_agg( max, (s.result for s in phase.steps), Result.unknown) db.session.add(phase) elif any(s.status != Status.finished for s in phase_steps): phase.status = Status.in_progress db.session.add(phase) if db.session.is_modified(phase): phase.date_modified = datetime.utcnow() db.session.commit()
def sync_phase(phase): phase_steps = list(phase.steps) if phase.date_started is None: phase.date_started = safe_agg(min, (s.date_started for s in phase_steps)) db.session.add(phase) if phase_steps: if all(s.status == Status.finished for s in phase_steps): phase.status = Status.finished phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps)) elif any(s.status != Status.finished for s in phase_steps): phase.status = Status.in_progress if any(s.result is Result.failed for s in phase_steps): phase.result = Result.failed elif phase.status == Status.finished: phase.result = safe_agg(max, (s.result for s in phase.steps)) if db.session.is_modified(phase): phase.date_modified = datetime.utcnow() db.session.add(phase) db.session.commit()
def sync_phase(phase, implementation): _find_and_retry_jobsteps(phase, implementation) phase_steps = list(phase.steps) if phase.date_started is None: phase.date_started = safe_agg(min, (s.date_started for s in phase_steps)) db.session.add(phase) if phase_steps: if all(s.status == Status.finished for s in phase_steps): phase.status = Status.finished phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps)) else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((s.status for s in phase_steps)) if new_status != Status.finished: phase.status = new_status if any(s.result is Result.failed for s in phase_steps): phase.result = Result.failed if phase.status == Status.finished: # Sets the final phase result. implementation.validate_phase(phase=phase) if db.session.is_modified(phase): phase.date_modified = datetime.utcnow() db.session.add(phase) db.session.commit()
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter( Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_jobs): is_finished = False prev_started = build.date_started build.date_started = safe_agg( min, (j.date_started for j in all_jobs if j.date_started)) # We want to report how long we waited for the build to start once and only once, # so we do it at the transition from not started to started. if not prev_started and build.date_started: queued_time = build.date_started - build.date_created statsreporter.stats().log_timing('build_start_latency', _timedelta_to_millis(queued_time)) if is_finished: # If there are no jobs (or no jobs with a finished date) fall back to # finishing now, since at this point, the build is done executing. build.date_finished = safe_agg( max, (j.date_finished for j in all_jobs if j.date_finished), datetime.utcnow()) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = _timedelta_to_millis(build.date_finished - build.date_started) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = aggregate_result((j.result for j in all_jobs)) else: build.result = Result.unknown if is_finished: build.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_jobs)) if new_status != Status.finished: build.status = new_status if is_finished: build.date_decided = datetime.utcnow() decided_latency = build.date_decided - build.date_finished statsreporter.stats().log_timing('build_decided_latency', _timedelta_to_millis(decided_latency)) else: build.date_decided = None if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() if not is_finished: raise sync_build.NotFinished with statsreporter.stats().timer('build_stat_aggregation'): try: aggregate_build_stat(build, 'test_count') aggregate_build_stat(build, 'test_duration') aggregate_build_stat(build, 'test_failures') aggregate_build_stat(build, 'test_rerun_count') aggregate_build_stat(build, 'tests_missing') aggregate_build_stat(build, 'lines_covered') aggregate_build_stat(build, 'lines_uncovered') aggregate_build_stat(build, 'diff_lines_covered') aggregate_build_stat(build, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for build %s', build.id) fire_signal.delay( signal='build.finished', kwargs={'build_id': build.id.hex}, ) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_job(job_id): job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step job_plan = JobPlan.query.options(subqueryload_all('plan.steps')).filter( JobPlan.job_id == job.id, ).join(Plan).first() try: if not job_plan: raise UnrecoverableException( 'Got sync_job task without job plan: %s' % (job.id, )) try: step = job_plan.plan.steps[0] except IndexError: raise UnrecoverableException('Missing steps for plan') implementation = step.get_implementation() implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.aborted current_app.logger.exception('Unrecoverable exception syncing %s', job.id) is_finished = sync_job.verify_all_children() == Status.finished if is_finished: job.status = Status.finished all_phases = list(job.phases) job.date_started = safe_agg(min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg(max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int( (job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # if any test cases were marked as failing, fail the build elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: job.result = safe_agg(max, (j.result for j in all_phases), Result.unknown) else: job.result = Result.unknown if is_finished: job.status = Status.finished elif any(j.status is Status.in_progress for j in all_phases): job.status = Status.in_progress else: job.status = Status.queued if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() publish_job_update(job) if not is_finished: raise sync_job.NotFinished _record_tests_missing(job) queue.delay('notify_job_finished', kwargs={ 'job_id': job.id.hex, }) if job_plan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': job_plan.plan_id.hex, }, countdown=1)
def _sync_step_from_active(self, step): try: job_name = step.data['job_name'] build_no = step.data['build_no'] except KeyError: raise UnrecoverableException('Missing Jenkins job information') try: item = self._get_response('/job/{}/{}'.format( job_name, build_no)) except NotFound: raise UnrecoverableException('Unable to find job in Jenkins') # TODO(dcramer): we're doing a lot of work here when we might # not need to due to it being sync'd previously node, _ = get_or_create(Node, where={ 'label': item['builtOn'], }) step.node = node step.label = item['fullDisplayName'] step.date_started = datetime.utcfromtimestamp( item['timestamp'] / 1000) if item['building']: step.status = Status.in_progress else: step.status = Status.finished step.result = RESULT_MAP[item['result']] # values['duration'] = item['duration'] or None step.date_finished = datetime.utcfromtimestamp( (item['timestamp'] + item['duration']) / 1000) # step.data.update({ # 'backend': { # 'uri': item['url'], # 'label': item['fullDisplayName'], # } # }) db.session.add(step) # TODO(dcramer): we shoudl abstract this into a sync_phase phase = step.phase if not phase.date_started: phase.date_started = safe_agg( min, (s.date_started for s in phase.steps), step.date_started) db.session.add(phase) if phase.status != step.status: phase.status = step.status db.session.add(phase) if step.status == Status.finished: phase.status = Status.finished phase.date_finished = safe_agg( max, (s.date_finished for s in phase.steps), step.date_finished) if any(s.result is Result.failed for s in phase.steps): phase.result = Result.failed else: phase.result = safe_agg( max, (s.result for s in phase.steps), Result.unknown) db.session.add(phase) db.session.commit() if step.status != Status.finished: return # sync artifacts for artifact in item.get('artifacts', ()): artifact, created = get_or_create(Artifact, where={ 'step': step, 'name': artifact['fileName'], }, defaults={ 'project': step.project, 'job': step.job, 'data': artifact, }) sync_artifact.delay_if_needed( artifact_id=artifact.id.hex, task_id=artifact.id.hex, parent_task_id=step.id.hex, ) # sync test results try: with db.session.begin_nested(): self._sync_test_results( step=step, job_name=job_name, build_no=build_no, ) except Exception: self.logger.exception( 'Failed to sync test results for %s #%s', job_name, build_no) else: db.session.commit() # sync console log try: result = True while result: result = self._sync_log( jobstep=step, name=step.label, job_name=job_name, build_no=build_no, ) except Exception: db.session.rollback() current_app.logger.exception( 'Unable to sync console log for job step %r', step.id.hex)
def sync_job(job_id): with RCount('sync_job'): job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.aborted current_app.logger.exception('Unrecoverable exception syncing %s', job.id) all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases) is_finished = sync_job.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_phases): is_finished = False job.date_started = safe_agg( min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg( max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # if any test cases were marked as failing, fail the build elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: job.result = aggregate_result((j.result for j in all_phases)) else: job.result = Result.unknown if is_finished: job.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_phases)) if new_status != Status.finished: job.status = new_status elif job.status == Status.finished: job.status = Status.in_progress current_app.logger.exception('Job incorrectly marked as finished: %s', job.id) if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter(Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished build.date_started = safe_agg(min, (j.date_started for j in all_jobs if j.date_started)) if is_finished: build.date_finished = safe_agg(max, (j.date_finished for j in all_jobs if j.date_finished)) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = int( (build.date_finished - build.date_started).total_seconds() * 1000) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = safe_agg(max, (j.result for j in all_jobs), Result.unknown) else: build.result = Result.unknown if is_finished: build.status = Status.finished elif any(j.status is Status.in_progress for j in all_jobs): build.status = Status.in_progress else: build.status = Status.queued if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() publish_build_update(build) if not is_finished: raise sync_build.NotFinished _record_tests_missing(build) queue.delay('notify_build_finished', kwargs={ 'build_id': build.id.hex, }) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter( Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished build.date_started = safe_agg( min, (j.date_started for j in all_jobs if j.date_started)) if is_finished: build.date_finished = safe_agg( max, (j.date_finished for j in all_jobs if j.date_finished)) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = safe_agg( max, (j.result for j in all_jobs), Result.unknown) else: build.result = Result.unknown if is_finished: build.status = Status.finished elif any(j.status is Status.in_progress for j in all_jobs): build.status = Status.in_progress else: build.status = Status.queued if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() publish_build_update(build) if not is_finished: raise sync_build.NotFinished _record_tests_missing(build) queue.delay('notify_build_finished', kwargs={ 'build_id': build.id.hex, }) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_job(job_id): """ Updates jobphase and job statuses based on the status of the constituent jobsteps. """ job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.infra_failed current_app.logger.exception('Unrecoverable exception syncing %s', job.id) all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases, implementation) is_finished = sync_job.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_phases): is_finished = False job.date_started = safe_agg(min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg(max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int( (job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # If any test cases were marked as failing, fail the build. # The exception is if the only failing test case occurred in a JobStep that # had an infra failure. In this case we can't trust the test case result as # being meaningful and so we ignore these. elif TestCase.query.join(JobStep, JobStep.id == TestCase.step_id).filter( TestCase.result == Result.failed, TestCase.job_id == job.id, JobStep.result != Result.infra_failed).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: # Sets the final job result. implementation.validate(job=job) else: job.result = Result.unknown if is_finished: job.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_phases)) if new_status != Status.finished: job.status = new_status elif job.status == Status.finished: job.status = Status.in_progress current_app.logger.exception( 'Job incorrectly marked as finished: %s', job.id) if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception( 'Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter( Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_jobs): is_finished = False build.date_started = safe_agg( min, (j.date_started for j in all_jobs if j.date_started)) if is_finished: build.date_finished = safe_agg( max, (j.date_finished for j in all_jobs if j.date_finished)) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = aggregate_result((j.result for j in all_jobs)) else: build.result = Result.unknown if is_finished: build.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_jobs)) if new_status != Status.finished: build.status = new_status if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() if not is_finished: raise sync_build.NotFinished try: aggregate_build_stat(build, 'test_count') aggregate_build_stat(build, 'test_duration') aggregate_build_stat(build, 'test_failures') aggregate_build_stat(build, 'test_rerun_count') aggregate_build_stat(build, 'tests_missing') aggregate_build_stat(build, 'lines_covered') aggregate_build_stat(build, 'lines_uncovered') aggregate_build_stat(build, 'diff_lines_covered') aggregate_build_stat(build, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for build %s', build.id) fire_signal.delay( signal='build.finished', kwargs={'build_id': build.id.hex}, ) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_job(job_id): job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step job_plan = JobPlan.query.options( subqueryload_all('plan.steps') ).filter( JobPlan.job_id == job.id, ).join(Plan).first() try: if not job_plan: raise UnrecoverableException('Got sync_job task without job plan: %s' % (job.id,)) try: step = job_plan.plan.steps[0] except IndexError: raise UnrecoverableException('Missing steps for plan') implementation = step.get_implementation() if has_timed_out(job, job_plan): remaining_steps = list(JobStep.query.filter( JobStep.status != Status.finished, JobStep.job_id == job.id, )) implementation.cancel(job=job) for step in remaining_steps: step.result = Result.failed step.status = Status.finished db.session.add(step) try_create(FailureReason, { 'step_id': step.id, 'job_id': job.id, 'build_id': job.build_id, 'project_id': job.project_id, 'reason': 'timeout' }) # ensure the job result actually reflects a failure job.result = Result.failed job.status = Status.finished db.session.add(job) else: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.aborted current_app.logger.exception('Unrecoverable exception syncing %s', job.id) is_finished = sync_job.verify_all_children() == Status.finished if is_finished: job.status = Status.finished db.session.flush() all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases) job.date_started = safe_agg( min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg( max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # if any test cases were marked as failing, fail the build elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: job.result = safe_agg(max, (j.result for j in all_phases)) else: job.result = Result.unknown if is_finished: job.status = Status.finished elif any(j.status is not Status.queued for j in all_phases): job.status = Status.in_progress else: job.status = Status.queued if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if job_plan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': job_plan.plan_id.hex, }, countdown=1)
def sync_job(job_id): """ Updates jobphase and job statuses based on the status of the constituent jobsteps. """ job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.infra_failed current_app.logger.exception('Unrecoverable exception syncing %s', job.id) all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases, implementation) is_finished = sync_job.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_phases): is_finished = False job.date_started = safe_agg( min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg( max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # If any test cases were marked as failing, fail the build. # The exception is if the only failing test case occurred in a JobStep that # had an infra failure. In this case we can't trust the test case result as # being meaningful and so we ignore these. elif TestCase.query.join(JobStep, JobStep.id == TestCase.step_id).filter( TestCase.result == Result.failed, TestCase.job_id == job.id, JobStep.result != Result.infra_failed ).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: # Sets the final job result. implementation.validate(job=job) else: job.result = Result.unknown if is_finished: job.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_phases)) if new_status != Status.finished: job.status = new_status elif job.status == Status.finished: job.status = Status.in_progress current_app.logger.exception('Job incorrectly marked as finished: %s', job.id) if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_job(job_id): job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.aborted current_app.logger.exception('Unrecoverable exception syncing %s', job.id) is_finished = sync_job.verify_all_children() == Status.finished if is_finished: job.status = Status.finished db.session.flush() all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases) job.date_started = safe_agg(min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg(max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int( (job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # if any test cases were marked as failing, fail the build elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: job.result = safe_agg(max, (j.result for j in all_phases)) else: job.result = Result.unknown if is_finished: job.status = Status.finished elif any(j.status is not Status.queued for j in all_phases): job.status = Status.in_progress else: job.status = Status.queued if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception( 'Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter(Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished build.date_started = safe_agg(min, (j.date_started for j in all_jobs if j.date_started)) if is_finished: build.date_finished = safe_agg(max, (j.date_finished for j in all_jobs if j.date_finished)) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = int( (build.date_finished - build.date_started).total_seconds() * 1000) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = safe_agg(max, (j.result for j in all_jobs)) else: build.result = Result.unknown if is_finished: build.status = Status.finished elif any(j.status is not Status.queued for j in all_jobs): build.status = Status.in_progress else: build.status = Status.queued if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() if not is_finished: raise sync_build.NotFinished try: aggregate_build_stat(build, 'test_count') aggregate_build_stat(build, 'test_duration') aggregate_build_stat(build, 'test_failures') aggregate_build_stat(build, 'test_rerun_count') aggregate_build_stat(build, 'tests_missing') aggregate_build_stat(build, 'lines_covered') aggregate_build_stat(build, 'lines_uncovered') aggregate_build_stat(build, 'diff_lines_covered') aggregate_build_stat(build, 'diff_lines_uncovered') except Exception: current_app.logger.exception( 'Failing recording aggregate stats for build %s', build.id) fire_signal.delay( signal='build.finished', kwargs={'build_id': build.id.hex}, ) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_job(job_id): job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step job_plan = JobPlan.query.options( subqueryload_all('plan.steps') ).filter( JobPlan.job_id == job.id, ).join(Plan).first() try: if not job_plan: raise UnrecoverableException('Got sync_job task without job plan: %s' % (job.id,)) try: step = job_plan.plan.steps[0] except IndexError: raise UnrecoverableException('Missing steps for plan') implementation = step.get_implementation() implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.aborted current_app.logger.exception('Unrecoverable exception syncing %s', job.id) is_finished = sync_job.verify_all_children() == Status.finished if is_finished: job.status = Status.finished all_phases = list(job.phases) job.date_started = safe_agg( min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg( max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # if any test cases were marked as failing, fail the build elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: job.result = safe_agg( max, (j.result for j in all_phases), Result.unknown) else: job.result = Result.unknown if is_finished: job.status = Status.finished elif any(j.status is Status.in_progress for j in all_phases): job.status = Status.in_progress else: job.status = Status.queued if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() publish_job_update(job) if not is_finished: raise sync_job.NotFinished _record_tests_missing(job) queue.delay('notify_job_finished', kwargs={ 'job_id': job.id.hex, }) if job_plan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': job_plan.plan_id.hex, }, countdown=1)
def _sync_step_from_active(self, step): try: job_name = step.data['job_name'] build_no = step.data['build_no'] except KeyError: raise UnrecoverableException('Missing Jenkins job information') try: item = self._get_response('/job/{}/{}'.format( job_name, build_no)) except NotFound: raise UnrecoverableException('Unable to find job in Jenkins') # TODO(dcramer): we're doing a lot of work here when we might # not need to due to it being sync'd previously node, _ = get_or_create(Node, where={ 'label': item['builtOn'], }) step.node = node step.label = item['fullDisplayName'] step.date_started = datetime.utcfromtimestamp( item['timestamp'] / 1000) if item['building']: step.status = Status.in_progress else: step.status = Status.finished step.result = RESULT_MAP[item['result']] # values['duration'] = item['duration'] or None step.date_finished = datetime.utcfromtimestamp( (item['timestamp'] + item['duration']) / 1000) # step.data.update({ # 'backend': { # 'uri': item['url'], # 'label': item['fullDisplayName'], # } # }) db.session.add(step) db.session.commit() # TODO(dcramer): we shoudl abstract this into a sync_phase phase = step.phase if not phase.date_started: phase.date_started = safe_agg( min, (s.date_started for s in phase.steps), step.date_started) db.session.add(phase) if phase.status != step.status: phase.status = step.status db.session.add(phase) if step.status == Status.finished: phase.status = Status.finished phase.date_finished = safe_agg( max, (s.date_finished for s in phase.steps), step.date_finished) if any(s.result is Result.failed for s in phase.steps): phase.result = Result.failed else: phase.result = safe_agg( max, (s.result for s in phase.steps), Result.unknown) db.session.add(phase) db.session.commit() if step.status != Status.finished: return # sync artifacts for artifact in item.get('artifacts', ()): artifact, created = get_or_create(Artifact, where={ 'step': step, 'name': artifact['fileName'], }, defaults={ 'project': step.project, 'job': step.job, 'data': artifact, }) db.session.commit() sync_artifact.delay_if_needed( artifact_id=artifact.id.hex, task_id=artifact.id.hex, parent_task_id=step.id.hex, ) # sync test results try: with db.session.begin_nested(): self._sync_test_results( step=step, job_name=job_name, build_no=build_no, ) except Exception: self.logger.exception( 'Failed to sync test results for %s #%s', job_name, build_no) else: db.session.commit() # sync console log try: result = True while result: result = self._sync_log( jobstep=step, name=step.label, job_name=job_name, build_no=build_no, ) except Exception: db.session.rollback() current_app.logger.exception( 'Unable to sync console log for job step %r', step.id.hex)