def test_aggregate_status(): status_list = [Status.finished, Status.queued, Status.in_progress, Status.unknown] assert aggregate_status(status_list) == Status.in_progress status_list = [Status.finished, Status.queued, Status.unknown] assert aggregate_status(status_list) == Status.queued status_list = [Status.finished, Status.unknown] assert aggregate_status(status_list) == Status.finished
def sync_phase(phase, implementation): _find_and_retry_jobsteps(phase, implementation) phase_steps = list(phase.steps) if phase.date_started is None: phase.date_started = safe_agg(min, (s.date_started for s in phase_steps)) db.session.add(phase) if phase_steps: if all(s.status == Status.finished for s in phase_steps): phase.status = Status.finished phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps)) else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((s.status for s in phase_steps)) if new_status != Status.finished: phase.status = new_status if any(s.result is Result.failed for s in phase_steps): phase.result = Result.failed if phase.status == Status.finished: # Sets the final phase result. implementation.validate_phase(phase=phase) if db.session.is_modified(phase): phase.date_modified = datetime.utcnow() db.session.add(phase) db.session.commit()
def sync_phase(phase): phase_steps = list(phase.steps) if phase.date_started is None: phase.date_started = safe_agg(min, (s.date_started for s in phase_steps)) db.session.add(phase) if phase_steps: if all(s.status == Status.finished for s in phase_steps): phase.status = Status.finished phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps)) else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((s.status for s in phase_steps)) if new_status != Status.finished: phase.status = new_status if any(s.result is Result.failed for s in phase_steps): phase.result = Result.failed elif phase.status == Status.finished: phase.result = aggregate_result((s.result for s in phase_steps)) if db.session.is_modified(phase): phase.date_modified = datetime.utcnow() db.session.add(phase) db.session.commit()
def sync_job(job_id): """ Updates jobphase and job statuses based on the status of the constituent jobsteps. """ job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.infra_failed current_app.logger.exception('Unrecoverable exception syncing %s', job.id) all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases, implementation) is_finished = sync_job.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_phases): is_finished = False job.date_started = safe_agg( min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg( max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # If any test cases were marked as failing, fail the build. # The exception is if the only failing test case occurred in a JobStep that # had an infra failure. In this case we can't trust the test case result as # being meaningful and so we ignore these. elif TestCase.query.join(JobStep, JobStep.id == TestCase.step_id).filter( TestCase.result == Result.failed, TestCase.job_id == job.id, JobStep.result != Result.infra_failed ).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: # Sets the final job result. implementation.validate(job=job) else: job.result = Result.unknown if is_finished: job.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_phases)) if new_status != Status.finished: job.status = new_status elif job.status == Status.finished: job.status = Status.in_progress current_app.logger.exception('Job incorrectly marked as finished: %s', job.id) if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter( Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_jobs): is_finished = False prev_started = build.date_started build.date_started = safe_agg( min, (j.date_started for j in all_jobs if j.date_started)) # We want to report how long we waited for the build to start once and only once, # so we do it at the transition from not started to started. if not prev_started and build.date_started: queued_time = build.date_started - build.date_created statsreporter.stats().log_timing('build_start_latency', _timedelta_to_millis(queued_time)) if is_finished: # If there are no jobs (or no jobs with a finished date) fall back to # finishing now, since at this point, the build is done executing. build.date_finished = safe_agg( max, (j.date_finished for j in all_jobs if j.date_finished), datetime.utcnow()) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = _timedelta_to_millis(build.date_finished - build.date_started) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = aggregate_result((j.result for j in all_jobs)) else: build.result = Result.unknown if is_finished: build.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_jobs)) if new_status != Status.finished: build.status = new_status if is_finished: build.date_decided = datetime.utcnow() decided_latency = build.date_decided - build.date_finished statsreporter.stats().log_timing('build_decided_latency', _timedelta_to_millis(decided_latency)) else: build.date_decided = None if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() if not is_finished: raise sync_build.NotFinished with statsreporter.stats().timer('build_stat_aggregation'): try: aggregate_build_stat(build, 'test_count') aggregate_build_stat(build, 'test_duration') aggregate_build_stat(build, 'test_failures') aggregate_build_stat(build, 'test_rerun_count') aggregate_build_stat(build, 'tests_missing') aggregate_build_stat(build, 'lines_covered') aggregate_build_stat(build, 'lines_uncovered') aggregate_build_stat(build, 'diff_lines_covered') aggregate_build_stat(build, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for build %s', build.id) fire_signal.delay( signal='build.finished', kwargs={'build_id': build.id.hex}, ) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_job(job_id): with RCount('sync_job'): job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.aborted current_app.logger.exception('Unrecoverable exception syncing %s', job.id) all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases) is_finished = sync_job.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_phases): is_finished = False job.date_started = safe_agg( min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg( max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # if any test cases were marked as failing, fail the build elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: job.result = aggregate_result((j.result for j in all_phases)) else: job.result = Result.unknown if is_finished: job.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_phases)) if new_status != Status.finished: job.status = new_status elif job.status == Status.finished: job.status = Status.in_progress current_app.logger.exception('Job incorrectly marked as finished: %s', job.id) if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_job(job_id): """ Updates jobphase and job statuses based on the status of the constituent jobsteps. """ job = Job.query.get(job_id) if not job: return if job.status == Status.finished: return # TODO(dcramer): we make an assumption that there is a single step jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id) try: implementation.update(job=job) except UnrecoverableException: job.status = Status.finished job.result = Result.infra_failed current_app.logger.exception('Unrecoverable exception syncing %s', job.id) all_phases = list(job.phases) # propagate changes to any phases as they live outside of the # normalize synchronization routines sync_job_phases(job, all_phases, implementation) is_finished = sync_job.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_phases): is_finished = False job.date_started = safe_agg(min, (j.date_started for j in all_phases if j.date_started)) if is_finished: job.date_finished = safe_agg(max, (j.date_finished for j in all_phases if j.date_finished)) else: job.date_finished = None if job.date_started and job.date_finished: job.duration = int( (job.date_finished - job.date_started).total_seconds() * 1000) else: job.duration = None # if any phases are marked as failing, fail the build if any(j.result is Result.failed for j in all_phases): job.result = Result.failed # If any test cases were marked as failing, fail the build. # The exception is if the only failing test case occurred in a JobStep that # had an infra failure. In this case we can't trust the test case result as # being meaningful and so we ignore these. elif TestCase.query.join(JobStep, JobStep.id == TestCase.step_id).filter( TestCase.result == Result.failed, TestCase.job_id == job.id, JobStep.result != Result.infra_failed).first(): job.result = Result.failed # if we've finished all phases, use the best result available elif is_finished: # Sets the final job result. implementation.validate(job=job) else: job.result = Result.unknown if is_finished: job.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_phases)) if new_status != Status.finished: job.status = new_status elif job.status == Status.finished: job.status = Status.in_progress current_app.logger.exception( 'Job incorrectly marked as finished: %s', job.id) if db.session.is_modified(job): job.date_modified = datetime.utcnow() db.session.add(job) db.session.commit() if not is_finished: raise sync_job.NotFinished try: aggregate_job_stat(job, 'test_count') aggregate_job_stat(job, 'test_duration') aggregate_job_stat(job, 'test_failures') aggregate_job_stat(job, 'test_rerun_count') aggregate_job_stat(job, 'tests_missing') aggregate_job_stat(job, 'lines_covered') aggregate_job_stat(job, 'lines_uncovered') aggregate_job_stat(job, 'diff_lines_covered') aggregate_job_stat(job, 'diff_lines_uncovered') except Exception: current_app.logger.exception( 'Failing recording aggregate stats for job %s', job.id) fire_signal.delay( signal='job.finished', kwargs={'job_id': job.id.hex}, ) if jobplan: queue.delay('update_project_plan_stats', kwargs={ 'project_id': job.project_id.hex, 'plan_id': jobplan.plan_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter( Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_jobs): is_finished = False build.date_started = safe_agg( min, (j.date_started for j in all_jobs if j.date_started)) if is_finished: build.date_finished = safe_agg( max, (j.date_finished for j in all_jobs if j.date_finished)) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = aggregate_result((j.result for j in all_jobs)) else: build.result = Result.unknown if is_finished: build.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_jobs)) if new_status != Status.finished: build.status = new_status if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() if not is_finished: raise sync_build.NotFinished try: aggregate_build_stat(build, 'test_count') aggregate_build_stat(build, 'test_duration') aggregate_build_stat(build, 'test_failures') aggregate_build_stat(build, 'test_rerun_count') aggregate_build_stat(build, 'tests_missing') aggregate_build_stat(build, 'lines_covered') aggregate_build_stat(build, 'lines_uncovered') aggregate_build_stat(build, 'diff_lines_covered') aggregate_build_stat(build, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for build %s', build.id) fire_signal.delay( signal='build.finished', kwargs={'build_id': build.id.hex}, ) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)
def sync_build(build_id): """ Synchronizing the build happens continuously until all jobs have reported in as finished or have failed/aborted. This task is responsible for: - Checking in with jobs - Aborting/retrying them if they're beyond limits - Aggregating the results from jobs into the build itself """ with RCount('sync_build'): build = Build.query.get(build_id) if not build: return if build.status == Status.finished: return all_jobs = list(Job.query.filter( Job.build_id == build_id, )) is_finished = sync_build.verify_all_children() == Status.finished if any(p.status != Status.finished for p in all_jobs): is_finished = False build.date_started = safe_agg( min, (j.date_started for j in all_jobs if j.date_started)) if is_finished: build.date_finished = safe_agg( max, (j.date_finished for j in all_jobs if j.date_finished)) else: build.date_finished = None if build.date_started and build.date_finished: build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000) else: build.duration = None if any(j.result is Result.failed for j in all_jobs): build.result = Result.failed elif is_finished: build.result = aggregate_result((j.result for j in all_jobs)) else: build.result = Result.unknown if is_finished: build.status = Status.finished else: # ensure we dont set the status to finished unless it actually is new_status = aggregate_status((j.status for j in all_jobs)) if new_status != Status.finished: build.status = new_status if db.session.is_modified(build): build.date_modified = datetime.utcnow() db.session.add(build) db.session.commit() if not is_finished: raise sync_build.NotFinished try: aggregate_build_stat(build, 'test_count') aggregate_build_stat(build, 'test_duration') aggregate_build_stat(build, 'test_failures') aggregate_build_stat(build, 'test_rerun_count') aggregate_build_stat(build, 'tests_missing') aggregate_build_stat(build, 'lines_covered') aggregate_build_stat(build, 'lines_uncovered') aggregate_build_stat(build, 'diff_lines_covered') aggregate_build_stat(build, 'diff_lines_uncovered') except Exception: current_app.logger.exception('Failing recording aggregate stats for build %s', build.id) fire_signal.delay( signal='build.finished', kwargs={'build_id': build.id.hex}, ) queue.delay('update_project_stats', kwargs={ 'project_id': build.project_id.hex, }, countdown=1)