Ejemplo n.º 1
0
def sync_phase(phase):
    phase_steps = list(phase.steps)

    if phase.date_started is None:
        phase.date_started = safe_agg(min, (s.date_started for s in phase_steps))
        db.session.add(phase)

    if phase_steps:
        if all(s.status == Status.finished for s in phase_steps):
            phase.status = Status.finished
            phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps))
        else:
            # ensure we dont set the status to finished unless it actually is
            new_status = aggregate_status((s.status for s in phase_steps))
            if new_status != Status.finished:
                phase.status = new_status

        if any(s.result is Result.failed for s in phase_steps):
            phase.result = Result.failed

        elif phase.status == Status.finished:
            phase.result = aggregate_result((s.result for s in phase_steps))

    if db.session.is_modified(phase):
        phase.date_modified = datetime.utcnow()
        db.session.add(phase)
        db.session.commit()
Ejemplo n.º 2
0
def sync_phase(phase):
    phase_steps = list(phase.steps)

    if phase.date_started is None:
        phase.date_started = safe_agg(
            min, (s.date_started for s in phase_steps), phase.date_started)
        db.session.add(phase)

    if all(s.status == Status.finished for s in phase_steps):
        phase.status = Status.finished
        phase.date_finished = safe_agg(
            max, (s.date_finished for s in phase_steps), phase.date_finished)

        if any(s.result is Result.failed for s in phase_steps):
            phase.result = Result.failed
        else:
            phase.result = safe_agg(
                max, (s.result for s in phase.steps), Result.unknown)

        db.session.add(phase)
    elif any(s.status != Status.finished for s in phase_steps):
        phase.status = Status.in_progress
        db.session.add(phase)

    if db.session.is_modified(phase):
        phase.date_modified = datetime.utcnow()
        db.session.commit()
Ejemplo n.º 3
0
def sync_phase(phase):
    phase_steps = list(phase.steps)

    if phase.date_started is None:
        phase.date_started = safe_agg(min,
                                      (s.date_started for s in phase_steps))
        db.session.add(phase)

    if phase_steps:
        if all(s.status == Status.finished for s in phase_steps):
            phase.status = Status.finished
            phase.date_finished = safe_agg(max, (s.date_finished
                                                 for s in phase_steps))
        else:
            # ensure we dont set the status to finished unless it actually is
            new_status = aggregate_status((s.status for s in phase_steps))
            if new_status != Status.finished:
                phase.status = new_status

        if any(s.result is Result.failed for s in phase_steps):
            phase.result = Result.failed

        elif phase.status == Status.finished:
            phase.result = aggregate_result((s.result for s in phase_steps))

    if db.session.is_modified(phase):
        phase.date_modified = datetime.utcnow()
        db.session.add(phase)
        db.session.commit()
Ejemplo n.º 4
0
def sync_phase(phase):
    phase_steps = list(phase.steps)

    if phase.date_started is None:
        phase.date_started = safe_agg(min,
                                      (s.date_started for s in phase_steps))
        db.session.add(phase)

    if phase_steps:
        if all(s.status == Status.finished for s in phase_steps):
            phase.status = Status.finished
            phase.date_finished = safe_agg(max, (s.date_finished
                                                 for s in phase_steps))

        elif any(s.status != Status.finished for s in phase_steps):
            phase.status = Status.in_progress

        if any(s.result is Result.failed for s in phase_steps):
            phase.result = Result.failed

        elif phase.status == Status.finished:
            phase.result = safe_agg(max, (s.result for s in phase.steps))

    if db.session.is_modified(phase):
        phase.date_modified = datetime.utcnow()
        db.session.add(phase)
        db.session.commit()
Ejemplo n.º 5
0
def sync_phase(phase, implementation):
    _find_and_retry_jobsteps(phase, implementation)
    phase_steps = list(phase.steps)

    if phase.date_started is None:
        phase.date_started = safe_agg(min,
                                      (s.date_started for s in phase_steps))
        db.session.add(phase)

    if phase_steps:
        if all(s.status == Status.finished for s in phase_steps):
            phase.status = Status.finished
            phase.date_finished = safe_agg(max, (s.date_finished
                                                 for s in phase_steps))
        else:
            # ensure we dont set the status to finished unless it actually is
            new_status = aggregate_status((s.status for s in phase_steps))
            if new_status != Status.finished:
                phase.status = new_status

        if any(s.result is Result.failed for s in phase_steps):
            phase.result = Result.failed

        if phase.status == Status.finished:
            # Sets the final phase result.
            implementation.validate_phase(phase=phase)

    if db.session.is_modified(phase):
        phase.date_modified = datetime.utcnow()
        db.session.add(phase)
        db.session.commit()
Ejemplo n.º 6
0
def sync_phase(phase, implementation):
    _find_and_retry_jobsteps(phase, implementation)
    phase_steps = list(phase.steps)

    if phase.date_started is None:
        phase.date_started = safe_agg(min, (s.date_started for s in phase_steps))
        db.session.add(phase)

    if phase_steps:
        if all(s.status == Status.finished for s in phase_steps):
            phase.status = Status.finished
            phase.date_finished = safe_agg(max, (s.date_finished for s in phase_steps))
        else:
            # ensure we dont set the status to finished unless it actually is
            new_status = aggregate_status((s.status for s in phase_steps))
            if new_status != Status.finished:
                phase.status = new_status

        if any(s.result is Result.failed for s in phase_steps):
            phase.result = Result.failed

        if phase.status == Status.finished:
            # Sets the final phase result.
            implementation.validate_phase(phase=phase)

    if db.session.is_modified(phase):
        phase.date_modified = datetime.utcnow()
        db.session.add(phase)
        db.session.commit()
Ejemplo n.º 7
0
def sync_build(build_id):
    """
    Synchronizing the build happens continuously until all jobs have reported in
    as finished or have failed/aborted.

    This task is responsible for:
    - Checking in with jobs
    - Aborting/retrying them if they're beyond limits
    - Aggregating the results from jobs into the build itself
    """
    build = Build.query.get(build_id)
    if not build:
        return

    if build.status == Status.finished:
        return

    all_jobs = list(Job.query.filter(
        Job.build_id == build_id,
    ))

    is_finished = sync_build.verify_all_children() == Status.finished
    if any(p.status != Status.finished for p in all_jobs):
        is_finished = False

    prev_started = build.date_started
    build.date_started = safe_agg(
        min, (j.date_started for j in all_jobs if j.date_started))

    # We want to report how long we waited for the build to start once and only once,
    # so we do it at the transition from not started to started.
    if not prev_started and build.date_started:
        queued_time = build.date_started - build.date_created
        statsreporter.stats().log_timing('build_start_latency', _timedelta_to_millis(queued_time))

    if is_finished:
        # If there are no jobs (or no jobs with a finished date) fall back to
        # finishing now, since at this point, the build is done executing.
        build.date_finished = safe_agg(
            max, (j.date_finished for j in all_jobs if j.date_finished), datetime.utcnow())
    else:
        build.date_finished = None

    if build.date_started and build.date_finished:
        build.duration = _timedelta_to_millis(build.date_finished - build.date_started)
    else:
        build.duration = None

    if any(j.result is Result.failed for j in all_jobs):
        build.result = Result.failed
    elif is_finished:
        build.result = aggregate_result((j.result for j in all_jobs))
    else:
        build.result = Result.unknown

    if is_finished:
        build.status = Status.finished
    else:
        # ensure we dont set the status to finished unless it actually is
        new_status = aggregate_status((j.status for j in all_jobs))
        if new_status != Status.finished:
            build.status = new_status

    if is_finished:
        build.date_decided = datetime.utcnow()
        decided_latency = build.date_decided - build.date_finished
        statsreporter.stats().log_timing('build_decided_latency', _timedelta_to_millis(decided_latency))
    else:
        build.date_decided = None

    if db.session.is_modified(build):
        build.date_modified = datetime.utcnow()
        db.session.add(build)
        db.session.commit()

    if not is_finished:
        raise sync_build.NotFinished

    with statsreporter.stats().timer('build_stat_aggregation'):
        try:
            aggregate_build_stat(build, 'test_count')
            aggregate_build_stat(build, 'test_duration')
            aggregate_build_stat(build, 'test_failures')
            aggregate_build_stat(build, 'test_rerun_count')
            aggregate_build_stat(build, 'tests_missing')
            aggregate_build_stat(build, 'lines_covered')
            aggregate_build_stat(build, 'lines_uncovered')
            aggregate_build_stat(build, 'diff_lines_covered')
            aggregate_build_stat(build, 'diff_lines_uncovered')
        except Exception:
            current_app.logger.exception('Failing recording aggregate stats for build %s', build.id)

    fire_signal.delay(
        signal='build.finished',
        kwargs={'build_id': build.id.hex},
    )

    queue.delay('update_project_stats', kwargs={
        'project_id': build.project_id.hex,
    }, countdown=1)
Ejemplo n.º 8
0
def sync_job(job_id):
    job = Job.query.get(job_id)
    if not job:
        return

    if job.status == Status.finished:
        return

    # TODO(dcramer): we make an assumption that there is a single step
    job_plan = JobPlan.query.options(subqueryload_all('plan.steps')).filter(
        JobPlan.job_id == job.id, ).join(Plan).first()
    try:
        if not job_plan:
            raise UnrecoverableException(
                'Got sync_job task without job plan: %s' % (job.id, ))

        try:
            step = job_plan.plan.steps[0]
        except IndexError:
            raise UnrecoverableException('Missing steps for plan')

        implementation = step.get_implementation()
        implementation.update(job=job)

    except UnrecoverableException:
        job.status = Status.finished
        job.result = Result.aborted
        current_app.logger.exception('Unrecoverable exception syncing %s',
                                     job.id)

    is_finished = sync_job.verify_all_children() == Status.finished
    if is_finished:
        job.status = Status.finished

    all_phases = list(job.phases)

    job.date_started = safe_agg(min, (j.date_started
                                      for j in all_phases if j.date_started))

    if is_finished:
        job.date_finished = safe_agg(max,
                                     (j.date_finished
                                      for j in all_phases if j.date_finished))
    else:
        job.date_finished = None

    if job.date_started and job.date_finished:
        job.duration = int(
            (job.date_finished - job.date_started).total_seconds() * 1000)
    else:
        job.duration = None

    # if any phases are marked as failing, fail the build
    if any(j.result is Result.failed for j in all_phases):
        job.result = Result.failed
    # if any test cases were marked as failing, fail the build
    elif TestCase.query.filter(TestCase.result == Result.failed,
                               TestCase.job_id == job.id).first():
        job.result = Result.failed
    # if we've finished all phases, use the best result available
    elif is_finished:
        job.result = safe_agg(max, (j.result for j in all_phases),
                              Result.unknown)
    else:
        job.result = Result.unknown

    if is_finished:
        job.status = Status.finished
    elif any(j.status is Status.in_progress for j in all_phases):
        job.status = Status.in_progress
    else:
        job.status = Status.queued

    if db.session.is_modified(job):
        job.date_modified = datetime.utcnow()

        db.session.add(job)
        db.session.commit()
        publish_job_update(job)

    if not is_finished:
        raise sync_job.NotFinished

    _record_tests_missing(job)

    queue.delay('notify_job_finished', kwargs={
        'job_id': job.id.hex,
    })

    if job_plan:
        queue.delay('update_project_plan_stats',
                    kwargs={
                        'project_id': job.project_id.hex,
                        'plan_id': job_plan.plan_id.hex,
                    },
                    countdown=1)
Ejemplo n.º 9
0
    def _sync_step_from_active(self, step):
        try:
            job_name = step.data['job_name']
            build_no = step.data['build_no']
        except KeyError:
            raise UnrecoverableException('Missing Jenkins job information')

        try:
            item = self._get_response('/job/{}/{}'.format(
                job_name, build_no))
        except NotFound:
            raise UnrecoverableException('Unable to find job in Jenkins')

        # TODO(dcramer): we're doing a lot of work here when we might
        # not need to due to it being sync'd previously
        node, _ = get_or_create(Node, where={
            'label': item['builtOn'],
        })

        step.node = node
        step.label = item['fullDisplayName']
        step.date_started = datetime.utcfromtimestamp(
            item['timestamp'] / 1000)

        if item['building']:
            step.status = Status.in_progress
        else:
            step.status = Status.finished
            step.result = RESULT_MAP[item['result']]
            # values['duration'] = item['duration'] or None
            step.date_finished = datetime.utcfromtimestamp(
                (item['timestamp'] + item['duration']) / 1000)

        # step.data.update({
        #     'backend': {
        #         'uri': item['url'],
        #         'label': item['fullDisplayName'],
        #     }
        # })
        db.session.add(step)

        # TODO(dcramer): we shoudl abstract this into a sync_phase
        phase = step.phase

        if not phase.date_started:
            phase.date_started = safe_agg(
                min, (s.date_started for s in phase.steps), step.date_started)
            db.session.add(phase)

        if phase.status != step.status:
            phase.status = step.status
            db.session.add(phase)

        if step.status == Status.finished:
            phase.status = Status.finished
            phase.date_finished = safe_agg(
                max, (s.date_finished for s in phase.steps), step.date_finished)

            if any(s.result is Result.failed for s in phase.steps):
                phase.result = Result.failed
            else:
                phase.result = safe_agg(
                    max, (s.result for s in phase.steps), Result.unknown)

            db.session.add(phase)

        db.session.commit()

        if step.status != Status.finished:
            return

        # sync artifacts
        for artifact in item.get('artifacts', ()):
            artifact, created = get_or_create(Artifact, where={
                'step': step,
                'name': artifact['fileName'],
            }, defaults={
                'project': step.project,
                'job': step.job,
                'data': artifact,
            })
            sync_artifact.delay_if_needed(
                artifact_id=artifact.id.hex,
                task_id=artifact.id.hex,
                parent_task_id=step.id.hex,
            )

        # sync test results
        try:
            with db.session.begin_nested():
                self._sync_test_results(
                    step=step,
                    job_name=job_name,
                    build_no=build_no,
                )
        except Exception:
            self.logger.exception(
                'Failed to sync test results for %s #%s', job_name, build_no)
        else:
            db.session.commit()

        # sync console log
        try:
            result = True
            while result:
                result = self._sync_log(
                    jobstep=step,
                    name=step.label,
                    job_name=job_name,
                    build_no=build_no,
                )

        except Exception:
            db.session.rollback()
            current_app.logger.exception(
                'Unable to sync console log for job step %r',
                step.id.hex)
Ejemplo n.º 10
0
def sync_job(job_id):
    with RCount('sync_job'):
        job = Job.query.get(job_id)
        if not job:
            return

        if job.status == Status.finished:
            return

        # TODO(dcramer): we make an assumption that there is a single step
        jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)

        try:
            implementation.update(job=job)

        except UnrecoverableException:
            job.status = Status.finished
            job.result = Result.aborted
            current_app.logger.exception('Unrecoverable exception syncing %s', job.id)

        all_phases = list(job.phases)

        # propagate changes to any phases as they live outside of the
        # normalize synchronization routines
        sync_job_phases(job, all_phases)

        is_finished = sync_job.verify_all_children() == Status.finished
        if any(p.status != Status.finished for p in all_phases):
            is_finished = False

        job.date_started = safe_agg(
            min, (j.date_started for j in all_phases if j.date_started))

        if is_finished:
            job.date_finished = safe_agg(
                max, (j.date_finished for j in all_phases if j.date_finished))
        else:
            job.date_finished = None

        if job.date_started and job.date_finished:
            job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000)
        else:
            job.duration = None

        # if any phases are marked as failing, fail the build
        if any(j.result is Result.failed for j in all_phases):
            job.result = Result.failed
        # if any test cases were marked as failing, fail the build
        elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first():
            job.result = Result.failed
        # if we've finished all phases, use the best result available
        elif is_finished:
            job.result = aggregate_result((j.result for j in all_phases))
        else:
            job.result = Result.unknown

        if is_finished:
            job.status = Status.finished
        else:
            # ensure we dont set the status to finished unless it actually is
            new_status = aggregate_status((j.status for j in all_phases))
            if new_status != Status.finished:
                job.status = new_status
            elif job.status == Status.finished:
                job.status = Status.in_progress
                current_app.logger.exception('Job incorrectly marked as finished: %s', job.id)

        if db.session.is_modified(job):
            job.date_modified = datetime.utcnow()

            db.session.add(job)
            db.session.commit()

        if not is_finished:
            raise sync_job.NotFinished

        try:
            aggregate_job_stat(job, 'test_count')
            aggregate_job_stat(job, 'test_duration')
            aggregate_job_stat(job, 'test_failures')
            aggregate_job_stat(job, 'test_rerun_count')
            aggregate_job_stat(job, 'tests_missing')
            aggregate_job_stat(job, 'lines_covered')
            aggregate_job_stat(job, 'lines_uncovered')
            aggregate_job_stat(job, 'diff_lines_covered')
            aggregate_job_stat(job, 'diff_lines_uncovered')
        except Exception:
            current_app.logger.exception('Failing recording aggregate stats for job %s', job.id)

        fire_signal.delay(
            signal='job.finished',
            kwargs={'job_id': job.id.hex},
        )

        if jobplan:
            queue.delay('update_project_plan_stats', kwargs={
                'project_id': job.project_id.hex,
                'plan_id': jobplan.plan_id.hex,
            }, countdown=1)
Ejemplo n.º 11
0
def sync_build(build_id):
    """
    Synchronizing the build happens continuously until all jobs have reported in
    as finished or have failed/aborted.

    This task is responsible for:
    - Checking in with jobs
    - Aborting/retrying them if they're beyond limits
    - Aggregating the results from jobs into the build itself
    """
    build = Build.query.get(build_id)
    if not build:
        return

    if build.status == Status.finished:
        return

    all_jobs = list(Job.query.filter(Job.build_id == build_id, ))

    is_finished = sync_build.verify_all_children() == Status.finished

    build.date_started = safe_agg(min, (j.date_started
                                        for j in all_jobs if j.date_started))

    if is_finished:
        build.date_finished = safe_agg(max,
                                       (j.date_finished
                                        for j in all_jobs if j.date_finished))
    else:
        build.date_finished = None

    if build.date_started and build.date_finished:
        build.duration = int(
            (build.date_finished - build.date_started).total_seconds() * 1000)
    else:
        build.duration = None

    if any(j.result is Result.failed for j in all_jobs):
        build.result = Result.failed
    elif is_finished:
        build.result = safe_agg(max, (j.result for j in all_jobs),
                                Result.unknown)
    else:
        build.result = Result.unknown

    if is_finished:
        build.status = Status.finished
    elif any(j.status is Status.in_progress for j in all_jobs):
        build.status = Status.in_progress
    else:
        build.status = Status.queued

    if db.session.is_modified(build):
        build.date_modified = datetime.utcnow()
        db.session.add(build)
        db.session.commit()
        publish_build_update(build)

    if not is_finished:
        raise sync_build.NotFinished

    _record_tests_missing(build)

    queue.delay('notify_build_finished', kwargs={
        'build_id': build.id.hex,
    })

    queue.delay('update_project_stats',
                kwargs={
                    'project_id': build.project_id.hex,
                },
                countdown=1)
Ejemplo n.º 12
0
def sync_build(build_id):
    """
    Synchronizing the build happens continuously until all jobs have reported in
    as finished or have failed/aborted.

    This task is responsible for:
    - Checking in with jobs
    - Aborting/retrying them if they're beyond limits
    - Aggregating the results from jobs into the build itself
    """
    build = Build.query.get(build_id)
    if not build:
        return

    if build.status == Status.finished:
        return

    all_jobs = list(Job.query.filter(
        Job.build_id == build_id,
    ))

    is_finished = sync_build.verify_all_children() == Status.finished

    build.date_started = safe_agg(
        min, (j.date_started for j in all_jobs if j.date_started))

    if is_finished:
        build.date_finished = safe_agg(
            max, (j.date_finished for j in all_jobs if j.date_finished))
    else:
        build.date_finished = None

    if build.date_started and build.date_finished:
        build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000)
    else:
        build.duration = None

    if any(j.result is Result.failed for j in all_jobs):
        build.result = Result.failed
    elif is_finished:
        build.result = safe_agg(
            max, (j.result for j in all_jobs), Result.unknown)
    else:
        build.result = Result.unknown

    if is_finished:
        build.status = Status.finished
    elif any(j.status is Status.in_progress for j in all_jobs):
        build.status = Status.in_progress
    else:
        build.status = Status.queued

    if db.session.is_modified(build):
        build.date_modified = datetime.utcnow()
        db.session.add(build)
        db.session.commit()
        publish_build_update(build)

    if not is_finished:
        raise sync_build.NotFinished

    _record_tests_missing(build)

    queue.delay('notify_build_finished', kwargs={
        'build_id': build.id.hex,
    })

    queue.delay('update_project_stats', kwargs={
        'project_id': build.project_id.hex,
    }, countdown=1)
Ejemplo n.º 13
0
def sync_job(job_id):
    """
    Updates jobphase and job statuses based on the status of the constituent jobsteps.
    """
    job = Job.query.get(job_id)
    if not job:
        return

    if job.status == Status.finished:
        return

    # TODO(dcramer): we make an assumption that there is a single step
    jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)

    try:
        implementation.update(job=job)

    except UnrecoverableException:
        job.status = Status.finished
        job.result = Result.infra_failed
        current_app.logger.exception('Unrecoverable exception syncing %s',
                                     job.id)

    all_phases = list(job.phases)

    # propagate changes to any phases as they live outside of the
    # normalize synchronization routines
    sync_job_phases(job, all_phases, implementation)

    is_finished = sync_job.verify_all_children() == Status.finished
    if any(p.status != Status.finished for p in all_phases):
        is_finished = False

    job.date_started = safe_agg(min, (j.date_started
                                      for j in all_phases if j.date_started))

    if is_finished:
        job.date_finished = safe_agg(max,
                                     (j.date_finished
                                      for j in all_phases if j.date_finished))
    else:
        job.date_finished = None

    if job.date_started and job.date_finished:
        job.duration = int(
            (job.date_finished - job.date_started).total_seconds() * 1000)
    else:
        job.duration = None

    # if any phases are marked as failing, fail the build
    if any(j.result is Result.failed for j in all_phases):
        job.result = Result.failed
    # If any test cases were marked as failing, fail the build.
    # The exception is if the only failing test case occurred in a JobStep that
    # had an infra failure. In this case we can't trust the test case result as
    # being meaningful and so we ignore these.
    elif TestCase.query.join(JobStep, JobStep.id == TestCase.step_id).filter(
            TestCase.result == Result.failed, TestCase.job_id == job.id,
            JobStep.result != Result.infra_failed).first():
        job.result = Result.failed
    # if we've finished all phases, use the best result available
    elif is_finished:
        # Sets the final job result.
        implementation.validate(job=job)
    else:
        job.result = Result.unknown

    if is_finished:
        job.status = Status.finished
    else:
        # ensure we dont set the status to finished unless it actually is
        new_status = aggregate_status((j.status for j in all_phases))
        if new_status != Status.finished:
            job.status = new_status
        elif job.status == Status.finished:
            job.status = Status.in_progress
            current_app.logger.exception(
                'Job incorrectly marked as finished: %s', job.id)

    if db.session.is_modified(job):
        job.date_modified = datetime.utcnow()

        db.session.add(job)
        db.session.commit()

    if not is_finished:
        raise sync_job.NotFinished

    try:
        aggregate_job_stat(job, 'test_count')
        aggregate_job_stat(job, 'test_duration')
        aggregate_job_stat(job, 'test_failures')
        aggregate_job_stat(job, 'test_rerun_count')
        aggregate_job_stat(job, 'tests_missing')
        aggregate_job_stat(job, 'lines_covered')
        aggregate_job_stat(job, 'lines_uncovered')
        aggregate_job_stat(job, 'diff_lines_covered')
        aggregate_job_stat(job, 'diff_lines_uncovered')
    except Exception:
        current_app.logger.exception(
            'Failing recording aggregate stats for job %s', job.id)

    fire_signal.delay(
        signal='job.finished',
        kwargs={'job_id': job.id.hex},
    )

    if jobplan:
        queue.delay('update_project_plan_stats',
                    kwargs={
                        'project_id': job.project_id.hex,
                        'plan_id': jobplan.plan_id.hex,
                    },
                    countdown=1)
Ejemplo n.º 14
0
def sync_build(build_id):
    """
    Synchronizing the build happens continuously until all jobs have reported in
    as finished or have failed/aborted.

    This task is responsible for:
    - Checking in with jobs
    - Aborting/retrying them if they're beyond limits
    - Aggregating the results from jobs into the build itself
    """
    build = Build.query.get(build_id)
    if not build:
        return

    if build.status == Status.finished:
        return

    all_jobs = list(Job.query.filter(
        Job.build_id == build_id,
    ))

    is_finished = sync_build.verify_all_children() == Status.finished
    if any(p.status != Status.finished for p in all_jobs):
        is_finished = False

    build.date_started = safe_agg(
        min, (j.date_started for j in all_jobs if j.date_started))

    if is_finished:
        build.date_finished = safe_agg(
            max, (j.date_finished for j in all_jobs if j.date_finished))
    else:
        build.date_finished = None

    if build.date_started and build.date_finished:
        build.duration = int((build.date_finished - build.date_started).total_seconds() * 1000)
    else:
        build.duration = None

    if any(j.result is Result.failed for j in all_jobs):
        build.result = Result.failed
    elif is_finished:
        build.result = aggregate_result((j.result for j in all_jobs))
    else:
        build.result = Result.unknown

    if is_finished:
        build.status = Status.finished
    else:
        # ensure we dont set the status to finished unless it actually is
        new_status = aggregate_status((j.status for j in all_jobs))
        if new_status != Status.finished:
            build.status = new_status

    if db.session.is_modified(build):
        build.date_modified = datetime.utcnow()
        db.session.add(build)
        db.session.commit()

    if not is_finished:
        raise sync_build.NotFinished

    try:
        aggregate_build_stat(build, 'test_count')
        aggregate_build_stat(build, 'test_duration')
        aggregate_build_stat(build, 'test_failures')
        aggregate_build_stat(build, 'test_rerun_count')
        aggregate_build_stat(build, 'tests_missing')
        aggregate_build_stat(build, 'lines_covered')
        aggregate_build_stat(build, 'lines_uncovered')
        aggregate_build_stat(build, 'diff_lines_covered')
        aggregate_build_stat(build, 'diff_lines_uncovered')
    except Exception:
        current_app.logger.exception('Failing recording aggregate stats for build %s', build.id)

    fire_signal.delay(
        signal='build.finished',
        kwargs={'build_id': build.id.hex},
    )

    queue.delay('update_project_stats', kwargs={
        'project_id': build.project_id.hex,
    }, countdown=1)
Ejemplo n.º 15
0
def sync_job(job_id):
    job = Job.query.get(job_id)
    if not job:
        return

    if job.status == Status.finished:
        return

    # TODO(dcramer): we make an assumption that there is a single step
    job_plan = JobPlan.query.options(
        subqueryload_all('plan.steps')
    ).filter(
        JobPlan.job_id == job.id,
    ).join(Plan).first()
    try:
        if not job_plan:
            raise UnrecoverableException('Got sync_job task without job plan: %s' % (job.id,))

        try:
            step = job_plan.plan.steps[0]
        except IndexError:
            raise UnrecoverableException('Missing steps for plan')

        implementation = step.get_implementation()

        if has_timed_out(job, job_plan):
            remaining_steps = list(JobStep.query.filter(
                JobStep.status != Status.finished,
                JobStep.job_id == job.id,
            ))

            implementation.cancel(job=job)

            for step in remaining_steps:
                step.result = Result.failed
                step.status = Status.finished
                db.session.add(step)

                try_create(FailureReason, {
                    'step_id': step.id,
                    'job_id': job.id,
                    'build_id': job.build_id,
                    'project_id': job.project_id,
                    'reason': 'timeout'
                })

            # ensure the job result actually reflects a failure
            job.result = Result.failed
            job.status = Status.finished
            db.session.add(job)
        else:
            implementation.update(job=job)

    except UnrecoverableException:
        job.status = Status.finished
        job.result = Result.aborted
        current_app.logger.exception('Unrecoverable exception syncing %s', job.id)

    is_finished = sync_job.verify_all_children() == Status.finished
    if is_finished:
        job.status = Status.finished

    db.session.flush()

    all_phases = list(job.phases)

    # propagate changes to any phases as they live outside of the
    # normalize synchronization routines
    sync_job_phases(job, all_phases)

    job.date_started = safe_agg(
        min, (j.date_started for j in all_phases if j.date_started))

    if is_finished:
        job.date_finished = safe_agg(
            max, (j.date_finished for j in all_phases if j.date_finished))
    else:
        job.date_finished = None

    if job.date_started and job.date_finished:
        job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000)
    else:
        job.duration = None

    # if any phases are marked as failing, fail the build
    if any(j.result is Result.failed for j in all_phases):
        job.result = Result.failed
    # if any test cases were marked as failing, fail the build
    elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first():
        job.result = Result.failed
    # if we've finished all phases, use the best result available
    elif is_finished:
        job.result = safe_agg(max, (j.result for j in all_phases))
    else:
        job.result = Result.unknown

    if is_finished:
        job.status = Status.finished
    elif any(j.status is not Status.queued for j in all_phases):
        job.status = Status.in_progress
    else:
        job.status = Status.queued

    if db.session.is_modified(job):
        job.date_modified = datetime.utcnow()

        db.session.add(job)
        db.session.commit()

    if not is_finished:
        raise sync_job.NotFinished

    try:
        aggregate_job_stat(job, 'test_count')
        aggregate_job_stat(job, 'test_duration')
        aggregate_job_stat(job, 'test_failures')
        aggregate_job_stat(job, 'test_rerun_count')
        aggregate_job_stat(job, 'tests_missing')
        aggregate_job_stat(job, 'lines_covered')
        aggregate_job_stat(job, 'lines_uncovered')
        aggregate_job_stat(job, 'diff_lines_covered')
        aggregate_job_stat(job, 'diff_lines_uncovered')
    except Exception:
        current_app.logger.exception('Failing recording aggregate stats for job %s', job.id)

    fire_signal.delay(
        signal='job.finished',
        kwargs={'job_id': job.id.hex},
    )

    if job_plan:
        queue.delay('update_project_plan_stats', kwargs={
            'project_id': job.project_id.hex,
            'plan_id': job_plan.plan_id.hex,
        }, countdown=1)
Ejemplo n.º 16
0
def sync_job(job_id):
    """
    Updates jobphase and job statuses based on the status of the constituent jobsteps.
    """
    job = Job.query.get(job_id)
    if not job:
        return

    if job.status == Status.finished:
        return

    jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)

    try:
        implementation.update(job=job)

    except UnrecoverableException:
        job.status = Status.finished
        job.result = Result.infra_failed
        current_app.logger.exception('Unrecoverable exception syncing %s', job.id)

    all_phases = list(job.phases)

    # propagate changes to any phases as they live outside of the
    # normalize synchronization routines
    sync_job_phases(job, all_phases, implementation)

    is_finished = sync_job.verify_all_children() == Status.finished
    if any(p.status != Status.finished for p in all_phases):
        is_finished = False

    job.date_started = safe_agg(
        min, (j.date_started for j in all_phases if j.date_started))

    if is_finished:
        job.date_finished = safe_agg(
            max, (j.date_finished for j in all_phases if j.date_finished))
    else:
        job.date_finished = None

    if job.date_started and job.date_finished:
        job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000)
    else:
        job.duration = None

    # if any phases are marked as failing, fail the build
    if any(j.result is Result.failed for j in all_phases):
        job.result = Result.failed
    # If any test cases were marked as failing, fail the build.
    # The exception is if the only failing test case occurred in a JobStep that
    # had an infra failure. In this case we can't trust the test case result as
    # being meaningful and so we ignore these.
    elif TestCase.query.join(JobStep, JobStep.id == TestCase.step_id).filter(
                    TestCase.result == Result.failed, TestCase.job_id == job.id,
                    JobStep.result != Result.infra_failed
    ).first():
        job.result = Result.failed
    # if we've finished all phases, use the best result available
    elif is_finished:
        # Sets the final job result.
        implementation.validate(job=job)
    else:
        job.result = Result.unknown

    if is_finished:
        job.status = Status.finished
    else:
        # ensure we dont set the status to finished unless it actually is
        new_status = aggregate_status((j.status for j in all_phases))
        if new_status != Status.finished:
            job.status = new_status
        elif job.status == Status.finished:
            job.status = Status.in_progress
            current_app.logger.exception('Job incorrectly marked as finished: %s', job.id)

    if db.session.is_modified(job):
        job.date_modified = datetime.utcnow()

        db.session.add(job)
        db.session.commit()

    if not is_finished:
        raise sync_job.NotFinished

    try:
        aggregate_job_stat(job, 'test_count')
        aggregate_job_stat(job, 'test_duration')
        aggregate_job_stat(job, 'test_failures')
        aggregate_job_stat(job, 'test_rerun_count')
        aggregate_job_stat(job, 'tests_missing')
        aggregate_job_stat(job, 'lines_covered')
        aggregate_job_stat(job, 'lines_uncovered')
        aggregate_job_stat(job, 'diff_lines_covered')
        aggregate_job_stat(job, 'diff_lines_uncovered')
    except Exception:
        current_app.logger.exception('Failing recording aggregate stats for job %s', job.id)

    fire_signal.delay(
        signal='job.finished',
        kwargs={'job_id': job.id.hex},
    )

    if jobplan:
        queue.delay('update_project_plan_stats', kwargs={
            'project_id': job.project_id.hex,
            'plan_id': jobplan.plan_id.hex,
        }, countdown=1)
Ejemplo n.º 17
0
def sync_job(job_id):
    job = Job.query.get(job_id)
    if not job:
        return

    if job.status == Status.finished:
        return

    # TODO(dcramer): we make an assumption that there is a single step
    jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)

    try:
        implementation.update(job=job)

    except UnrecoverableException:
        job.status = Status.finished
        job.result = Result.aborted
        current_app.logger.exception('Unrecoverable exception syncing %s',
                                     job.id)

    is_finished = sync_job.verify_all_children() == Status.finished
    if is_finished:
        job.status = Status.finished

    db.session.flush()

    all_phases = list(job.phases)

    # propagate changes to any phases as they live outside of the
    # normalize synchronization routines
    sync_job_phases(job, all_phases)

    job.date_started = safe_agg(min, (j.date_started
                                      for j in all_phases if j.date_started))

    if is_finished:
        job.date_finished = safe_agg(max,
                                     (j.date_finished
                                      for j in all_phases if j.date_finished))
    else:
        job.date_finished = None

    if job.date_started and job.date_finished:
        job.duration = int(
            (job.date_finished - job.date_started).total_seconds() * 1000)
    else:
        job.duration = None

    # if any phases are marked as failing, fail the build
    if any(j.result is Result.failed for j in all_phases):
        job.result = Result.failed
    # if any test cases were marked as failing, fail the build
    elif TestCase.query.filter(TestCase.result == Result.failed,
                               TestCase.job_id == job.id).first():
        job.result = Result.failed
    # if we've finished all phases, use the best result available
    elif is_finished:
        job.result = safe_agg(max, (j.result for j in all_phases))
    else:
        job.result = Result.unknown

    if is_finished:
        job.status = Status.finished
    elif any(j.status is not Status.queued for j in all_phases):
        job.status = Status.in_progress
    else:
        job.status = Status.queued

    if db.session.is_modified(job):
        job.date_modified = datetime.utcnow()

        db.session.add(job)
        db.session.commit()

    if not is_finished:
        raise sync_job.NotFinished

    try:
        aggregate_job_stat(job, 'test_count')
        aggregate_job_stat(job, 'test_duration')
        aggregate_job_stat(job, 'test_failures')
        aggregate_job_stat(job, 'test_rerun_count')
        aggregate_job_stat(job, 'tests_missing')
        aggregate_job_stat(job, 'lines_covered')
        aggregate_job_stat(job, 'lines_uncovered')
        aggregate_job_stat(job, 'diff_lines_covered')
        aggregate_job_stat(job, 'diff_lines_uncovered')
    except Exception:
        current_app.logger.exception(
            'Failing recording aggregate stats for job %s', job.id)

    fire_signal.delay(
        signal='job.finished',
        kwargs={'job_id': job.id.hex},
    )

    if jobplan:
        queue.delay('update_project_plan_stats',
                    kwargs={
                        'project_id': job.project_id.hex,
                        'plan_id': jobplan.plan_id.hex,
                    },
                    countdown=1)
Ejemplo n.º 18
0
def sync_build(build_id):
    """
    Synchronizing the build happens continuously until all jobs have reported in
    as finished or have failed/aborted.

    This task is responsible for:
    - Checking in with jobs
    - Aborting/retrying them if they're beyond limits
    - Aggregating the results from jobs into the build itself
    """
    build = Build.query.get(build_id)
    if not build:
        return

    if build.status == Status.finished:
        return

    all_jobs = list(Job.query.filter(Job.build_id == build_id, ))

    is_finished = sync_build.verify_all_children() == Status.finished

    build.date_started = safe_agg(min, (j.date_started
                                        for j in all_jobs if j.date_started))

    if is_finished:
        build.date_finished = safe_agg(max,
                                       (j.date_finished
                                        for j in all_jobs if j.date_finished))
    else:
        build.date_finished = None

    if build.date_started and build.date_finished:
        build.duration = int(
            (build.date_finished - build.date_started).total_seconds() * 1000)
    else:
        build.duration = None

    if any(j.result is Result.failed for j in all_jobs):
        build.result = Result.failed
    elif is_finished:
        build.result = safe_agg(max, (j.result for j in all_jobs))
    else:
        build.result = Result.unknown

    if is_finished:
        build.status = Status.finished
    elif any(j.status is not Status.queued for j in all_jobs):
        build.status = Status.in_progress
    else:
        build.status = Status.queued

    if db.session.is_modified(build):
        build.date_modified = datetime.utcnow()
        db.session.add(build)
        db.session.commit()

    if not is_finished:
        raise sync_build.NotFinished

    try:
        aggregate_build_stat(build, 'test_count')
        aggregate_build_stat(build, 'test_duration')
        aggregate_build_stat(build, 'test_failures')
        aggregate_build_stat(build, 'test_rerun_count')
        aggregate_build_stat(build, 'tests_missing')
        aggregate_build_stat(build, 'lines_covered')
        aggregate_build_stat(build, 'lines_uncovered')
        aggregate_build_stat(build, 'diff_lines_covered')
        aggregate_build_stat(build, 'diff_lines_uncovered')
    except Exception:
        current_app.logger.exception(
            'Failing recording aggregate stats for build %s', build.id)

    fire_signal.delay(
        signal='build.finished',
        kwargs={'build_id': build.id.hex},
    )

    queue.delay('update_project_stats',
                kwargs={
                    'project_id': build.project_id.hex,
                },
                countdown=1)
Ejemplo n.º 19
0
def sync_job(job_id):
    job = Job.query.get(job_id)
    if not job:
        return

    if job.status == Status.finished:
        return

    # TODO(dcramer): we make an assumption that there is a single step
    job_plan = JobPlan.query.options(
        subqueryload_all('plan.steps')
    ).filter(
        JobPlan.job_id == job.id,
    ).join(Plan).first()
    try:
        if not job_plan:
            raise UnrecoverableException('Got sync_job task without job plan: %s' % (job.id,))

        try:
            step = job_plan.plan.steps[0]
        except IndexError:
            raise UnrecoverableException('Missing steps for plan')

        implementation = step.get_implementation()
        implementation.update(job=job)

    except UnrecoverableException:
        job.status = Status.finished
        job.result = Result.aborted
        current_app.logger.exception('Unrecoverable exception syncing %s', job.id)

    is_finished = sync_job.verify_all_children() == Status.finished
    if is_finished:
        job.status = Status.finished

    all_phases = list(job.phases)

    job.date_started = safe_agg(
        min, (j.date_started for j in all_phases if j.date_started))

    if is_finished:
        job.date_finished = safe_agg(
            max, (j.date_finished for j in all_phases if j.date_finished))
    else:
        job.date_finished = None

    if job.date_started and job.date_finished:
        job.duration = int((job.date_finished - job.date_started).total_seconds() * 1000)
    else:
        job.duration = None

    # if any phases are marked as failing, fail the build
    if any(j.result is Result.failed for j in all_phases):
        job.result = Result.failed
    # if any test cases were marked as failing, fail the build
    elif TestCase.query.filter(TestCase.result == Result.failed, TestCase.job_id == job.id).first():
        job.result = Result.failed
    # if we've finished all phases, use the best result available
    elif is_finished:
        job.result = safe_agg(
            max, (j.result for j in all_phases), Result.unknown)
    else:
        job.result = Result.unknown

    if is_finished:
        job.status = Status.finished
    elif any(j.status is Status.in_progress for j in all_phases):
        job.status = Status.in_progress
    else:
        job.status = Status.queued

    if db.session.is_modified(job):
        job.date_modified = datetime.utcnow()

        db.session.add(job)
        db.session.commit()
        publish_job_update(job)

    if not is_finished:
        raise sync_job.NotFinished

    _record_tests_missing(job)

    queue.delay('notify_job_finished', kwargs={
        'job_id': job.id.hex,
    })

    if job_plan:
        queue.delay('update_project_plan_stats', kwargs={
            'project_id': job.project_id.hex,
            'plan_id': job_plan.plan_id.hex,
        }, countdown=1)
Ejemplo n.º 20
0
    def _sync_step_from_active(self, step):
        try:
            job_name = step.data['job_name']
            build_no = step.data['build_no']
        except KeyError:
            raise UnrecoverableException('Missing Jenkins job information')

        try:
            item = self._get_response('/job/{}/{}'.format(
                job_name, build_no))
        except NotFound:
            raise UnrecoverableException('Unable to find job in Jenkins')

        # TODO(dcramer): we're doing a lot of work here when we might
        # not need to due to it being sync'd previously
        node, _ = get_or_create(Node, where={
            'label': item['builtOn'],
        })

        step.node = node
        step.label = item['fullDisplayName']
        step.date_started = datetime.utcfromtimestamp(
            item['timestamp'] / 1000)

        if item['building']:
            step.status = Status.in_progress
        else:
            step.status = Status.finished
            step.result = RESULT_MAP[item['result']]
            # values['duration'] = item['duration'] or None
            step.date_finished = datetime.utcfromtimestamp(
                (item['timestamp'] + item['duration']) / 1000)

        # step.data.update({
        #     'backend': {
        #         'uri': item['url'],
        #         'label': item['fullDisplayName'],
        #     }
        # })
        db.session.add(step)
        db.session.commit()

        # TODO(dcramer): we shoudl abstract this into a sync_phase
        phase = step.phase

        if not phase.date_started:
            phase.date_started = safe_agg(
                min, (s.date_started for s in phase.steps), step.date_started)
            db.session.add(phase)

        if phase.status != step.status:
            phase.status = step.status
            db.session.add(phase)

        if step.status == Status.finished:
            phase.status = Status.finished
            phase.date_finished = safe_agg(
                max, (s.date_finished for s in phase.steps), step.date_finished)

            if any(s.result is Result.failed for s in phase.steps):
                phase.result = Result.failed
            else:
                phase.result = safe_agg(
                    max, (s.result for s in phase.steps), Result.unknown)

            db.session.add(phase)

        db.session.commit()

        if step.status != Status.finished:
            return

        # sync artifacts
        for artifact in item.get('artifacts', ()):
            artifact, created = get_or_create(Artifact, where={
                'step': step,
                'name': artifact['fileName'],
            }, defaults={
                'project': step.project,
                'job': step.job,
                'data': artifact,
            })
            db.session.commit()
            sync_artifact.delay_if_needed(
                artifact_id=artifact.id.hex,
                task_id=artifact.id.hex,
                parent_task_id=step.id.hex,
            )

        # sync test results
        try:
            with db.session.begin_nested():
                self._sync_test_results(
                    step=step,
                    job_name=job_name,
                    build_no=build_no,
                )
        except Exception:
            self.logger.exception(
                'Failed to sync test results for %s #%s', job_name, build_no)
        else:
            db.session.commit()

        # sync console log
        try:
            result = True
            while result:
                result = self._sync_log(
                    jobstep=step,
                    name=step.label,
                    job_name=job_name,
                    build_no=build_no,
                )

        except Exception:
            db.session.rollback()
            current_app.logger.exception(
                'Unable to sync console log for job step %r',
                step.id.hex)