def test_correct_format(self): project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) handler = ManifestJsonHandler(jobstep) fp = StringIO(self.json_file_format % jobstep.id.hex) handler.process(fp) assert not FailureReason.query.filter(FailureReason.step_id == jobstep.id).first()
def test_correct_format(self): project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) artifact = self.create_artifact(jobstep, 'manifest.json') handler = ManifestJsonHandler(jobstep) fp = StringIO(self.json_file_format % jobstep.id.hex) handler.process(fp, artifact) assert not FailureReason.query.filter( FailureReason.step_id == jobstep.id).first()
def test_malformed(self): project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) artifact = self.create_artifact(jobstep, 'manifest.json') handler = ManifestJsonHandler(jobstep) fp = StringIO('invalid_file') handler.process(fp, artifact) assert Result.infra_failed assert FailureReason.query.filter( FailureReason.step_id == jobstep.id, FailureReason.reason == 'malformed_manifest_json', ).first()
def test_malformed(self): project = self.create_project() build = self.create_build(project) job = self.create_job(build) jobphase = self.create_jobphase(job) jobstep = self.create_jobstep(jobphase) handler = ManifestJsonHandler(jobstep) # TODO(nate): temporarily disabled if False: fp = StringIO('invalid_file') handler.process(fp) assert Result.infra_failed assert FailureReason.query.filter( FailureReason.step_id == jobstep.id, FailureReason.reason == 'malformed_manifest_json', ).first()
def verify_final_artifacts(self, step, artifacts): # If the Jenkins run was aborted, we don't expect a manifest file. if (step.result != Result.aborted and not any(ManifestJsonHandler.can_process(a.name) for a in artifacts)): db.session.add(FailureReason( step_id=step.id, job_id=step.job.id, build_id=step.job.build_id, project_id=step.job.project_id, reason='missing_manifest_json', )) step.result = Result.infra_failed db.session.add(step) db.session.commit()
def verify_final_artifacts(self, step, artifacts): # If the Jenkins run was aborted or timed out, we don't expect a manifest file. if (step.result != Result.aborted and not step.data.get('timed_out', False) and not any(ManifestJsonHandler.can_process(a.name) for a in artifacts)): db.session.add(FailureReason( step_id=step.id, job_id=step.job.id, build_id=step.job.build_id, project_id=step.job.project_id, reason='missing_manifest_json', )) step.result = Result.infra_failed db.session.add(step) db.session.commit()
def _sync_results(self, step, item): job_name = step.data['job_name'] build_no = step.data['build_no'] artifacts = item.get('artifacts', ()) # If the Jenkins run was aborted, we don't expect a manifest file. if (step.result != Result.aborted and not any(ManifestJsonHandler.can_process(os.path.basename(a['fileName'])) for a in artifacts)): db.session.add(FailureReason( step_id=step.id, job_id=step.job.id, build_id=step.job.build_id, project_id=step.job.project_id, reason='missing_manifest_json', )) step.result = Result.infra_failed db.session.add(step) db.session.commit() # Detect and warn if there are duplicate artifact file names as we were relying on # uniqueness before. artifact_filenames = set() for artifact in artifacts: if artifact['fileName'] in artifact_filenames: self.logger.warning('Duplicate artifact filename found: %s', artifact['fileName']) artifact_filenames.add(artifact['fileName']) self._sync_generic_results(step, artifacts) # sync console log self.logger.info('Syncing console log for %s', step.id) try: result = True while result: result = self._sync_log( jobstep=step, name=step.label, job_name=job_name, build_no=build_no, ) except Exception: db.session.rollback() current_app.logger.exception( 'Unable to sync console log for job step %r', step.id.hex)
def _sync_results(self, step, item): job_name = step.data["job_name"] build_no = step.data["build_no"] artifacts = item.get("artifacts", ()) if self.sync_phase_artifacts: # if we are allowing phase artifacts and we find *any* artifacts # that resemble a phase we need to change the behavior of the # the remainder of tasks phased_results = any(a["fileName"].endswith("phase.json") for a in artifacts) else: phased_results = False # If the Jenkins run was aborted, we don't expect a manifest file. if step.result != Result.aborted: if not any(ManifestJsonHandler.can_process(os.path.basename(a["fileName"])) for a in artifacts): db.session.add( FailureReason( step_id=step.id, job_id=step.job.id, build_id=step.job.build_id, project_id=step.job.project_id, reason="missing_manifest_json", ) ) step.result = Result.infra_failed db.session.add(step) db.session.commit() # artifacts sync differently depending on the style of job results if phased_results: self._sync_phased_results(step, artifacts) else: self._sync_generic_results(step, artifacts) # sync console log self.logger.info("Syncing console log for %s", step.id) try: result = True while result: result = self._sync_log(jobstep=step, name=step.label, job_name=job_name, build_no=build_no) except Exception: db.session.rollback() current_app.logger.exception("Unable to sync console log for job step %r", step.id.hex)
def _sync_phased_results(self, step, artifacts): # due to the limitations of Jenkins and our requirement to have more # insight into the actual steps a build process takes and unfortunately # the best way to do this is to rewrite history within Changes job = step.job is_diff = not job.source.is_commit() project = step.project artifacts_by_name = dict((a["fileName"], a) for a in artifacts) pending_artifacts = set(artifacts_by_name.keys()) phase_steps = set() phase_step_data = { "job_name": step.data["job_name"], "build_no": step.data["build_no"], "generated": True, # TODO: _pick_master here seems very suspicious. "master": self._pick_master(step.data["job_name"], is_diff), } phases = set() # fetch each phase and create it immediately (as opposed to async) for artifact_data in artifacts: artifact_filename = artifact_data["fileName"] if not artifact_filename.endswith("phase.json"): continue pending_artifacts.remove(artifact_filename) resp = self.fetch_artifact(step, artifact_data) phase_data = resp.json() if phase_data["retcode"]: result = Result.failed else: result = Result.passed date_started = datetime.utcfromtimestamp(phase_data["startTime"]) date_finished = datetime.utcfromtimestamp(phase_data["endTime"]) jobphase, created = get_or_create( JobPhase, where={"job": job, "label": phase_data["name"]}, defaults={ "project": project, "result": result, "status": Status.finished, "date_started": date_started, "date_finished": date_finished, }, ) phases.add(jobphase) jobstep, created = get_or_create( JobStep, where={"phase": jobphase, "label": step.label}, defaults={ "job": job, "node": step.node, "project": project, "result": result, "status": Status.finished, "date_started": date_started, "date_finished": date_finished, "data": phase_step_data, }, ) sync_job_step.delay_if_needed(task_id=jobstep.id.hex, parent_task_id=job.id.hex, step_id=jobstep.id.hex) phase_steps.add(jobstep) # capture the log if available try: log_artifact = artifacts_by_name[phase_data["log"]] except KeyError: self.logger.warning("Unable to find logfile for phase: %s", phase_data) else: pending_artifacts.remove(log_artifact["fileName"]) self._handle_generic_artifact(jobstep=jobstep, artifact=log_artifact, skip_checks=True) # ideally we don't mark the base step as a failure if any of the phases # report more correct results if phases and step.result == Result.failed and any(p.result == Result.failed for p in phases): step.result = Result.passed db.session.add(step) if not pending_artifacts: return # Alias to clarify that this is the JobStep that actually ran on a slave. original_step = step # all remaining artifacts get bound to the final phase final_step = sorted(phase_steps, key=lambda x: x.date_finished, reverse=True)[0] for artifact_name in pending_artifacts: # Manifest files are associated with the original step so we can validate that the ID is correct. responsible_step = original_step if ManifestJsonHandler.can_process(artifact_name) else final_step self._handle_generic_artifact(jobstep=responsible_step, artifact=artifacts_by_name[artifact_name])