def _snapshot_to_realization_progress(self, snapshot): realization_progress = {} for iens, real in snapshot["reals"].items(): iens_int = int(iens) queue_state_str = real["stages"]["0"].get("queue_state") queue_state = (None if queue_state_str is None else JobStatusType.from_string(queue_state_str)) realization_progress[iens_int] = ( [ ForwardModelJobStatus( job["name"], status=job["status"], start_time=dateutil.parser.parse(job["start_time"]) if job["start_time"] is not None else None, end_time=dateutil.parser.parse(job["end_time"]) if job["end_time"] is not None else None, current_memory_usage=job.get( "data", {}).get("current_memory_usage"), max_memory_usage=job.get("data", {}).get("max_memory_usage"), ) for job in real["stages"]["0"]["steps"]["0"] ["jobs"].values() ], queue_state, ) return realization_progress
def _snapshot_to_realization_progress(self, snapshot): realization_progress = {} for iens, real in snapshot["reals"].items(): iens_int = int(iens) queue_state_str = ids.STATUS_QUEUE_STATE.get( real.get("status"), None) queue_state = (None if queue_state_str is None else JobStatusType.from_string(queue_state_str)) job_statuses = {} for real_id, stage in real.get("stages", {}).items(): for step_id, step in stage.get("steps", {}).items(): for job_id, job in step.get("jobs", {}).items(): job_statuses[job_id] = ForwardModelJobStatus( job["name"], status=job["status"], start_time=dateutil.parser.parse(job["start_time"]) if job["start_time"] is not None else None, end_time=dateutil.parser.parse(job["end_time"]) if job["end_time"] is not None else None, current_memory_usage=job.get( "data", {}).get("current_memory_usage"), max_memory_usage=job.get( "data", {}).get("max_memory_usage"), ) realization_progress[iens_int] = ( job_statuses, queue_state, ) return realization_progress
def snapshot(self) -> typing.Optional[typing.Dict[int, str]]: """Return the whole state, or None if there was no snapshot.""" snapshot = {} for q_index, state_val in enumerate(self._state): st = str(JobStatusType(state_val)) try: snapshot[self._qindex_to_iens[q_index]] = st except KeyError as e: logger.debug(f"differ could produce no snapshot due to {e}") return None return snapshot
def __checkForUnusedEnums(self): for enum in JobStatusType.enums(): # The status check routines can return this status; if e.g. the bjobs call fails, # but a job will never get this status. if enum == JobStatusType.JOB_QUEUE_STATUS_FAILURE: continue used = False for state in self.states: if enum in state.state: used = True if not used: raise AssertionError("Enum identifier '%s' not used!" % enum)
def _diff_states( self, old_state: typing.List[JobStatusType], new_state: typing.List[JobStatusType], ) -> typing.Dict[int, str]: """Return the diff between old_state and new_state.""" changes = {} diff = list(map(lambda s: s[0] == s[1], zip(old_state, new_state))) if len(diff) > 0: for q_index, equal in enumerate(diff): if not equal: st = str(JobStatusType(new_state[q_index])) changes[self._qindex_to_iens[q_index]] = st return changes
def _update_states(self, snapshot_updates): realization_progress = self._get_most_recent_snapshot() if "reals" in snapshot_updates: for iens, real in snapshot_updates["reals"].items(): iens_int = int(iens) job_statuses = realization_progress[iens_int][0] queue_state = realization_progress[iens_int][1] if (real.get("stages", {}).get("0", {}).get("steps", {}).get("0", {}).get("status")): step_state = real["stages"]["0"]["steps"]["0"]["status"] if real.get("stages", {}).get("0", {}).get("queue_state"): queue_state_str = real["stages"]["0"].get("queue_state") queue_state = JobStatusType.from_string(queue_state_str) if "jobs" in real.get("stages", {}).get("0", {}).get("steps", {}).get("0", {}): for job_id, job in real["stages"]["0"]["steps"]["0"][ "jobs"].items(): job_id = int(job_id) status = realization_progress[iens_int][0][job_id] if job.get("status"): status.status = job["status"] if job.get("start_time"): status.start_time = dateutil.parser.parse( job["start_time"]) if job.get("end_time"): status.end_time = dateutil.parser.parse( job["end_time"]) if job.get("data", {}).get("current_memory_usage"): status.current_memory_usage = job["data"][ "current_memory_usage"] if job.get("data", {}).get("max_memory_usage"): status.max_memory_usage = job["data"][ "max_memory_usage"] realization_progress[iens_int] = ( job_statuses, queue_state, )
def _update_states(self, snapshot_updates): realization_progress = self._get_most_recent_snapshot() if "reals" in snapshot_updates: for iens, real in snapshot_updates["reals"].items(): iens_int = int(iens) job_statuses = realization_progress[iens_int][0] queue_state = realization_progress[iens_int][1] if real.get("status"): queue_state_str = ids.STATUS_QUEUE_STATE.get( real.get("status")) queue_state = JobStatusType.from_string(queue_state_str) for stage_id, stage in real.get("stages", {}).items(): for step_id, step in stage.get("steps", {}).items(): for job_id, job in step.get("jobs", {}).items(): status = realization_progress[iens_int][0][job_id] if job.get("status"): status.status = job["status"] if job.get("start_time"): status.start_time = dateutil.parser.parse( job["start_time"]) if job.get("end_time"): status.end_time = dateutil.parser.parse( job["end_time"]) if job.get("data", {}).get("current_memory_usage"): status.current_memory_usage = job["data"][ "current_memory_usage"] if job.get("data", {}).get("max_memory_usage"): status.max_memory_usage = job["data"][ "max_memory_usage"] if job.get("error"): status.error = job.get("error") realization_progress[iens_int] = ( job_statuses, queue_state, )
def getJobStatus(self, job_index): # See comment about return type in the prototype section at # the top of class. """ @rtype: res.job_queue.job_status_type_enum.JobStatusType """ int_status = self._job_status(job_index) return JobStatusType(int_status)
def getJobStatus(self, job_number): # See comment about return type in the prototype section at # the top of class. """ @rtype: JobStatusType """ int_status = self._get_job_status(job_number) return JobStatusType( int_status )