def standard(self) -> RunModel: jobs = [] s = DateUtil.parse_iso(self.meta.get("start")) f = DateUtil.parse_iso(self.meta.get("end")) st = 0 if s: ff = f if f else DateUtil.now() st = int(DateUtil.secs_difference(s, ff)) jid = self.meta.get("id") for stepname, call in self.meta.get("calls").items(): stepname = ".".join(stepname.split(".")[1:]) jobs.extend(self.parse_standard_calls(None, stepname, call)) model = RunModel( submission_id=None, id_=None, engine_id=jid, name=self.meta.get("workflowName"), # start=s, # finish=f, execution_dir=self.meta.get("workflowRoot"), status=cromwell_status_to_status(self.meta.get("status")), error=self.get_caused_by_text(), ) model.jobs = jobs return model
def __init__( self, id_: str, submission_id: str, run_id: str, output_name: Optional[str], output_folder: Optional[str], secondaries: Optional[List[str]], extension: Optional[str], is_copyable: bool, original_path: Optional[str], new_path: Optional[str], timestamp: Optional[Union[str, datetime.datetime]], value: Optional[any] = None, ): self.id_ = id_ self.submission_id = submission_id self.run_id = run_id if isinstance(output_name, bool): self.output_name = output_name else: self.output_name = stringify_value_or_array(output_name) self.output_folder = stringify_value_or_array(output_folder) self.secondaries = secondaries self.extension = extension self.is_copyable = is_copyable self.original_path = original_path self.new_path = new_path self.value = value if not isinstance(timestamp, datetime.datetime): timestamp = DateUtil.parse_iso(timestamp) self.timestamp = timestamp
def get_all(self) -> Dict[str, datetime]: with self.with_cursor() as cursor: cursor.execute( "SELECT key, timestamp FROM progress WHERE submission_id = ?", (self.submission_id, ), ) rows = cursor.fetchall() return {row[0]: DateUtil.parse_iso(row[1]) for row in rows}
def __init__(self, submission_id: str, run_id: str, status: TaskStatus, date: datetime): self.submission_id = submission_id self.run_id = run_id self.status = status if isinstance(status, TaskStatus) else TaskStatus(status) if not isinstance(date, datetime): date = DateUtil.parse_iso(date) self.date = date
def _generate_call_times_from_calls( cls, calls, prefix="", include_subworkflow_total=True ) -> dict: dcalls = {} for call_key in calls: call = calls[call_key][0] s, f = call["start"], call.get("end") sd = DateUtil.parse_iso(s) fd = DateUtil.parse_iso(f) if f else None prefixed_call_key = prefix + call_key if prefixed_call_key in dcalls: Logger.warn( "Doesn't know how to handle multiple instances of calls, skipping this one (with id: " + call["id"] ) continue is_subworkflow_total = "subWorkflowMetadata" in call if is_subworkflow_total: dcalls.update( cls._generate_call_times_from_calls( call["subWorkflowMetadata"]["calls"], prefix=call_key + "." ) ) if not is_subworkflow_total or include_subworkflow_total: dcalls[prefixed_call_key] = { "start": s, "end": f, "time": (fd - sd).total_seconds() if fd else "N/A", } return dcalls
def __init__( self, submission_id, output_dir: str, execution_dir: str, timestamp: Optional[Union[str, datetime]] = None, ): self.submission_id = submission_id self.output_dir = output_dir self.execution_dir = execution_dir if not timestamp: timestamp = DateUtil.now() elif isinstance(timestamp, str): timestamp = DateUtil.parse_iso(timestamp) self.timestamp = timestamp
def __init__( self, id_: str, submission_id: str, engine_id: str, status: TaskStatus, execution_dir: str, name: str, error: str = None, labels: List[str] = None, tags: List[str] = None, last_updated: datetime = None, # extra fields not populated by DB jobs: List[RunJobModel] = None, inputs: List[WorkflowInputModel] = None, outputs: List[WorkflowOutputModel] = None, events: List[RunStatusUpdate] = None, ): self.id_ = id_ self.submission_id = submission_id self.execution_dir = execution_dir self.engine_id = engine_id self.name = name self.status = status self.error = error self.labels = labels self.tags = tags self.jobs = jobs self.inputs = inputs self.outputs = outputs if isinstance(last_updated, str): last_updated = DateUtil.parse_iso(last_updated) self.last_updated = last_updated # to be calculated self.start = None self.finish = None # call setters self.events = None self.set_events(events)
def __init__( self, id_: str, output_dir: str, execution_dir: str, author: str, labels: List[str], tags: List[str], timestamp: Union[str, datetime], engine_type: str, # metadata not populated directly by DB, but might be used for formatting engine_url: Optional[str] = None, runs: List[RunModel] = None, name: Optional[str] = None, status: Optional[TaskStatus] = None, error: Optional[str] = None # From previous WorkflowModel, some data is now derived: # runs: # - Start / finish times # - Errors # - Last updated ): self.id_ = id_ self.output_dir = output_dir self.execution_dir = execution_dir self.author = author self.tags = tags self.labels = labels if not isinstance(timestamp, datetime): timestamp = DateUtil.parse_iso(timestamp) self.timestamp = timestamp self.engine_type = engine_type # other things self.runs = runs or [] self.engine_url = engine_url self.name = name self.status = status self.error = error
def parse_standard_call(cls, parentid, stepname, call): parent = parentid jid = f"{parentid}_{stepname}" if parentid else stepname shard = call.get("shardIndex") attempt = call.get("attempt") is_shard = shard is not None and shard >= 0 is_second_attempt = attempt is not None and attempt > 1 # We'll rebase the parent job if there are any shards (so they all sit under one heading) # OR if it's a second attempt, we'll group it under the first attempt. if is_shard: parent = jid # We rely on the calling function to create a job with the parentId we're setting. jid += f"_shard-{shard}" if is_second_attempt: # There will already be a parent with this id_ (as this is the second attempt) parent = jid jid += f"_attempt-{attempt}" subjobs = [] status = cromwell_status_to_status( call.get("status") or call.get("executionStatus") ) start = call.get("start") finish = call.get("end") # if the call has "subworkflowMetadata", we have a different schema: if "subWorkflowMetadata" in call: # it's actually a workflow submeta = call.get("subWorkflowMetadata") for sn, subcalls in submeta.get("calls", {}).items(): sn = ".".join(sn.split(".")[1:]) subjobs.extend(cls.parse_standard_calls(jid, sn, subcalls)) start = start or min(s.start for s in subjobs) finish = finish callroot = call.get("callRoot") return RunJobModel( id_=jid, submission_id=None, run_id=None, parent=parent, container=call.get( "dockerImageUsed", call.get("runtimeAttributes", {}).get("docker") ), name=stepname, status=status, batchid=call.get("jobId"), backend=None, stdout=call.get("stdout"), stderr=call.get("stderr"), start=DateUtil.parse_iso(start) if start is not None else DateUtil.now(), finish=DateUtil.parse_iso(finish) if finish is not None else None, jobs=subjobs or None, cached=call.get("callCaching").get("hit") if "callCaching" in call else False, shard=shard, attempt=attempt, analysis=None, memory=None, cpu=None, error=CromwellMetadata.unwrap_caused_by(call.get("failures")), returncode=call.get("returnCode"), workdir=call.get("callRoot"), script=("file:/" + os.path.join(callroot, "execution/script")) if callroot else None, )
def __init__( self, id_: str, submission_id: str, run_id: str, parent: Optional[str], name: str, batchid: Optional[str] = None, shard: Optional[int] = None, attempt: Optional[int] = None, container: Optional[str] = None, status: TaskStatus = None, start: Union[str, datetime] = None, finish: Optional[Union[str, datetime]] = None, backend: Optional[str] = None, cached: bool = None, stdout: Optional[str] = None, stderr: Optional[str] = None, script: Optional[str] = None, error: Optional[str] = None, returncode: Optional[str] = None, memory: Optional[str] = None, cpu: Optional[str] = None, analysis: Optional[str] = None, # Optional jobs: Optional[list] = None, lastupdated: Union[str, datetime] = None, workdir: Optional[str] = None, ): self.id_ = id_ self.submission_id = submission_id self.run_id = run_id self.parent = parent self.status = status if isinstance(status, TaskStatus) else TaskStatus(status) self.name = name self.batchid = batchid self.shard = None if shard is not None: if isinstance(shard, str) and shard.isdigit: shard = int(shard) if shard >= 0: self.shard = shard self.attempt = None if attempt is not None: if isinstance(attempt, str) and attempt.isdigit(): attempt = int(attempt) if attempt > 1: self.attempt = attempt self.container = container self.backend = backend self.cached = cached self.stderr = stderr self.stdout = stdout self.error = error self.returncode = returncode self.workdir = workdir self.lastupdated = lastupdated or DateUtil.now() if isinstance(lastupdated, str): self.lastupdated = DateUtil.parse_iso(lastupdated) self.start = start self.finish = finish if start and isinstance(start, str): self.start = DateUtil.parse_iso(start) if finish and isinstance(finish, str): self.finish = DateUtil.parse_iso(finish) self.script = script self.memory = memory self.cpu = cpu self.analysis = analysis self.jobs: Optional[List[RunJobModel]] = jobs or None self.events = None
def __init__(self, jid: str, status: TaskStatus, timestamp: str): self.jid = jid self.status = status self.timestamp = timestamp if timestamp and isinstance(timestamp, str): self.timestamp = DateUtil.parse_iso(timestamp)