def standard(self) -> RunModel:
jobs = []
s = DateUtil.parse_iso(self.meta.get("start"))
f = DateUtil.parse_iso(self.meta.get("end"))
st = 0
if s:
ff = f if f else DateUtil.now()
st = int(DateUtil.secs_difference(s, ff))
jid = self.meta.get("id")
for stepname, call in self.meta.get("calls").items():
stepname = ".".join(stepname.split(".")[1:])
jobs.extend(self.parse_standard_calls(None, stepname, call))
model = RunModel(
submission_id=None,
id_=None,
engine_id=jid,
name=self.meta.get("workflowName"),
# start=s,
# finish=f,
execution_dir=self.meta.get("workflowRoot"),
status=cromwell_status_to_status(self.meta.get("status")),
error=self.get_caused_by_text(),
)
model.jobs = jobs
return model
def __init__(
self,
id_: str,
submission_id: str,
run_id: str,
output_name: Optional[str],
output_folder: Optional[str],
secondaries: Optional[List[str]],
extension: Optional[str],
is_copyable: bool,
original_path: Optional[str],
new_path: Optional[str],
timestamp: Optional[Union[str, datetime.datetime]],
value: Optional[any] = None,
):
self.id_ = id_
self.submission_id = submission_id
self.run_id = run_id
if isinstance(output_name, bool):
self.output_name = output_name
else:
self.output_name = stringify_value_or_array(output_name)
self.output_folder = stringify_value_or_array(output_folder)
self.secondaries = secondaries
self.extension = extension
self.is_copyable = is_copyable
self.original_path = original_path
self.new_path = new_path
self.value = value
if not isinstance(timestamp, datetime.datetime):
timestamp = DateUtil.parse_iso(timestamp)
self.timestamp = timestamp
def get_all(self) -> Dict[str, datetime]:
with self.with_cursor() as cursor:
cursor.execute(
"SELECT key, timestamp FROM progress WHERE submission_id = ?",
(self.submission_id, ),
)
rows = cursor.fetchall()
return {row[0]: DateUtil.parse_iso(row[1]) for row in rows}
def __init__(self, submission_id: str, run_id: str, status: TaskStatus,
date: datetime):
self.submission_id = submission_id
self.run_id = run_id
self.status = status if isinstance(status,
TaskStatus) else TaskStatus(status)
if not isinstance(date, datetime):
date = DateUtil.parse_iso(date)
self.date = date
def _generate_call_times_from_calls(
cls, calls, prefix="", include_subworkflow_total=True
) -> dict:
dcalls = {}
for call_key in calls:
call = calls[call_key][0]
s, f = call["start"], call.get("end")
sd = DateUtil.parse_iso(s)
fd = DateUtil.parse_iso(f) if f else None
prefixed_call_key = prefix + call_key
if prefixed_call_key in dcalls:
Logger.warn(
"Doesn't know how to handle multiple instances of calls, skipping this one (with id: "
+ call["id"]
)
continue
is_subworkflow_total = "subWorkflowMetadata" in call
if is_subworkflow_total:
dcalls.update(
cls._generate_call_times_from_calls(
call["subWorkflowMetadata"]["calls"], prefix=call_key + "."
)
)
if not is_subworkflow_total or include_subworkflow_total:
dcalls[prefixed_call_key] = {
"start": s,
"end": f,
"time": (fd - sd).total_seconds() if fd else "N/A",
}
return dcalls
def __init__(
self,
submission_id,
output_dir: str,
execution_dir: str,
timestamp: Optional[Union[str, datetime]] = None,
):
self.submission_id = submission_id
self.output_dir = output_dir
self.execution_dir = execution_dir
if not timestamp:
timestamp = DateUtil.now()
elif isinstance(timestamp, str):
timestamp = DateUtil.parse_iso(timestamp)
self.timestamp = timestamp
def __init__(
self,
id_: str,
submission_id: str,
engine_id: str,
status: TaskStatus,
execution_dir: str,
name: str,
error: str = None,
labels: List[str] = None,
tags: List[str] = None,
last_updated: datetime = None,
# extra fields not populated by DB
jobs: List[RunJobModel] = None,
inputs: List[WorkflowInputModel] = None,
outputs: List[WorkflowOutputModel] = None,
events: List[RunStatusUpdate] = None,
):
self.id_ = id_
self.submission_id = submission_id
self.execution_dir = execution_dir
self.engine_id = engine_id
self.name = name
self.status = status
self.error = error
self.labels = labels
self.tags = tags
self.jobs = jobs
self.inputs = inputs
self.outputs = outputs
if isinstance(last_updated, str):
last_updated = DateUtil.parse_iso(last_updated)
self.last_updated = last_updated
# to be calculated
self.start = None
self.finish = None
# call setters
self.events = None
self.set_events(events)
def __init__(
self,
id_: str,
output_dir: str,
execution_dir: str,
author: str,
labels: List[str],
tags: List[str],
timestamp: Union[str, datetime],
engine_type: str,
# metadata not populated directly by DB, but might be used for formatting
engine_url: Optional[str] = None,
runs: List[RunModel] = None,
name: Optional[str] = None,
status: Optional[TaskStatus] = None,
error: Optional[str] = None
# From previous WorkflowModel, some data is now derived:
# runs:
# - Start / finish times
# - Errors
# - Last updated
):
self.id_ = id_
self.output_dir = output_dir
self.execution_dir = execution_dir
self.author = author
self.tags = tags
self.labels = labels
if not isinstance(timestamp, datetime):
timestamp = DateUtil.parse_iso(timestamp)
self.timestamp = timestamp
self.engine_type = engine_type
# other things
self.runs = runs or []
self.engine_url = engine_url
self.name = name
self.status = status
self.error = error
def parse_standard_call(cls, parentid, stepname, call):
parent = parentid
jid = f"{parentid}_{stepname}" if parentid else stepname
shard = call.get("shardIndex")
attempt = call.get("attempt")
is_shard = shard is not None and shard >= 0
is_second_attempt = attempt is not None and attempt > 1
# We'll rebase the parent job if there are any shards (so they all sit under one heading)
# OR if it's a second attempt, we'll group it under the first attempt.
if is_shard:
parent = jid
# We rely on the calling function to create a job with the parentId we're setting.
jid += f"_shard-{shard}"
if is_second_attempt:
# There will already be a parent with this id_ (as this is the second attempt)
parent = jid
jid += f"_attempt-{attempt}"
subjobs = []
status = cromwell_status_to_status(
call.get("status") or call.get("executionStatus")
)
start = call.get("start")
finish = call.get("end")
# if the call has "subworkflowMetadata", we have a different schema:
if "subWorkflowMetadata" in call:
# it's actually a workflow
submeta = call.get("subWorkflowMetadata")
for sn, subcalls in submeta.get("calls", {}).items():
sn = ".".join(sn.split(".")[1:])
subjobs.extend(cls.parse_standard_calls(jid, sn, subcalls))
start = start or min(s.start for s in subjobs)
finish = finish
callroot = call.get("callRoot")
return RunJobModel(
id_=jid,
submission_id=None,
run_id=None,
parent=parent,
container=call.get(
"dockerImageUsed", call.get("runtimeAttributes", {}).get("docker")
),
name=stepname,
status=status,
batchid=call.get("jobId"),
backend=None,
stdout=call.get("stdout"),
stderr=call.get("stderr"),
start=DateUtil.parse_iso(start) if start is not None else DateUtil.now(),
finish=DateUtil.parse_iso(finish) if finish is not None else None,
jobs=subjobs or None,
cached=call.get("callCaching").get("hit")
if "callCaching" in call
else False,
shard=shard,
attempt=attempt,
analysis=None,
memory=None,
cpu=None,
error=CromwellMetadata.unwrap_caused_by(call.get("failures")),
returncode=call.get("returnCode"),
workdir=call.get("callRoot"),
script=("file:/" + os.path.join(callroot, "execution/script"))
if callroot
else None,
)
def __init__(
self,
id_: str,
submission_id: str,
run_id: str,
parent: Optional[str],
name: str,
batchid: Optional[str] = None,
shard: Optional[int] = None,
attempt: Optional[int] = None,
container: Optional[str] = None,
status: TaskStatus = None,
start: Union[str, datetime] = None,
finish: Optional[Union[str, datetime]] = None,
backend: Optional[str] = None,
cached: bool = None,
stdout: Optional[str] = None,
stderr: Optional[str] = None,
script: Optional[str] = None,
error: Optional[str] = None,
returncode: Optional[str] = None,
memory: Optional[str] = None,
cpu: Optional[str] = None,
analysis: Optional[str] = None,
# Optional
jobs: Optional[list] = None,
lastupdated: Union[str, datetime] = None,
workdir: Optional[str] = None,
):
self.id_ = id_
self.submission_id = submission_id
self.run_id = run_id
self.parent = parent
self.status = status if isinstance(status, TaskStatus) else TaskStatus(status)
self.name = name
self.batchid = batchid
self.shard = None
if shard is not None:
if isinstance(shard, str) and shard.isdigit:
shard = int(shard)
if shard >= 0:
self.shard = shard
self.attempt = None
if attempt is not None:
if isinstance(attempt, str) and attempt.isdigit():
attempt = int(attempt)
if attempt > 1:
self.attempt = attempt
self.container = container
self.backend = backend
self.cached = cached
self.stderr = stderr
self.stdout = stdout
self.error = error
self.returncode = returncode
self.workdir = workdir
self.lastupdated = lastupdated or DateUtil.now()
if isinstance(lastupdated, str):
self.lastupdated = DateUtil.parse_iso(lastupdated)
self.start = start
self.finish = finish
if start and isinstance(start, str):
self.start = DateUtil.parse_iso(start)
if finish and isinstance(finish, str):
self.finish = DateUtil.parse_iso(finish)
self.script = script
self.memory = memory
self.cpu = cpu
self.analysis = analysis
self.jobs: Optional[List[RunJobModel]] = jobs or None
self.events = None
def __init__(self, jid: str, status: TaskStatus, timestamp: str):
self.jid = jid
self.status = status
self.timestamp = timestamp
if timestamp and isinstance(timestamp, str):
self.timestamp = DateUtil.parse_iso(timestamp)