def _get_nodes_state(adageobj) -> Generator[Dict, None, None]:
dag = adageobj.dag
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
is_pure_publishing = nodeobj.task.metadata["wflow_hints"].get(
"is_purepub", False)
if is_pure_publishing:
continue
if nodeobj.state == nodestate.RUNNING:
state = "running"
job_id = nodeobj.resultproxy.jobproxy["job_id"]
elif dagstate.node_status(nodeobj):
state = "finished"
job_id = nodeobj.resultproxy.jobproxy["job_id"]
elif dagstate.node_ran_and_failed(nodeobj):
state = "failed"
job_id = nodeobj.resultproxy.jobproxy["job_id"]
elif dagstate.upstream_failure(dag, nodeobj):
state = "planned"
job_id = None
else:
state = "total"
job_id = None
node_state = {
"state": state,
"job_id": job_id,
}
yield node_state
def analyze_progress(adageobj):
"""Analyze the workflow progress."""
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
nodestates = []
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
is_pure_publishing = nodeobj.task.metadata['wflow_hints'].get(
'is_purepub', False)
if is_pure_publishing:
continue
if nodeobj.state == nodestate.RUNNING:
nodestates.append({
'state': 'running',
'job_id': nodeobj.resultproxy.jobproxy['job_id']
})
elif dagstate.node_status(nodeobj):
nodestates.append({
'state': 'succeeded',
'job_id': nodeobj.resultproxy.jobproxy['job_id']
})
elif dagstate.node_ran_and_failed(nodeobj):
nodestates.append({
'state': 'failed',
'job_id': nodeobj.resultproxy.jobproxy['job_id']
})
elif dagstate.upstream_failure(dag, nodeobj):
nodestates.append({'state': 'unsubmittable', 'job_id': None})
else:
nodestates.append({'state': 'scheduled', 'job_id': None})
return nodestates
def analyze_progress(adageobj):
"""Analyze the workflow progress."""
dag = adageobj.dag
nodestates = []
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
is_pure_publishing = nodeobj.task.metadata["wflow_hints"].get(
"is_purepub", False
)
if is_pure_publishing:
continue
if nodeobj.state == nodestate.RUNNING:
nodestates.append(
{"state": "running", "job_id": nodeobj.resultproxy.jobproxy["job_id"]}
)
elif dagstate.node_status(nodeobj):
nodestates.append(
{"state": "finished", "job_id": nodeobj.resultproxy.jobproxy["job_id"]}
)
elif dagstate.node_ran_and_failed(nodeobj):
nodestates.append(
{"state": "failed", "job_id": nodeobj.resultproxy.jobproxy["job_id"]}
)
elif dagstate.upstream_failure(dag, nodeobj):
nodestates.append({"state": "unsubmittable", "job_id": None})
else:
nodestates.append({"state": "scheduled", "job_id": None})
return nodestates
def update(self, adageobj):
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
for node in dag.nodes():
nodeobj = dag.getNode(node)
if nodeobj.state == nodestate.RUNNING:
running += 1
if dagstate.node_status(nodeobj):
successful += 1
if dagstate.node_ran_and_failed(nodeobj):
failed += 1
reason = nodeobj.backend.fail_info(
nodeobj.resultproxy) if nodeobj.backend else 'unknown'
self.log.error("node: {} failed. reason: {}".format(
nodeobj, reason))
if dagstate.upstream_failure(dag, nodeobj):
unsubmittable += 1
self.log.info(
'unsubmittable: {unsubmittable} | submitted: {submitted} | successful: {successful} | failed: {failed} | total: {total} | open rules: {rules} | applied rules: {applied}'
.format(successful=successful,
failed=failed,
submitted=running,
unsubmittable=unsubmittable,
total=len(dag.nodes()),
rules=len(rules),
applied=len(applied)))
def analyze_progress(adageobj):
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
nodestates = []
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
if nodeobj.state == nodestate.RUNNING:
nodestates.append('running')
elif dagstate.node_status(nodeobj):
nodestates.append('success')
elif dagstate.node_ran_and_failed(nodeobj):
nodestates.append('failed')
elif dagstate.upstream_failure(dag, nodeobj):
nodestates.append('unsubmittable')
else:
nodestates.append('scheduled')
return nodestates
def update(self,adageobj):
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
for node in dag.nodes():
nodeobj = dag.getNode(node)
if nodeobj.state == nodestate.RUNNING:
running += 1
if dagstate.node_status(nodeobj):
successful+=1
if dagstate.node_ran_and_failed(nodeobj):
failed+=1
reason = nodeobj.backend.fail_info(nodeobj.resultproxy) if nodeobj.backend else 'unknown'
self.log.error("node: {} failed. reason: {}".format(nodeobj,reason))
if dagstate.upstream_failure(dag,nodeobj):
unsubmittable+=1
self.log.info('unsubmittable: {unsubmittable} | submitted: {submitted} | successful: {successful} | failed: {failed} | total: {total} | open rules: {rules} | applied rules: {applied}'.format(
successful = successful,
failed = failed,
submitted = running,
unsubmittable = unsubmittable,
total = len(dag.nodes()),
rules = len(rules),
applied = len(applied)))
