def _get_nodes_state(adageobj) -> Generator[Dict, None, None]:
dag = adageobj.dag
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
is_pure_publishing = nodeobj.task.metadata["wflow_hints"].get(
"is_purepub", False)
if is_pure_publishing:
continue
if nodeobj.state == nodestate.RUNNING:
state = "running"
job_id = nodeobj.resultproxy.jobproxy["job_id"]
elif dagstate.node_status(nodeobj):
state = "finished"
job_id = nodeobj.resultproxy.jobproxy["job_id"]
elif dagstate.node_ran_and_failed(nodeobj):
state = "failed"
job_id = nodeobj.resultproxy.jobproxy["job_id"]
elif dagstate.upstream_failure(dag, nodeobj):
state = "planned"
job_id = None
else:
state = "total"
job_id = None
node_state = {
"state": state,
"job_id": job_id,
}
yield node_state
def analyze_progress(adageobj):
"""Analyze the workflow progress."""
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
nodestates = []
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
is_pure_publishing = nodeobj.task.metadata['wflow_hints'].get(
'is_purepub', False)
if is_pure_publishing:
continue
if nodeobj.state == nodestate.RUNNING:
nodestates.append({
'state': 'running',
'job_id': nodeobj.resultproxy.jobproxy['job_id']
})
elif dagstate.node_status(nodeobj):
nodestates.append({
'state': 'succeeded',
'job_id': nodeobj.resultproxy.jobproxy['job_id']
})
elif dagstate.node_ran_and_failed(nodeobj):
nodestates.append({
'state': 'failed',
'job_id': nodeobj.resultproxy.jobproxy['job_id']
})
elif dagstate.upstream_failure(dag, nodeobj):
nodestates.append({'state': 'unsubmittable', 'job_id': None})
else:
nodestates.append({'state': 'scheduled', 'job_id': None})
return nodestates
def analyze_progress(adageobj):
"""Analyze the workflow progress."""
dag = adageobj.dag
nodestates = []
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
is_pure_publishing = nodeobj.task.metadata["wflow_hints"].get(
"is_purepub", False
)
if is_pure_publishing:
continue
if nodeobj.state == nodestate.RUNNING:
nodestates.append(
{"state": "running", "job_id": nodeobj.resultproxy.jobproxy["job_id"]}
)
elif dagstate.node_status(nodeobj):
nodestates.append(
{"state": "finished", "job_id": nodeobj.resultproxy.jobproxy["job_id"]}
)
elif dagstate.node_ran_and_failed(nodeobj):
nodestates.append(
{"state": "failed", "job_id": nodeobj.resultproxy.jobproxy["job_id"]}
)
elif dagstate.upstream_failure(dag, nodeobj):
nodestates.append({"state": "unsubmittable", "job_id": None})
else:
nodestates.append({"state": "scheduled", "job_id": None})
return nodestates
def nodes_left_or_rule_applicable(adageobj):
'''
:param adageobj: the adage workflow object
Main worklflow status function. It checks whether the overall workflow is finished by checking whether
any eligible node (i.e. one without upstream failure) has not yet finished, or if any rules are still
applicable.
:param adageobj: the adage workflow object
:return:
- True any eligible node are waiting to be run, or still running or a rule is applicable
- False in all other cases.
'''
dag,rules = adageobj.dag, adageobj.rules
nodes_we_could_run = [node for node in dag.nodes() if not dagstate.upstream_failure(dag,dag.getNode(node))]
nodes_running_or_defined = [x for x in nodes_we_could_run if dagstate.node_defined_or_running(dag.getNode(x))]
if any(rule.applicable(adageobj) for rule in rules):
return True
log.debug('nodes we could run: %s',nodes_we_could_run)
if nodes_running_or_defined:
log.debug('%s nodes that could be run or are running are left.',len(nodes_running_or_defined))
log.debug('nodes are: %s',[dag.node[n] for n in nodes_running_or_defined])
return True
if any(rule.applicable(adageobj) for rule in rules):
return True
log.info('no nodes can be run anymore and no rules are applicable')
return False
def update(self, adageobj):
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
for node in dag.nodes():
nodeobj = dag.getNode(node)
if nodeobj.state == nodestate.RUNNING:
running += 1
if dagstate.node_status(nodeobj):
successful += 1
if dagstate.node_ran_and_failed(nodeobj):
failed += 1
reason = nodeobj.backend.fail_info(
nodeobj.resultproxy) if nodeobj.backend else 'unknown'
self.log.error("node: {} failed. reason: {}".format(
nodeobj, reason))
if dagstate.upstream_failure(dag, nodeobj):
unsubmittable += 1
self.log.info(
'unsubmittable: {unsubmittable} | submitted: {submitted} | successful: {successful} | failed: {failed} | total: {total} | open rules: {rules} | applied rules: {applied}'
.format(successful=successful,
failed=failed,
submitted=running,
unsubmittable=unsubmittable,
total=len(dag.nodes()),
rules=len(rules),
applied=len(applied)))
def nodes_left_or_rule_applicable(adageobj):
'''
:param adageobj: the adage workflow object
Main worklflow status function. It checks whether the overall workflow is finished by checking whether
any eligible node (i.e. one without upstream failure) has not yet finished, or if any rules are still
applicable.
:param adageobj: the adage workflow object
:return:
- True any eligible node are waiting to be run, or still running or a rule is applicable
- False in all other cases.
'''
dag,rules = adageobj.dag, adageobj.rules
nodes_we_could_run = [node for node in dag.nodes() if not dagstate.upstream_failure(dag,dag.getNode(node))]
nodes_running_or_defined = [x for x in nodes_we_could_run if dagstate.node_defined_or_running(dag.getNode(x))]
if any(rule.applicable(adageobj) for rule in rules):
return True
log.debug('nodes we could run: %s',nodes_we_could_run)
if nodes_running_or_defined:
log.debug('%s nodes that could be run or are running are left.',len(nodes_running_or_defined))
log.debug('nodes are: %s',[dag.node[n] for n in nodes_running_or_defined])
return True
if any(rule.applicable(adageobj) for rule in rules):
return True
log.info('no nodes can be run anymore and no rules are applicable')
return False
def analyze_progress(adageobj):
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
nodestates = []
for node in nx.topological_sort(dag):
nodeobj = dag.getNode(node)
if nodeobj.state == nodestate.RUNNING:
nodestates.append('running')
elif dagstate.node_status(nodeobj):
nodestates.append('success')
elif dagstate.node_ran_and_failed(nodeobj):
nodestates.append('failed')
elif dagstate.upstream_failure(dag, nodeobj):
nodestates.append('unsubmittable')
else:
nodestates.append('scheduled')
return nodestates
def update(self,adageobj):
dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules
successful, failed, running, unsubmittable = 0, 0, 0, 0
for node in dag.nodes():
nodeobj = dag.getNode(node)
if nodeobj.state == nodestate.RUNNING:
running += 1
if dagstate.node_status(nodeobj):
successful+=1
if dagstate.node_ran_and_failed(nodeobj):
failed+=1
reason = nodeobj.backend.fail_info(nodeobj.resultproxy) if nodeobj.backend else 'unknown'
self.log.error("node: {} failed. reason: {}".format(nodeobj,reason))
if dagstate.upstream_failure(dag,nodeobj):
unsubmittable+=1
self.log.info('unsubmittable: {unsubmittable} | submitted: {submitted} | successful: {successful} | failed: {failed} | total: {total} | open rules: {rules} | applied rules: {applied}'.format(
successful = successful,
failed = failed,
submitted = running,
unsubmittable = unsubmittable,
total = len(dag.nodes()),
rules = len(rules),
applied = len(applied)))
