def _get_nodes_state(adageobj) -> Generator[Dict, None, None]: dag = adageobj.dag for node in nx.topological_sort(dag): nodeobj = dag.getNode(node) is_pure_publishing = nodeobj.task.metadata["wflow_hints"].get( "is_purepub", False) if is_pure_publishing: continue if nodeobj.state == nodestate.RUNNING: state = "running" job_id = nodeobj.resultproxy.jobproxy["job_id"] elif dagstate.node_status(nodeobj): state = "finished" job_id = nodeobj.resultproxy.jobproxy["job_id"] elif dagstate.node_ran_and_failed(nodeobj): state = "failed" job_id = nodeobj.resultproxy.jobproxy["job_id"] elif dagstate.upstream_failure(dag, nodeobj): state = "planned" job_id = None else: state = "total" job_id = None node_state = { "state": state, "job_id": job_id, } yield node_state
def analyze_progress(adageobj): """Analyze the workflow progress.""" dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules successful, failed, running, unsubmittable = 0, 0, 0, 0 nodestates = [] for node in nx.topological_sort(dag): nodeobj = dag.getNode(node) is_pure_publishing = nodeobj.task.metadata['wflow_hints'].get( 'is_purepub', False) if is_pure_publishing: continue if nodeobj.state == nodestate.RUNNING: nodestates.append({ 'state': 'running', 'job_id': nodeobj.resultproxy.jobproxy['job_id'] }) elif dagstate.node_status(nodeobj): nodestates.append({ 'state': 'succeeded', 'job_id': nodeobj.resultproxy.jobproxy['job_id'] }) elif dagstate.node_ran_and_failed(nodeobj): nodestates.append({ 'state': 'failed', 'job_id': nodeobj.resultproxy.jobproxy['job_id'] }) elif dagstate.upstream_failure(dag, nodeobj): nodestates.append({'state': 'unsubmittable', 'job_id': None}) else: nodestates.append({'state': 'scheduled', 'job_id': None}) return nodestates
def analyze_progress(adageobj): """Analyze the workflow progress.""" dag = adageobj.dag nodestates = [] for node in nx.topological_sort(dag): nodeobj = dag.getNode(node) is_pure_publishing = nodeobj.task.metadata["wflow_hints"].get( "is_purepub", False ) if is_pure_publishing: continue if nodeobj.state == nodestate.RUNNING: nodestates.append( {"state": "running", "job_id": nodeobj.resultproxy.jobproxy["job_id"]} ) elif dagstate.node_status(nodeobj): nodestates.append( {"state": "finished", "job_id": nodeobj.resultproxy.jobproxy["job_id"]} ) elif dagstate.node_ran_and_failed(nodeobj): nodestates.append( {"state": "failed", "job_id": nodeobj.resultproxy.jobproxy["job_id"]} ) elif dagstate.upstream_failure(dag, nodeobj): nodestates.append({"state": "unsubmittable", "job_id": None}) else: nodestates.append({"state": "scheduled", "job_id": None}) return nodestates
def nodes_left_or_rule_applicable(adageobj): ''' :param adageobj: the adage workflow object Main worklflow status function. It checks whether the overall workflow is finished by checking whether any eligible node (i.e. one without upstream failure) has not yet finished, or if any rules are still applicable. :param adageobj: the adage workflow object :return: - True any eligible node are waiting to be run, or still running or a rule is applicable - False in all other cases. ''' dag,rules = adageobj.dag, adageobj.rules nodes_we_could_run = [node for node in dag.nodes() if not dagstate.upstream_failure(dag,dag.getNode(node))] nodes_running_or_defined = [x for x in nodes_we_could_run if dagstate.node_defined_or_running(dag.getNode(x))] if any(rule.applicable(adageobj) for rule in rules): return True log.debug('nodes we could run: %s',nodes_we_could_run) if nodes_running_or_defined: log.debug('%s nodes that could be run or are running are left.',len(nodes_running_or_defined)) log.debug('nodes are: %s',[dag.node[n] for n in nodes_running_or_defined]) return True if any(rule.applicable(adageobj) for rule in rules): return True log.info('no nodes can be run anymore and no rules are applicable') return False
def update(self, adageobj): dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules successful, failed, running, unsubmittable = 0, 0, 0, 0 for node in dag.nodes(): nodeobj = dag.getNode(node) if nodeobj.state == nodestate.RUNNING: running += 1 if dagstate.node_status(nodeobj): successful += 1 if dagstate.node_ran_and_failed(nodeobj): failed += 1 reason = nodeobj.backend.fail_info( nodeobj.resultproxy) if nodeobj.backend else 'unknown' self.log.error("node: {} failed. reason: {}".format( nodeobj, reason)) if dagstate.upstream_failure(dag, nodeobj): unsubmittable += 1 self.log.info( 'unsubmittable: {unsubmittable} | submitted: {submitted} | successful: {successful} | failed: {failed} | total: {total} | open rules: {rules} | applied rules: {applied}' .format(successful=successful, failed=failed, submitted=running, unsubmittable=unsubmittable, total=len(dag.nodes()), rules=len(rules), applied=len(applied)))
def analyze_progress(adageobj): dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules successful, failed, running, unsubmittable = 0, 0, 0, 0 nodestates = [] for node in nx.topological_sort(dag): nodeobj = dag.getNode(node) if nodeobj.state == nodestate.RUNNING: nodestates.append('running') elif dagstate.node_status(nodeobj): nodestates.append('success') elif dagstate.node_ran_and_failed(nodeobj): nodestates.append('failed') elif dagstate.upstream_failure(dag, nodeobj): nodestates.append('unsubmittable') else: nodestates.append('scheduled') return nodestates
def update(self,adageobj): dag, rules, applied = adageobj.dag, adageobj.rules, adageobj.applied_rules successful, failed, running, unsubmittable = 0, 0, 0, 0 for node in dag.nodes(): nodeobj = dag.getNode(node) if nodeobj.state == nodestate.RUNNING: running += 1 if dagstate.node_status(nodeobj): successful+=1 if dagstate.node_ran_and_failed(nodeobj): failed+=1 reason = nodeobj.backend.fail_info(nodeobj.resultproxy) if nodeobj.backend else 'unknown' self.log.error("node: {} failed. reason: {}".format(nodeobj,reason)) if dagstate.upstream_failure(dag,nodeobj): unsubmittable+=1 self.log.info('unsubmittable: {unsubmittable} | submitted: {submitted} | successful: {successful} | failed: {failed} | total: {total} | open rules: {rules} | applied rules: {applied}'.format( successful = successful, failed = failed, submitted = running, unsubmittable = unsubmittable, total = len(dag.nodes()), rules = len(rules), applied = len(applied)))