def reset_stage(
name,
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
verbosity,
):
handle_common_options(verbosity)
ys = handle_connection_options(
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
)
controller = ys.controller
offset, name = name.rsplit("/", 1)
rule = controller.adageobj.view(offset).getRule(name)
if not rule:
click.secho("state not found!", fg="red")
r2s, _, _ = utils.rule_steps_indices(controller.adageobj)
steps_of_rule = r2s[rule.identifier]
controller.reset_nodes(steps_of_rule)
def submit(nodeid, allof, offset,
metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local,
verbosity
):
handle_common_options(verbosity)
ys = handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local)
controller = ys.controller
if not (allof or nodeid):
click_print_submittable_nodes(controller)
return
if nodeid:
nodes_to_submit = [nodeid] if nodeid in controller.submittable_nodes() else []
if allof:
offset, name = allof.rsplit('/',1)
rule = controller.adageobj.view().getRule(name = name, offset = offset)
if not rule:
click.secho('stage not found!', fg = 'red')
return
all_submittable = controller.submittable_nodes()
_, s2r, _ = utils.rule_steps_indices(controller.adageobj)
nodes_to_submit = [x for x in all_submittable if s2r[x] == rule.identifier]
if not nodes_to_submit:
click.secho('No nodes to submit (perhaps already submitted?)', fg = 'yellow')
return
controller.submit_nodes(nodes_to_submit)
click.secho('submitted: {}'.format(nodes_to_submit), fg = 'green')
def click_print_applied_stages(controller):
click.secho('Applied Stages: ', fg='blue')
r2s, s2r, r2sub = utils.rule_steps_indices(controller.adageobj)
step_status = {s: str(controller.adageobj.dag.getNode(s).state) for s in s2r.keys()}
# print(step_status)
for x in sorted(controller.adageobj.applied_rules,key = lambda r: '{}/{}'.format(r.offset,r.rule.name)):
click_print_rule(x, r2s, r2sub, step_status)
def click_print_submittable_nodes(controller):
click.secho('Submittable Nodes: ', fg='blue')
_, s2r,_ = utils.rule_steps_indices(controller.adageobj)
for x in controller.adageobj.dag.nodes():
node = controller.adageobj.dag.getNode(x)
rule = controller.adageobj.view().getRule(identifier = s2r[node.identifier])
if node.identifier in controller.submittable_nodes():
click.secho('node: {} ({}) part of stage {}'.format(node.name, node.identifier, '/'.join([rule.offset,rule.rule.name])))
def submit(
nodeid,
allof,
offset,
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
verbosity,
):
handle_common_options(verbosity)
ys = handle_connection_options(
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
)
controller = ys.controller
if not (allof or nodeid):
click_print_submittable_nodes(controller)
return
if nodeid:
nodes_to_submit = [
nodeid
] if nodeid in controller.submittable_nodes() else []
if allof:
offset, name = allof.rsplit("/", 1)
rule = controller.adageobj.view().getRule(name=name, offset=offset)
if not rule:
click.secho("stage not found!", fg="red")
return
all_submittable = controller.submittable_nodes()
_, s2r, _ = utils.rule_steps_indices(controller.adageobj)
nodes_to_submit = [
x for x in all_submittable if s2r[x] == rule.identifier
]
if not nodes_to_submit:
click.secho("No nodes to submit (perhaps already submitted?)",
fg="yellow")
return
controller.submit_nodes(nodes_to_submit)
click.secho("submitted: {}".format(nodes_to_submit), fg="green")
def click_print_submittable_nodes(controller):
click.secho("Submittable Nodes: ", fg="blue")
_, s2r, _ = utils.rule_steps_indices(controller.adageobj)
for x in controller.adageobj.dag.nodes():
node = controller.adageobj.dag.getNode(x)
rule = controller.adageobj.view().getRule(
identifier=s2r[node.identifier])
if node.identifier in controller.submittable_nodes():
click.secho("node: {} ({}) part of stage {}".format(
node.name, node.identifier,
"/".join([rule.offset, rule.rule.name])))
def click_print_applied_stages(controller):
click.secho('Applied Stages: ', fg='blue')
r2s, s2r, r2sub = utils.rule_steps_indices(controller.adageobj)
step_status = {
s: str(controller.adageobj.dag.getNode(s).state)
for s in s2r.keys()
}
# print(step_status)
for x in sorted(controller.adageobj.applied_rules,
key=lambda r: '{}/{}'.format(r.offset, r.rule.name)):
click_print_rule(x, r2s, r2sub, step_status)
def reset_stage(name,
metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local,
verbosity
):
handle_common_options(verbosity)
ys = handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local)
controller = ys.controller
offset, name = name.rsplit('/',1)
rule = controller.adageobj.view(offset).getRule(name)
if not rule:
click.secho('state not found!', fg = 'red')
r2s, _, _ = utils.rule_steps_indices(controller.adageobj)
steps_of_rule = r2s[rule.identifier]
controller.reset_nodes(steps_of_rule)
def apply_stage(name, metadir, accept_metadir, controller, ctrlopt, modelsetup,
modelopt, backend, local, verbosity, submit):
handle_common_options(verbosity)
ys = handle_connection_options(metadir, accept_metadir, controller,
ctrlopt, modelsetup, modelopt, backend,
local)
controller = ys.controller
if not name:
click_print_applicable_stages(controller)
return
for n in name:
offset, scopedname = n.rsplit('/', 1)
rule = controller.adageobj.view(offset).getRule(scopedname)
if not rule:
click.secho(
'No such stage {}, pick one of the applicable below:'.format(
n),
fg='red')
click_print_applicable_stages(controller)
return
if rule.identifier in [
r.identifier for r in controller.adageobj.applied_rules
]:
click.secho('Stage {} was already applied.'.format(n), fg='yellow')
continue
if rule.identifier not in controller.applicable_rules():
click.secho('Rule {} not yet applicable'.format(n), fg='red')
continue
controller.apply_rules([rule.identifier])
if submit:
_, s2r, _ = utils.rule_steps_indices(controller.adageobj)
nodes_to_submit = [
x for x in controller.submittable_nodes()
if s2r[x] == rule.identifier
]
controller.submit_nodes(nodes_to_submit)
click.secho('Stage {} applied'.format(n), fg='green')
def apply_stage(name,
metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local,
verbosity,
submit
):
handle_common_options(verbosity)
ys = handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local)
controller = ys.controller
if not name:
click_print_applicable_stages(controller)
return
for n in name:
offset, scopedname = n.rsplit('/',1)
rule = controller.adageobj.view(offset).getRule(scopedname)
if not rule:
click.secho('No such stage {}, pick one of the applicable below:'.format(n), fg='red')
click_print_applicable_stages(controller)
return
if rule.identifier in [r.identifier for r in controller.adageobj.applied_rules]:
click.secho('Stage {} was already applied.'.format(n), fg = 'yellow')
continue
if rule.identifier not in controller.applicable_rules():
click.secho('Rule {} not yet applicable'.format(n), fg = 'red')
continue
controller.apply_rules([rule.identifier])
if submit:
_, s2r, _ = utils.rule_steps_indices(controller.adageobj)
nodes_to_submit = [x for x in controller.submittable_nodes() if s2r[x] == rule.identifier]
controller.submit_nodes(nodes_to_submit)
click.secho('Stage {} applied'.format(n), fg = 'green')
def undo_rule(workflow, ruleid):
r2s, s2r, r2subscopes = utils.rule_steps_indices(workflow)
if not ruleid in [r.identifier for r in workflow.applied_rules]:
ruleobj = workflow.view().getRule(identifier = ruleid)
log.debug('rule %s/%s not in list of applied rules. possibly already undone during recursion.',ruleobj.offset,ruleobj.rule.name)
return
downstream_nodes_rules = list(set([s2r[s] for s in collective_downstream(workflow, r2s[ruleid])]))
r = workflow.view().getRule(identifier = ruleid)
log.debug('undoing %s/%s. downstream_nodes_rules: %s', r.offset, r.rule.name, len(downstream_nodes_rules))
assert ruleid not in downstream_nodes_rules
#TODO: not an issue now, but if stages schedule linked nodes we might
#we might see downstream steps that are part of the same rule
if not downstream_nodes_rules:
idx = workflow.applied_rules.index(r)
stepids = r2s[ruleid]
steps = [workflow.dag.getNode(nid) for nid in stepids]
# reset states of the step
reset_steps(workflow,stepids)
#
# delete the steps
for n in steps:
log.debug('remove DAG node %s/%s:%s',n.task.metadata['wflow_offset'],n.task.metadata['wflow_stage'],n.task.metadata['wflow_stage_node_idx'])
workflow.dag.removeNode(n)
for subscope in r2subscopes[ruleid]:
subrules = utils.stages_in_scope(workflow,subscope)
log.debug('remove subscope %s with %s subrules', subscope, len(subrules))
for subruleidx,subrule in enumerate(subrules):
subruleobj = workflow.view().getRule(identifier = subrule)
log.debug('undo sub DAG rule %s/%s', subruleobj.offset, subruleobj.rule.name)
undo_rule(workflow,subrule)
log.debug('removing %s subrules', len(subrules))
for subruleidx,subrule in enumerate(subrules):
subruleobj = workflow.view().getRule(identifier = subrule)
log.debug('remove sub DAG rule %s/%s', subruleobj.offset, subruleobj.rule.name)
remove_rule(workflow,subrule)
#remove rule from applied list
r = workflow.applied_rules.pop(idx)
#reset all index data for stage
workflow.view(r.offset).steps.pop(r.rule.name)
#remove steps and stage from bookkeeping
for s in stepids:
workflow.view(r.offset).bookkeeper['_meta']['steps'].remove(s)
workflow.view(r.offset).bookkeeper['_meta']['stages'].remove(ruleid)
#re-append the rule
log.debug('re-appened {}/{}'.format(r.offset,r.rule.name))
# workflow.steps[stage][-1]['_offset'] = offset
newid = workflow.view(r.offset).addRule(r.rule, identifier = r.identifier)
assert newid == r.identifier
log.debug('undo any rules that would not be applicable now')
for rule in workflow.applied_rules:
if not rule.applicable(workflow):
ruleobj = workflow.view().getRule(identifier = rule.identifier)
log.debug('rule would not be appilcable in current state so undo >> %s/%s', ruleobj.offset, ruleobj.rule.name)
undo_rule(workflow,rule.identifier)
return
log.debug('to undo a rule, we need to undo rules responsible for any of the downstream nodes of the rules of this stage')
while True:
if not downstream_nodes_rules:
break
for r in downstream_nodes_rules:
log.debug('undoing a downstream rule')
undo_rule(workflow,r)
log.debug('undone a downstream rule')
r2s, s2r, _ = utils.rule_steps_indices(workflow)
downstream_nodes_rules = list(set([s2r[s] for s in collective_downstream(workflow, r2s[ruleid])]))
log.debug('re-asses if there are still any downstream rules: {}'.format(len(downstream_nodes_rules)))
log.debug('undo rule (now without any downstream rules)')
undo_rule(workflow,ruleid)
def undo_rule(workflow, ruleid):
r2s, s2r, r2subscopes = utils.rule_steps_indices(workflow)
if not ruleid in [r.identifier for r in workflow.applied_rules]:
ruleobj = workflow.view().getRule(identifier=ruleid)
log.debug(
'rule %s/%s not in list of applied rules. possibly already undone during recursion.',
ruleobj.offset, ruleobj.rule.name)
return
downstream_nodes_rules = list(
set([s2r[s] for s in collective_downstream(workflow, r2s[ruleid])]))
r = workflow.view().getRule(identifier=ruleid)
log.debug('undoing %s/%s. downstream_nodes_rules: %s', r.offset,
r.rule.name, len(downstream_nodes_rules))
assert ruleid not in downstream_nodes_rules
#TODO: not an issue now, but if stages schedule linked nodes we might
#we might see downstream steps that are part of the same rule
if not downstream_nodes_rules:
idx = workflow.applied_rules.index(r)
stepids = r2s[ruleid]
steps = [workflow.dag.getNode(nid) for nid in stepids]
# reset states of the step
reset_steps(workflow, stepids)
#
# delete the steps
for n in steps:
log.debug('remove DAG node %s/%s:%s',
n.task.metadata['wflow_offset'],
n.task.metadata['wflow_stage'],
n.task.metadata['wflow_stage_node_idx'])
workflow.dag.removeNode(n)
for subscope in r2subscopes[ruleid]:
subrules = utils.stages_in_scope(workflow, subscope)
log.debug('remove subscope %s with %s subrules', subscope,
len(subrules))
for subruleidx, subrule in enumerate(subrules):
subruleobj = workflow.view().getRule(identifier=subrule)
log.debug('undo sub DAG rule %s/%s', subruleobj.offset,
subruleobj.rule.name)
undo_rule(workflow, subrule)
log.debug('removing %s subrules', len(subrules))
for subruleidx, subrule in enumerate(subrules):
subruleobj = workflow.view().getRule(identifier=subrule)
log.debug('remove sub DAG rule %s/%s', subruleobj.offset,
subruleobj.rule.name)
remove_rule(workflow, subrule)
#remove rule from applied list
r = workflow.applied_rules.pop(idx)
#reset all index data for stage
workflow.view(r.offset).steps.pop(r.rule.name)
#remove steps and stage from bookkeeping
for s in stepids:
workflow.view(r.offset).bookkeeper['_meta']['steps'].remove(s)
workflow.view(r.offset).bookkeeper['_meta']['stages'].remove(ruleid)
#re-append the rule
log.debug('re-appened {}/{}'.format(r.offset, r.rule.name))
# workflow.steps[stage][-1]['_offset'] = offset
newid = workflow.view(r.offset).addRule(r.rule,
identifier=r.identifier)
assert newid == r.identifier
log.debug('undo any rules that would not be applicable now')
for rule in workflow.applied_rules:
if not rule.applicable(workflow):
ruleobj = workflow.view().getRule(identifier=rule.identifier)
log.debug(
'rule would not be appilcable in current state so undo >> %s/%s',
ruleobj.offset, ruleobj.rule.name)
undo_rule(workflow, rule.identifier)
return
log.debug(
'to undo a rule, we need to undo rules responsible for any of the downstream nodes of the rules of this stage'
)
while True:
if not downstream_nodes_rules:
break
for r in downstream_nodes_rules:
log.debug('undoing a downstream rule')
undo_rule(workflow, r)
log.debug('undone a downstream rule')
r2s, s2r, _ = utils.rule_steps_indices(workflow)
downstream_nodes_rules = list(
set([s2r[s]
for s in collective_downstream(workflow, r2s[ruleid])]))
log.debug(
're-asses if there are still any downstream rules: {}'.format(
len(downstream_nodes_rules)))
log.debug('undo rule (now without any downstream rules)')
undo_rule(workflow, ruleid)