def get_wfs_with_parameter(parameter, wf_class='Workflow'):
"""
Find workflows of a given class, with a given parameter (which must be a
node)
:param parameter: an AiiDA node
:param wf_class: the name of the workflow class
:return: an AiiDA query set with all workflows that have this parameter
"""
from aiida.common.datastructures import wf_data_types
from aiida.orm.workflow import Workflow
try:
from aiida.backends.djsite.db import models
except ImportError:
from aiida.djsite.db import models
# Find attributes with this name
qdata = models.DbWorkflowData.objects.filter(aiida_obj=parameter,
data_type=wf_data_types.PARAMETER)
# Find workflows with those attributes
if wf_class == 'Workflow':
qwf = Workflow.query(data__in=qdata)
else:
qwf = Workflow.query(module_class=wf_class,data__in=qdata)
#q2 = wf_class.query(data__in=q1)
# return a Django QuerySet with the resulting class instances
return qwf.distinct().order_by('ctime')
def wipe_all_scratch(w, results_to_save):
"""
Wipe out all the scratch on the remote cluster used by a workflow and all
its subworkflows (found recursively)
:param results_to_save: a list of Calculation objects that will be skipped
:w: the workflow instance to clean
"""
from aiida.orm.workflow import Workflow
from aiida.orm.calculation.job import JobCalculation
if not isinstance(w, Workflow):
raise TypeError("Parameter w should be a workflow")
try:
if not all( [ isinstance(_,JobCalculation) for _ in results_to_save ] ):
raise TypeError("Parameter results_to_save should be a list of calculations")
except TypeError:
raise TypeError("Parameter results_to_save should be a list of calculations")
steps = w.dbworkflowinstance.steps.all()
this_calcs = JobCalculation.query(workflow_step__in=steps)
this_wfs = Workflow.query(parent_workflow_step__in=steps)
for c in this_calcs:
if c.pk not in [_.pk for _ in results_to_save]:
try:
c.out.remote_folder._clean()
except AttributeError:
# remote folder does not exist (probably submission of calc. failed)
pass
except OSError:
# work directory was already removed
pass
for this_wf in this_wfs:
wipe_all_scratch(this_wf, results_to_save)
def execute_steps():
"""
This method loops on the RUNNING workflows and handled the execution of the steps until
each workflow reaches an end (or gets stopped for errors).
In the loop for each RUNNING workflow the method loops also in each of its RUNNING steps,
testing if all the calculation and subworkflows attached to the step are FINISHED. In this
case the step is set as FINISHED and the workflow is advanced to the step's next method present
in the db with ``advance_workflow``, otherwise if any step's JobCalculation is found in NEW state
the method will submit. If none of the previous conditions apply the step is flagged as
ERROR and cannot proceed anymore, blocking the future execution of the step and, connected,
the workflow.
Finally, for each workflow the method tests if there are INITIALIZED steps to be launched,
and in case reloads the workflow and execute the specific those steps. In case or error
the step is flagged in ERROR state and the stack is reported in the workflow report.
"""
from aiida.backends.utils import get_automatic_user
from aiida.orm.workflow import Workflow
from aiida.common.datastructures import wf_states
from aiida.orm import JobCalculation
logger.info("Querying the worflow DB")
w_list = Workflow.query(user=get_automatic_user(), state=wf_states.RUNNING)
for w in w_list:
logger.info("Found active workflow: {0}".format(w.uuid))
# Launch INITIALIZED Workflows
#
running_steps = w.get_steps(state=wf_states.RUNNING)
for s in running_steps:
logger.info("[{0}] Found active step: {1}".format(w.uuid, s.name))
s_calcs_new = [c.uuid for c in s.get_calculations() if c._is_new()]
s_calcs_running = [
c.uuid for c in s.get_calculations() if c._is_running()
]
s_calcs_finished = [
c.uuid for c in s.get_calculations() if c.has_finished_ok()
]
s_calcs_failed = [
c.uuid for c in s.get_calculations() if c.has_failed()
]
s_calcs_num = len(s.get_calculations())
s_sub_wf_running = [
sw.uuid for sw in s.get_sub_workflows() if sw.is_running()
]
s_sub_wf_finished = [
sw.uuid for sw in s.get_sub_workflows()
if sw.has_finished_ok()
]
s_sub_wf_failed = [
sw.uuid for sw in s.get_sub_workflows() if sw.has_failed()
]
s_sub_wf_num = len(s.get_sub_workflows())
if s_calcs_num == (len(s_calcs_finished) + len(s_calcs_failed)
) and s_sub_wf_num == (len(s_sub_wf_finished) +
len(s_sub_wf_failed)):
logger.info("[{0}] Step: {1} ready to move".format(
w.uuid, s.name))
s.set_state(wf_states.FINISHED)
advance_workflow(w, s)
elif len(s_calcs_new) > 0:
for uuid in s_calcs_new:
obj_calc = JobCalculation.get_subclass_from_uuid(uuid=uuid)
try:
obj_calc.submit()
logger.info(
"[{0}] Step: {1} launched calculation {2}".format(
w.uuid, s.name, uuid))
except:
logger.error(
"[{0}] Step: {1} cannot launch calculation {2}".
format(w.uuid, s.name, uuid))
## DO NOT STOP ANYMORE IF A CALCULATION FAILS
# elif s_calcs_failed:
#s.set_state(wf_states.ERROR)
initialized_steps = w.get_steps(state=wf_states.INITIALIZED)
for s in initialized_steps:
import sys
try:
w_class = Workflow.get_subclass_from_uuid(w.uuid)
getattr(w, s.name)()
return True
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
w.append_to_report(
"ERROR ! This workflow got an error in the {0} method, we report down the stack trace"
.format(s.name))
w.append_to_report("full traceback: {0}".format(
exc_traceback.format_exc()))
s.set_state(wf_states.ERROR)
w.set_state(wf_states.ERROR)
for w in w_list:
if w.get_steps(state=wf_states.ERROR):
w.set_state(wf_states.ERROR)