def _record_item(self,view,complete,alarm_name):
if view.get('Disable',False): return
if view.get('Dummy',False): return
my_completes = view.get_complete_dep()
self.__all_defined.add(view.path)
if my_completes is not FALSE_DEPENDENCY:
complete=complete | my_completes
self.__completes[view.path]=[view, complete]
if 'AlarmName' in view:
if alarm_name:
raise ValueError('{view.task_path_var}: nested alarms are not supported in crow.metascheduler.to_rocoto()')
else:
alarm_name=view.AlarmName
self.__alarms[view.path]=alarm_name
if view.is_task():
return
self.__families.add(SuitePath(view.path[1:-1]))
for key,child in view.items():
if key in [ 'up', 'this' ]: continue
if not isinstance(child,SuiteView):
continue
if child.path[1:] == ['final']:
if not child.is_task():
raise RocotoConfigError(
'The "final" task must be a Task, not a '
+type(child.viewed).__name__)
self.__final_task=child
else:
self._record_item(child,complete,alarm_name)
def _rocotoify_dep_impl(self, dep, defining_path):
if isinstance(dep, StateDependency):
dep_path = SuitePath([_ZERO_DT] + dep.view.path[1:])
if dep_path not in self.__all_defined:
#_logger.info(
#f'/{"/".join(defining_path[1:])}: '
#'has a dependency on undefined task '
#f'/{"/".join(dep_path[1:])}')
return TRUE_DEPENDENCY
if isinstance(dep,StateDependency) and not dep.view.is_task() and \
dep.state==RUNNING:
deplist = list()
for t in dep.view.walk_task_tree():
if t.is_task():
deplist.append(t.is_running())
if not deplist: return FALSE_DEPENDENCY # no tasks
return _dep_rel(dep.view.path[0], OrDependency(*deplist))
elif isinstance(dep, StateDependency) and dep.state == COMPLETED:
zero_path = SuitePath([_ZERO_DT] + dep.view.path[1:])
if dep.view.is_task():
completes = self._completes_for(dep.view)
return dep | _dep_rel(dep.view.path[0], completes)
elif SuitePath(
dep.view.path[1:]) in self.__families_with_completes:
deplist = TRUE_DEPENDENCY
for t in dep.view.walk_task_tree():
if t.is_task():
deplist = deplist & \
_dep_rel(dep.view.path[0],self._rocotoify_dep(
t.is_completed(),defining_path))
deplist = deplist | _dep_rel(dep.path[0], \
self._rocotoify_dep(self._completes_for(dep.view),
defining_path))
return deplist
elif isinstance(dep, NotDependency):
return NotDependency(self._rocotoify_dep(dep.depend,
defining_path))
elif isinstance(dep, OrDependency) or isinstance(dep, AndDependency):
cls = type(dep)
for i in range(len(dep.depends)):
dep.depends[i] = self._rocotoify_dep(dep.depends[i],
defining_path)
return dep
def _completes_for(self, item):
dep = FALSE_DEPENDENCY
for i in range(1, len(item.path)):
zero_path = SuitePath([_ZERO_DT] + item.path[1:i + 1])
# item_path=SuitePath(item.path[0:i+1])
if zero_path in self.__completes:
dep = dep | self.__completes[zero_path][1]
return dep
def make_task_xml(self, indent=1):
fd = StringIO()
self._record_item(self.suite, FALSE_DEPENDENCY, '')
# Find all families that have tasks with completes:
for path, view_condition in self.__completes.items():
(view, condition) = view_condition
for i in range(1, len(path)):
family_path = SuitePath(path[1:i])
self.__families_with_completes.add(family_path)
for path, alarm_name in self.__alarms.items():
for i in range(1, len(path)):
family_path = SuitePath(path[1:i])
self.__families_with_alarms.add(family_path)
self._convert_item(fd, max(0, indent - 1), self.suite, TRUE_DEPENDENCY,
FALSE_DEPENDENCY, timedelta.min, '')
self._handle_final_task(fd, indent)
result = fd.getvalue()
fd.close()
return result
def remove_undefined_tasks(self, tree):
typecheck('tree', tree, LogicalDependency)
# NOTE: Do not remove event dependencies for undefined tasks.
# They are critical to allow ecflow to use a task that waits
# for data and sets an event while rocoto uses a data event
# with no task.
if isinstance(tree, StateDependency):
# Node is not defined, so assume it is complete
dep_path = SuitePath([_ZERO_DT] + tree.view.path[1:])
if dep_path not in self.__all_defined:
tree = TRUE_DEPENDENCY
elif isinstance(tree, NotDependency):
tree = ~self.remove_undefined_tasks(tree.depend)
elif isinstance(tree, AndDependency) or isinstance(tree, OrDependency):
deplist = [self.remove_undefined_tasks(t) for t in tree]
tree = type(tree)(*deplist)
return tree
def _final_task_deps_no_alarms(self, item):
path = SuitePath(item.path[1:])
with_completes = self.__families_with_completes
if 'Disable' in item and item.Disable:
return TRUE_DEPENDENCY
if 'Dummy' in item and item.Dummy:
return TRUE_DEPENDENCY
if item.is_task():
dep = item.is_completed()
if item.path in self.__completes:
dep = dep | self.__completes[item.path][1]
return dep
# Initial completion dependency is the task or family
# completion unless this item is the Suite. Suites must be
# handled differently.
if path:
dep = item.is_completed() # Family SuiteView
else:
dep = FALSE_DEPENDENCY # Suite
if path and path not in with_completes:
# Families with no "complete" dependency in their entire
# tree have no further dependencies to identify. Their
# own completion is the entirety of the completion
# dependency.
return dep
subdep = TRUE_DEPENDENCY
for subitem in item.child_iter():
if not path and subitem.path[1:] == ['final']:
# Special case. Do not include final task's
# dependency in the final task's dependency.
continue
if not subitem.is_task() and not subitem.is_family():
continue
subdep = subdep & self._final_task_deps_no_alarms(subitem)
if dep is FALSE_DEPENDENCY:
dep = subdep
else:
dep = dep | subdep
return dep
def _handle_final_task(self, fd, indent):
# Find and validate the "final" task:
final = None
if 'final' in self.suite:
final = self.suite.final
if not final.is_task():
raise RocotoConfigError(
'For a workflow suite to be expressed in Rocoto, it '
'must have a "final" task')
for elem in ['Trigger', 'Complete', 'Time']:
if elem in final:
raise RocotoConfigError(
f'{elem}: In a Rocoto workflow, the "final" task '
'must have no dependencies.')
if final is None:
if self.__completes:
raise RocotoConfigError(
'If a workflow suite has any "complete" conditions, '
'then it must have a "final" task with no dependencies.')
else:
return # No final task, and no need to generate one
if len(self.__alarms_used) < 2:
# There are no alarms in use, so there is only one final task.
# Generate dependency for it:
fd.write(
f'\n{self.__spacing*indent}<!-- The final task dependencies are automatically generated to handle Complate and Trigger conditions. -->\n\n'
)
dep = self._final_task_deps_no_alarms(self.suite)
self._write_task_text(fd, ' final="true"', indent, final, dep,
timedelta.min, '')
return
fd.write(
f'\n{self.__spacing*indent}<!-- These final tasks are automatically generated to handle Complate and Trigger conditions, and alarms. -->\n\n'
)
# There are alarms, so things get... complicated.
manual_dependency = f'''<dependency>
{self.__spacing*indent}<and>
{self.__spacing*(indent+1)}<!-- All tasks must be complete or invalid for this cycle -->\n'''
alarms = set(self.__alarms_used)
alarms.add('')
# Reverse-sort the alarm names so the final tasks show up in
# the same order each time, with the task "final" at the end.
for alarm_name in reversed(sorted(alarms)):
#print(f'find final for {alarm_name}')
dep = self._final_task_deps_for_alarm(self.suite, alarm_name)
dep = simplify(dep)
task_name = f'final_for_{alarm_name}' if alarm_name else 'final_no_alarm'
new_task = copy(self.suite.final.viewed)
new_task['_is_final_'] = True
new_task['AlarmName'] = alarm_name
invalidate_cache(self.suite, recurse=True)
self.suite.viewed[task_name] = new_task
new_task_view = self.suite[task_name]
del new_task
self.__all_defined.add(
SuitePath([_ZERO_DT] + new_task_view.path[1:]))
assert (dep is not FALSE_DEPENDENCY)
self._write_task_text(fd, '', indent, new_task_view, dep,
timedelta.min, alarm_name)
manual_dependency += f'''{self.__spacing*(indent+1)}<or>
{self.__spacing*(indent+2)}<taskdep task="{task_name}"/>
{self.__spacing*(indent+2)}<not><taskvalid task="{task_name}"/></not>
{self.__spacing*(indent+1)}</or>\n'''
manual_dependency += f'{self.__spacing*indent}</and>\n</dependency>\n'
self._write_task_text(fd,
' final="true"',
indent,
final,
TRUE_DEPENDENCY,
timedelta.min,
'',
manual_dependency=manual_dependency)
def _final_task_deps_for_alarm(self, item, for_alarm, alarm_name=''):
# For tasks:
# item is not in alarm
# OR
# item is complete
# OR
# item completion condition is met
# OR
# item is disabled
# For families:
# item completion condition is met
# OR
# item is disabled
# OR
# entirety of family is not in alarm
# OR
# final_task_deps... for any children are not met
if 'AlarmName' in item:
alarm_name = item.AlarmName
path = SuitePath(item.path[1:])
with_completes = self.__families_with_completes
with_alarms = self.__families_with_alarms
# Disabled applies recursively to families, so if this node is
# disabled, the netire tree is done:
if 'Disable' in item and item.Disable:
#print(f'{path}: disabled')
return TRUE_DEPENDENCY
if 'Dummy' in item and item.Dummy:
#print(f'{path}: dummy')
return TRUE_DEPENDENCY
# If nothing in the entire tree is in the alarm, then we're done.
if _none_are_in_alarm(item, for_alarm, alarm_name):
#print(f'{path}: entire tree is not in alarm')
return TRUE_DEPENDENCY
if len(path) == 1 and '_is_final_' in path:
# Do not have final tasks depend on one another
return TRUE_DEPENDENCY
elif path:
dep = item.is_completed()
if item.path in self.__completes:
dep = dep | self.__completes[item.path][1]
if item.is_task():
# No children. We're done.
#print(f'{path}: task dep {dep}')
return dep
else:
# This is a suite.
dep = FALSE_DEPENDENCY
if path and _entirely_in_alarm(item,for_alarm,alarm_name) and \
path not in with_completes:
# Families with no "complete" dependency in their entire
# tree have no further dependencies to identify. Their
# own completion is the entirety of the completion
# dependency.
return dep
subdep = TRUE_DEPENDENCY
for subitem in item.child_iter():
dep2 = self._final_task_deps_for_alarm(subitem, for_alarm,
alarm_name)
subdep = subdep & dep2
del dep2
if subdep is not TRUE_DEPENDENCY:
dep = subdep
if item.path in self.__completes:
dep = self.__completes[item.path][1] | subdep
#print(f'{path}: family or suite dep {dep}')
return dep
def _has_completes(self, item):
for i in range(2, len(item)):
path = SuitePath([_ZERO_DT] + item.path[1:i])
if path in self.__completes:
return True
return False