def generate_ghost_task(self, t_id, tp_id, point_string):
"""Create task-point element populated with static data.
Args:
t_id (str): data-store task ID.
tp_id (str): data-store task proxy ID.
point_string (str): Valid cycle point string.
Returns:
None
"""
task_proxies = self.data[self.workflow_id][TASK_PROXIES]
if tp_id in task_proxies or tp_id in self.added[TASK_PROXIES]:
return
taskdef = self.data[self.workflow_id][TASKS].get(
t_id, self.added[TASKS].get(t_id))
update_time = time()
tp_stamp = f'{tp_id}@{update_time}'
tproxy = PbTaskProxy(
stamp=tp_stamp,
id=tp_id,
task=t_id,
cycle_point=point_string,
depth=taskdef.depth,
name=taskdef.name,
)
tproxy.namespace[:] = taskdef.namespace
tproxy.ancestors[:] = [
f'{self.workflow_id}{ID_DELIM}{point_string}{ID_DELIM}{a_name}'
for a_name in self.ancestors[taskdef.name]
if a_name != taskdef.name
]
tproxy.first_parent = tproxy.ancestors[0]
self.added[TASK_PROXIES][tp_id] = tproxy
getattr(self.updated[WORKFLOW], TASK_PROXIES).append(tp_id)
self.updated[TASKS].setdefault(
t_id, PbTask(
stamp=f'{t_id}@{update_time}',
id=t_id,
)).proxies.append(tp_id)
self.generate_ghost_family(tproxy.first_parent, child_task=tp_id)
def update_task_proxies(self, updated_tasks=None):
"""Update dynamic fields of task nodes/proxies.
Args:
updated_tasks (list): [cylc.flow.task_proxy.TaskProxy]
Update task-node from corresponding given list of
task proxy objects from the workflow task pool.
"""
if not updated_tasks:
return
tasks = self.data[self.workflow_id][TASKS]
task_proxies = self.data[self.workflow_id][TASK_PROXIES]
update_time = time()
task_defs = {}
# update task instance
for itask in updated_tasks:
name, point_string = TaskID.split(itask.identity)
tp_id = (
f'{self.workflow_id}{ID_DELIM}{point_string}{ID_DELIM}{name}')
if (tp_id not in task_proxies
and tp_id not in self.added[TASK_PROXIES]):
continue
# Gather task definitions for elapsed time recalculation.
if name not in task_defs:
task_defs[name] = itask.tdef
# Create new message and copy existing message content.
tp_delta = self.updated[TASK_PROXIES].setdefault(
tp_id, PbTaskProxy(id=tp_id))
tp_delta.stamp = f'{tp_id}@{update_time}'
tp_delta.state = itask.state.status
if tp_id in task_proxies:
self.state_update_families.add(
task_proxies[tp_id].first_parent)
else:
self.state_update_families.add(
self.added[TASK_PROXIES][tp_id].first_parent)
tp_delta.is_held = itask.state.is_held
tp_delta.flow_label = itask.flow_label
tp_delta.job_submits = itask.submit_num
tp_delta.latest_message = itask.summary['latest_message']
tp_delta.jobs[:] = [
j_id for j_id in self.schd.job_pool.task_jobs.get(tp_id, [])
if j_id not in task_proxies.get(tp_id, PbTaskProxy()).jobs
]
prereq_list = []
for prereq in itask.state.prerequisites:
# Protobuf messages populated within
prereq_obj = prereq.api_dump(self.workflow_id)
if prereq_obj:
prereq_list.append(prereq_obj)
tp_delta.prerequisites.extend(prereq_list)
tp_delta.outputs = json.dumps({
trigger: is_completed
for trigger, _, is_completed in itask.state.outputs.get_all()
})
extras = {}
if itask.tdef.clocktrigger_offset is not None:
extras['Clock trigger time reached'] = (
itask.is_waiting_clock_done())
extras['Triggers at'] = time2str(itask.clock_trigger_time)
for trig, satisfied in itask.state.external_triggers.items():
key = f'External trigger "{trig}"'
if satisfied:
extras[key] = 'satisfied'
else:
extras[key] = 'NOT satisfied'
for label, satisfied in itask.state.xtriggers.items():
sig = self.schd.xtrigger_mgr.get_xtrig_ctx(
itask, label).get_signature()
extra = f'xtrigger "{label} = {sig}"'
if satisfied:
extras[extra] = 'satisfied'
else:
extras[extra] = 'NOT satisfied'
tp_delta.extras = json.dumps(extras)
# Recalculate effected task def elements elapsed time.
for name, tdef in task_defs.items():
elapsed_time = task_mean_elapsed_time(tdef)
if elapsed_time:
t_id = f'{self.workflow_id}{ID_DELIM}{name}'
t_delta = PbTask(stamp=f'{t_id}@{update_time}',
mean_elapsed_time=elapsed_time)
self.updated[TASKS].setdefault(
t_id, PbTask(id=t_id)).MergeFrom(t_delta)
tasks[t_id].MergeFrom(t_delta)
def update_task_proxies(self, updated_tasks=None):
"""Update dynamic fields of task nodes/proxies.
Args:
updated_tasks (list): [cylc.flow.task_proxy.TaskProxy]
Update task-node from corresponding given list of
task proxy objects from the workflow task pool.
"""
if not updated_tasks:
return
tasks = self.data[self.workflow_id][TASKS]
task_proxies = self.data[self.workflow_id][TASK_PROXIES]
update_time = time()
task_defs = {}
# update task instance
for itask in updated_tasks:
name, point_string = TaskID.split(itask.identity)
tp_id = (
f'{self.workflow_id}{ID_DELIM}{point_string}{ID_DELIM}{name}')
if (tp_id not in task_proxies and
tp_id not in self.updates[TASK_PROXIES]):
continue
self.cycle_states.setdefault(point_string, {})[name] = (
itask.state.status, itask.state.is_held)
# Gather task definitions for elapsed time recalculation.
if name not in task_defs:
task_defs[name] = itask.tdef
# Create new message and copy existing message content.
tp_delta = self.updates[TASK_PROXIES].setdefault(
tp_id, PbTaskProxy(id=tp_id))
tp_delta.stamp = f'{tp_id}@{update_time}'
tp_delta.state = itask.state.status
tp_delta.is_held = itask.state.is_held
tp_delta.job_submits = itask.submit_num
tp_delta.spawned = itask.has_spawned
tp_delta.latest_message = itask.summary['latest_message']
tp_delta.jobs[:] = [
j_id
for j_id in self.schd.job_pool.task_jobs.get(tp_id, [])
if j_id not in task_proxies.get(tp_id, PbTaskProxy()).jobs
]
tp_delta.broadcasts[:] = [
f'{key}={val}' for key, val in
self.schd.task_events_mgr.broadcast_mgr.get_broadcast(
itask.identity).items()]
prereq_list = []
for prereq in itask.state.prerequisites:
# Protobuf messages populated within
prereq_obj = prereq.api_dump(self.workflow_id)
if prereq_obj:
prereq_list.append(prereq_obj)
tp_delta.prerequisites.extend(prereq_list)
tp_delta.outputs[:] = [
f'{trigger}={is_completed}'
for trigger, _, is_completed in itask.state.outputs.get_all()
]
# Recalculate effected task def elements elapsed time.
for name, tdef in task_defs.items():
elapsed_time = task_mean_elapsed_time(tdef)
if elapsed_time:
t_id = f'{self.workflow_id}{ID_DELIM}{name}'
t_delta = PbTask(
stamp=f'{t_id}@{update_time}',
mean_elapsed_time=elapsed_time
)
self.updates[TASKS].setdefault(
t_id,
PbTask(id=t_id)).MergeFrom(t_delta)
tasks[t_id].MergeFrom(t_delta)
def generate_definition_elements(self):
"""Generate static definition data elements.
Populates the tasks, families, and workflow elements
with data from and/or derived from the workflow definition.
"""
config = self.schd.config
update_time = time()
tasks = self.added[TASKS]
families = self.added[FAMILIES]
workflow = self.added[WORKFLOW]
workflow.id = self.workflow_id
workflow.last_updated = update_time
workflow.stamp = f'{workflow.id}@{workflow.last_updated}'
graph = workflow.edges
graph.leaves[:] = config.leaves
graph.feet[:] = config.feet
for key, info in config.suite_polling_tasks.items():
graph.workflow_polling_tasks.add(
local_proxy=key,
workflow=info[0],
remote_proxy=info[1],
req_state=info[2],
graph_string=info[3],
)
ancestors = config.get_first_parent_ancestors()
descendants = config.get_first_parent_descendants()
parents = config.get_parent_lists()
# Create definition elements for graphed tasks.
for name, tdef in config.taskdefs.items():
t_id = f'{self.workflow_id}{ID_DELIM}{name}'
t_stamp = f'{t_id}@{update_time}'
task = PbTask(
stamp=t_stamp,
id=t_id,
name=name,
depth=len(ancestors[name]) - 1,
)
task.namespace[:] = tdef.namespace_hierarchy
task.first_parent = (
f'{self.workflow_id}{ID_DELIM}{ancestors[name][1]}')
user_defined_meta = {}
for key, val in dict(tdef.describe()).items():
if key in ['title', 'description', 'URL']:
setattr(task.meta, key, val)
else:
user_defined_meta[key] = val
task.meta.user_defined = json.dumps(user_defined_meta)
elapsed_time = task_mean_elapsed_time(tdef)
if elapsed_time:
task.mean_elapsed_time = elapsed_time
task.parents.extend([
f'{self.workflow_id}{ID_DELIM}{p_name}'
for p_name in parents[name]
])
tasks[t_id] = task
# Created family definition elements for first parent
# ancestors of graphed tasks.
for key, names in ancestors.items():
for name in names:
if (key == name or name in families):
continue
f_id = f'{self.workflow_id}{ID_DELIM}{name}'
f_stamp = f'{f_id}@{update_time}'
family = PbFamily(
stamp=f_stamp,
id=f_id,
name=name,
depth=len(ancestors[name]) - 1,
)
famcfg = config.cfg['runtime'][name]
user_defined_meta = {}
for key, val in famcfg.get('meta', {}).items():
if key in ['title', 'description', 'URL']:
setattr(family.meta, key, val)
else:
user_defined_meta[key] = val
family.meta.user_defined = json.dumps(user_defined_meta)
family.parents.extend([
f'{self.workflow_id}{ID_DELIM}{p_name}'
for p_name in parents[name]
])
try:
family.first_parent = (
f'{self.workflow_id}{ID_DELIM}{ancestors[name][1]}')
except IndexError:
pass
families[f_id] = family
for name, parent_list in parents.items():
if not parent_list:
continue
fam = parent_list[0]
f_id = f'{self.workflow_id}{ID_DELIM}{fam}'
if f_id in families:
ch_id = f'{self.workflow_id}{ID_DELIM}{name}'
if name in config.taskdefs:
families[f_id].child_tasks.append(ch_id)
else:
families[f_id].child_families.append(ch_id)
# Populate static fields of workflow
workflow.api_version = API
workflow.cylc_version = CYLC_VERSION
workflow.name = self.schd.suite
workflow.owner = self.schd.owner
workflow.host = self.schd.host
workflow.port = self.schd.port or -1
workflow.pub_port = self.schd.pub_port or -1
user_defined_meta = {}
for key, val in config.cfg['meta'].items():
if key in ['title', 'description', 'URL']:
setattr(workflow.meta, key, val)
else:
user_defined_meta[key] = val
workflow.meta.user_defined = json.dumps(user_defined_meta)
workflow.tree_depth = max([
len(val)
for val in config.get_first_parent_ancestors(pruned=True).values()
]) - 1
if get_utc_mode():
time_zone_info = TIME_ZONE_UTC_INFO
else:
time_zone_info = TIME_ZONE_LOCAL_INFO
for key, val in time_zone_info.items():
setattr(workflow.time_zone_info, key, val)
workflow.run_mode = config.run_mode()
workflow.cycling_mode = config.cfg['scheduling']['cycling mode']
workflow.workflow_log_dir = self.schd.suite_log_dir
workflow.job_log_names.extend(list(JOB_LOG_OPTS.values()))
workflow.ns_def_order.extend(config.ns_defn_order)
workflow.broadcasts = json.dumps(self.schd.broadcast_mgr.broadcasts)
workflow.tasks.extend(list(tasks))
workflow.families.extend(list(families))
self.ancestors = ancestors
self.descendants = descendants
self.parents = parents
def generate_graph_elements(self, start_point=None, stop_point=None):
"""Generate edges and [ghost] nodes (family and task proxy elements).
Args:
start_point (cylc.flow.cycling.PointBase):
Edge generation start point.
stop_point (cylc.flow.cycling.PointBase):
Edge generation stop point.
"""
if not self.pool_points:
return
config = self.schd.config
tasks = self.data[self.workflow_id][TASKS]
if not tasks:
tasks = self.updates[TASKS]
task_proxies = self.data[self.workflow_id][TASK_PROXIES]
if start_point is None:
start_point = min(self.pool_points)
if stop_point is None:
stop_point = max(self.pool_points)
# Used for generating family [ghost] nodes
new_points = set()
# Generate ungrouped edges
for edge in config.get_graph_edges(start_point, stop_point):
# Reference or create edge source & target nodes/proxies
s_node = edge[0]
t_node = edge[1]
if s_node is None:
continue
# Is the source cycle point in the task pool?
s_name, s_point = TaskID.split(s_node)
s_point_cls = get_point(s_point)
s_pool_point = False
s_valid = TaskID.is_valid_id(s_node)
if s_valid:
s_pool_point = s_point_cls in self.pool_points
# Is the target cycle point in the task pool?
t_pool_point = False
t_valid = t_node and TaskID.is_valid_id(t_node)
if t_valid:
t_name, t_point = TaskID.split(t_node)
t_point_cls = get_point(t_point)
t_pool_point = get_point(t_point) in self.pool_points
# Proceed if either source or target cycle points
# are in the task pool.
if not s_pool_point and not t_pool_point:
continue
# If source/target is valid add/create the corresponding items.
# TODO: if xtrigger is suite_state create remote ID
source_id = (
f'{self.workflow_id}{ID_DELIM}{s_point}{ID_DELIM}{s_name}')
if s_valid:
s_task_id = f'{self.workflow_id}{ID_DELIM}{s_name}'
new_points.add(s_point)
# Add source points for pruning.
self.edge_points.setdefault(s_point_cls, set())
if (source_id not in task_proxies and
source_id not in self.updates[TASK_PROXIES]):
self.updates[TASK_PROXIES][source_id] = (
self.generate_ghost_task(s_node))
getattr(
self.deltas[WORKFLOW], TASK_PROXIES).append(source_id)
if (source_id not in tasks[s_task_id].proxies and
source_id not in self.updates[TASKS].get(
s_task_id, PbTask()).proxies):
self.updates[TASKS].setdefault(
s_task_id,
PbTask(
stamp='f{s_task_id}@{update_time}',
id=s_task_id,
)).proxies.append(source_id)
# Add valid source before checking for no target,
# as source may be an isolate (hence no edges).
# At present targets can't be xtriggers.
if t_valid:
target_id = (
f'{self.workflow_id}{ID_DELIM}{t_point}{ID_DELIM}{t_name}')
t_task_id = f'{self.workflow_id}{ID_DELIM}{t_name}'
new_points.add(t_point)
# Add target points to associated source points for pruning.
self.edge_points.setdefault(s_point_cls, set())
self.edge_points[s_point_cls].add(t_point_cls)
if (target_id not in task_proxies and
target_id not in self.updates[TASK_PROXIES]):
self.updates[TASK_PROXIES][target_id] = (
self.generate_ghost_task(t_node))
getattr(self.deltas[WORKFLOW], TASK_PROXIES).append(
target_id)
if (target_id not in tasks[t_task_id].proxies and
target_id not in self.updates[TASKS].get(
t_task_id, PbTask()).proxies):
self.updates[TASKS].setdefault(
t_task_id,
PbTask(
stamp='f{t_task_id}@{update_time}',
id=t_task_id,
)).proxies.append(target_id)
# Initiate edge element.
e_id = (
f'{self.workflow_id}{ID_DELIM}{s_node}{ID_DELIM}{t_node}')
self.updates[EDGES][e_id] = PbEdge(
id=e_id,
suicide=edge[3],
cond=edge[4],
source=source_id,
target=target_id,
)
# Add edge id to node field for resolver reference
self.updates[TASK_PROXIES].setdefault(
target_id,
PbTaskProxy(id=target_id)).edges.append(e_id)
if s_valid:
self.updates[TASK_PROXIES].setdefault(
source_id,
PbTaskProxy(id=source_id)).edges.append(e_id)
getattr(
self.deltas.setdefault(WORKFLOW, PbWorkflow()),
EDGES).edges.extend(self.updates[EDGES].keys())
if new_points:
self.generate_ghost_families(new_points)