def generate_definition_elements(self):
"""Generate static definition data elements"""
config = self.schd.config
update_time = time()
tasks = {}
families = {}
workflow = PbWorkflow(
checksum=f"{self.workflow_id}@{update_time}",
id=self.workflow_id,
)
ancestors = config.get_first_parent_ancestors()
descendants = config.get_first_parent_descendants()
parents = config.get_parent_lists()
# create task definition data objects
for name, tdef in config.taskdefs.items():
t_id = f"{self.workflow_id}/{name}"
t_check = f"{name}@{update_time}"
task = PbTask(
checksum=t_check,
id=t_id,
name=name,
depth=len(ancestors[name]) - 1,
)
task.namespace[:] = tdef.namespace_hierarchy
for key, val in dict(tdef.describe()).items():
if key in ['title', 'description', 'url']:
setattr(task.meta, key, val)
else:
task.meta.user_defined.append(f"{key}={val}")
ntimes = len(tdef.elapsed_times)
if ntimes:
task.mean_elapsed_time = sum(tdef.elapsed_times) / ntimes
elif tdef.rtconfig['job']['execution time limit']:
task.mean_elapsed_time = \
tdef.rtconfig['job']['execution time limit']
tasks[name] = task
# family definition data objects creation
for name in ancestors.keys():
if name in config.taskdefs.keys():
continue
f_id = f"{self.workflow_id}/{name}"
f_check = f"{name}@{update_time}"
family = PbFamily(
checksum=f_check,
id=f_id,
name=name,
depth=len(ancestors[name]) - 1,
)
famcfg = config.cfg['runtime'][name]
for key, val in famcfg.get('meta', {}).items():
if key in ['title', 'description', 'url']:
setattr(family.meta, key, val)
else:
family.meta.user_defined.append(f"{key}={val}")
family.parents.extend(
[f"{self.workflow_id}/{p_name}"
for p_name in parents[name]])
families[name] = family
for name, parent_list in parents.items():
if parent_list and parent_list[0] in families:
ch_id = f"{self.workflow_id}/{name}"
if name in config.taskdefs:
families[parent_list[0]].child_tasks.append(ch_id)
else:
families[parent_list[0]].child_families.append(ch_id)
workflow.api_version = self.schd.server.API
workflow.cylc_version = CYLC_VERSION
workflow.name = self.schd.suite
workflow.owner = self.schd.owner
workflow.host = self.schd.host
workflow.port = self.schd.port
for key, val in config.cfg['meta'].items():
if key in ['title', 'description', 'URL']:
setattr(workflow.meta, key, val)
else:
workflow.meta.user_defined.append(f"{key}={val}")
workflow.tree_depth = max(
[len(val) for key, val in ancestors.items()]) - 1
if get_utc_mode():
time_zone_info = TIME_ZONE_UTC_INFO
else:
time_zone_info = TIME_ZONE_LOCAL_INFO
for key, val in time_zone_info.items():
setattr(workflow.time_zone_info, key, val)
workflow.last_updated = update_time
workflow.run_mode = config.run_mode()
workflow.cycling_mode = config.cfg['scheduling']['cycling mode']
workflow.workflow_log_dir = self.schd.suite_log_dir
workflow.job_log_names.extend(list(JOB_LOG_OPTS.values()))
workflow.ns_defn_order.extend(config.ns_defn_order)
workflow.tasks.extend([t.id for t in tasks.values()])
workflow.families.extend([f.id for f in families.values()])
# replace the originals (atomic update, for access from other threads).
self.ancestors = ancestors
self.descendants = descendants
self.parents = parents
self.tasks = tasks
self.families = families
self.workflow = workflow
def generate_definition_elements(self):
"""Generate static definition data elements.
Populates the tasks, families, and workflow elements
with data from and/or derived from the workflow definition.
"""
config = self.schd.config
update_time = time()
tasks = self.added[TASKS]
families = self.added[FAMILIES]
workflow = self.added[WORKFLOW]
workflow.id = self.workflow_id
workflow.last_updated = update_time
workflow.stamp = f'{workflow.id}@{workflow.last_updated}'
graph = workflow.edges
graph.leaves[:] = config.leaves
graph.feet[:] = config.feet
for key, info in config.suite_polling_tasks.items():
graph.workflow_polling_tasks.add(
local_proxy=key,
workflow=info[0],
remote_proxy=info[1],
req_state=info[2],
graph_string=info[3],
)
ancestors = config.get_first_parent_ancestors()
descendants = config.get_first_parent_descendants()
parents = config.get_parent_lists()
# Create definition elements for graphed tasks.
for name, tdef in config.taskdefs.items():
t_id = f'{self.workflow_id}{ID_DELIM}{name}'
t_stamp = f'{t_id}@{update_time}'
task = PbTask(
stamp=t_stamp,
id=t_id,
name=name,
depth=len(ancestors[name]) - 1,
)
task.namespace[:] = tdef.namespace_hierarchy
task.first_parent = (
f'{self.workflow_id}{ID_DELIM}{ancestors[name][1]}')
user_defined_meta = {}
for key, val in dict(tdef.describe()).items():
if key in ['title', 'description', 'URL']:
setattr(task.meta, key, val)
else:
user_defined_meta[key] = val
task.meta.user_defined = json.dumps(user_defined_meta)
elapsed_time = task_mean_elapsed_time(tdef)
if elapsed_time:
task.mean_elapsed_time = elapsed_time
task.parents.extend([
f'{self.workflow_id}{ID_DELIM}{p_name}'
for p_name in parents[name]
])
tasks[t_id] = task
# Created family definition elements for first parent
# ancestors of graphed tasks.
for key, names in ancestors.items():
for name in names:
if (key == name or name in families):
continue
f_id = f'{self.workflow_id}{ID_DELIM}{name}'
f_stamp = f'{f_id}@{update_time}'
family = PbFamily(
stamp=f_stamp,
id=f_id,
name=name,
depth=len(ancestors[name]) - 1,
)
famcfg = config.cfg['runtime'][name]
user_defined_meta = {}
for key, val in famcfg.get('meta', {}).items():
if key in ['title', 'description', 'URL']:
setattr(family.meta, key, val)
else:
user_defined_meta[key] = val
family.meta.user_defined = json.dumps(user_defined_meta)
family.parents.extend([
f'{self.workflow_id}{ID_DELIM}{p_name}'
for p_name in parents[name]
])
try:
family.first_parent = (
f'{self.workflow_id}{ID_DELIM}{ancestors[name][1]}')
except IndexError:
pass
families[f_id] = family
for name, parent_list in parents.items():
if not parent_list:
continue
fam = parent_list[0]
f_id = f'{self.workflow_id}{ID_DELIM}{fam}'
if f_id in families:
ch_id = f'{self.workflow_id}{ID_DELIM}{name}'
if name in config.taskdefs:
families[f_id].child_tasks.append(ch_id)
else:
families[f_id].child_families.append(ch_id)
# Populate static fields of workflow
workflow.api_version = API
workflow.cylc_version = CYLC_VERSION
workflow.name = self.schd.suite
workflow.owner = self.schd.owner
workflow.host = self.schd.host
workflow.port = self.schd.port or -1
workflow.pub_port = self.schd.pub_port or -1
user_defined_meta = {}
for key, val in config.cfg['meta'].items():
if key in ['title', 'description', 'URL']:
setattr(workflow.meta, key, val)
else:
user_defined_meta[key] = val
workflow.meta.user_defined = json.dumps(user_defined_meta)
workflow.tree_depth = max([
len(val)
for val in config.get_first_parent_ancestors(pruned=True).values()
]) - 1
if get_utc_mode():
time_zone_info = TIME_ZONE_UTC_INFO
else:
time_zone_info = TIME_ZONE_LOCAL_INFO
for key, val in time_zone_info.items():
setattr(workflow.time_zone_info, key, val)
workflow.run_mode = config.run_mode()
workflow.cycling_mode = config.cfg['scheduling']['cycling mode']
workflow.workflow_log_dir = self.schd.suite_log_dir
workflow.job_log_names.extend(list(JOB_LOG_OPTS.values()))
workflow.ns_def_order.extend(config.ns_defn_order)
workflow.broadcasts = json.dumps(self.schd.broadcast_mgr.broadcasts)
workflow.tasks.extend(list(tasks))
workflow.families.extend(list(families))
self.ancestors = ancestors
self.descendants = descendants
self.parents = parents
def generate_definition_elements(self):
"""Generate static definition data elements.
Populates the tasks, families, and workflow elements
with data from and/or derived from the workflow definition.
"""
config = self.schd.config
update_time = time()
tasks = {}
families = {}
workflow = PbWorkflow(
stamp=f'{self.workflow_id}@{update_time}',
id=self.workflow_id,
)
graph = self.data[self.workflow_id][GRAPH]
graph.leaves[:] = config.leaves
graph.feet[:] = config.feet
for key, info in config.suite_polling_tasks.items():
graph.workflow_polling_tasks.add(
local_proxy=key,
workflow=info[0],
remote_proxy=info[1],
req_state=info[2],
graph_string=info[3],
)
ancestors = config.get_first_parent_ancestors()
descendants = config.get_first_parent_descendants()
parents = config.get_parent_lists()
# Create task definition elements.
for name, tdef in config.taskdefs.items():
t_id = f'{self.workflow_id}{ID_DELIM}{name}'
t_check = f'{t_id}@{update_time}'
task = PbTask(
stamp=t_check,
id=t_id,
name=name,
depth=len(ancestors[name]) - 1,
)
task.namespace[:] = tdef.namespace_hierarchy
for key, val in dict(tdef.describe()).items():
if key in ['title', 'description', 'url']:
setattr(task.meta, key, val)
else:
task.meta.user_defined.append(f'{key}={val}')
elapsed_time = task_mean_elapsed_time(tdef)
if elapsed_time:
task.mean_elapsed_time = elapsed_time
tasks[t_id] = task
# Created family definition elements.
for name in ancestors.keys():
if name in config.taskdefs.keys():
continue
f_id = f'{self.workflow_id}{ID_DELIM}{name}'
f_check = f'{f_id}@{update_time}'
family = PbFamily(
stamp=f_check,
id=f_id,
name=name,
depth=len(ancestors[name]) - 1,
)
famcfg = config.cfg['runtime'][name]
for key, val in famcfg.get('meta', {}).items():
if key in ['title', 'description', 'url']:
setattr(family.meta, key, val)
else:
family.meta.user_defined.append(f'{key}={val}')
family.parents.extend([
f'{self.workflow_id}{ID_DELIM}{p_name}'
for p_name in parents[name]
])
families[f_id] = family
for name, parent_list in parents.items():
if not parent_list:
continue
fam = parent_list[0]
f_id = f'{self.workflow_id}{ID_DELIM}{fam}'
if f_id in families:
ch_id = f'{self.workflow_id}{ID_DELIM}{name}'
if name in config.taskdefs:
families[f_id].child_tasks.append(ch_id)
else:
families[f_id].child_families.append(ch_id)
# Populate static fields of workflow
workflow.api_version = self.schd.server.API
workflow.cylc_version = CYLC_VERSION
workflow.name = self.schd.suite
workflow.owner = self.schd.owner
workflow.host = self.schd.host
workflow.port = self.schd.port
for key, val in config.cfg['meta'].items():
if key in ['title', 'description', 'URL']:
setattr(workflow.meta, key, val)
else:
workflow.meta.user_defined.append(f'{key}={val}')
workflow.tree_depth = max([len(val)
for key, val in ancestors.items()]) - 1
if get_utc_mode():
time_zone_info = TIME_ZONE_UTC_INFO
else:
time_zone_info = TIME_ZONE_LOCAL_INFO
for key, val in time_zone_info.items():
setattr(workflow.time_zone_info, key, val)
workflow.last_updated = update_time
workflow.run_mode = config.run_mode()
workflow.cycling_mode = config.cfg['scheduling']['cycling mode']
workflow.workflow_log_dir = self.schd.suite_log_dir
workflow.job_log_names.extend(list(JOB_LOG_OPTS.values()))
workflow.ns_defn_order.extend(config.ns_defn_order)
workflow.tasks.extend(list(tasks))
workflow.families.extend(list(families))
# replace the originals (atomic update, for access from other threads).
self.ancestors = ancestors
self.descendants = descendants
self.parents = parents
self.data[self.workflow_id][TASKS] = tasks
self.data[self.workflow_id][FAMILIES] = families
self.data[self.workflow_id][WORKFLOW] = workflow