def _family_ascent_point_update(self, fp_id):
"""Updates the given family and children recursively.
First the child families that haven't been checked/updated are acted
on first by calling this function. This recursion ends at the family
first called with this function, which then adds it's first parent
ancestor to the set of families flagged for update.
"""
fp_added = self.added[FAMILY_PROXIES]
fp_data = self.data[self.workflow_id][FAMILY_PROXIES]
if fp_id in fp_data:
fam_node = fp_data[fp_id]
else:
fam_node = fp_added[fp_id]
# Gather child families, then check/update recursively
child_fam_nodes = [
n_id for n_id in fam_node.child_families
if n_id not in self.updated_state_families
]
for child_fam_id in child_fam_nodes:
self._family_ascent_point_update(child_fam_id)
if fp_id in self.state_update_families:
fp_updated = self.updated[FAMILY_PROXIES]
tp_data = self.data[self.workflow_id][TASK_PROXIES]
tp_updated = self.updated[TASK_PROXIES]
# gather child states for count and set is_held
state_counter = Counter({})
is_held_total = 0
for child_id in fam_node.child_families:
child_node = fp_updated.get(child_id, fp_data.get(child_id))
if child_node is not None:
is_held_total += child_node.is_held_total
state_counter += Counter(dict(child_node.state_totals))
task_states = []
for tp_id in fam_node.child_tasks:
tp_node = tp_updated.get(tp_id, tp_data.get(tp_id))
if tp_node is not None:
if tp_node.state:
task_states.append(tp_node.state)
if tp_node.is_held:
is_held_total += 1
state_counter += Counter(task_states)
# created delta data element
fp_delta = PbFamilyProxy(id=fp_id,
stamp=f'{fp_id}@{time()}',
state=extract_group_state(
state_counter.keys()),
is_held=(is_held_total > 0),
is_held_total=is_held_total)
fp_delta.states[:] = state_counter.keys()
for state, state_cnt in state_counter.items():
fp_delta.state_totals[state] = state_cnt
fp_updated.setdefault(fp_id, PbFamilyProxy()).MergeFrom(fp_delta)
# mark as updated in case parent family is updated next
self.updated_state_families.add(fp_id)
# mark parent for update
if fam_node.first_parent:
self.state_update_families.add(fam_node.first_parent)
self.state_update_families.remove(fp_id)
def update_family_proxies(self, cycle_points=None):
"""Update state of family proxies.
Args:
cycle_points (list):
Update family-node state from given list of
valid cycle point strings.
"""
family_proxies = self.data[self.workflow_id][FAMILY_PROXIES]
if cycle_points is None:
cycle_points = self.cycle_states.keys()
if not cycle_points:
return
update_time = time()
for point_string in cycle_points:
# For each cycle point, construct a family state tree
# based on the first-parent single-inheritance tree
c_task_states = self.cycle_states.get(point_string, None)
if c_task_states is None:
continue
c_fam_task_states = {}
c_fam_task_is_held = {}
for key in c_task_states:
state, is_held = c_task_states[key]
if state is None:
continue
for parent in self.ancestors.get(key, []):
if parent == key:
continue
c_fam_task_states.setdefault(parent, set())
c_fam_task_states[parent].add(state)
c_fam_task_is_held.setdefault(parent, False)
if is_held:
c_fam_task_is_held[parent] = is_held
for fam, child_states in c_fam_task_states.items():
state = extract_group_state(child_states)
fp_id = (
f'{self.workflow_id}{ID_DELIM}'
f'{point_string}{ID_DELIM}{fam}')
if state is None or (
fp_id not in family_proxies and
fp_id not in self.updates[FAMILY_PROXIES]):
continue
# Since two fields strings are reassigned,
# it should be safe without copy.
fp_delta = PbFamilyProxy(
id=fp_id,
stamp=f'{fp_id}@{update_time}',
state=state,
is_held=c_fam_task_is_held[fam]
)
self.updates[FAMILY_PROXIES].setdefault(
fp_id, PbFamilyProxy()).MergeFrom(fp_delta)
def _make_entire_workflow(workflow_id):
workflow = PbWorkflow()
workflow.id = workflow_id
entire_workflow = PbEntireWorkflow()
entire_workflow.workflow.CopyFrom(workflow)
root_family = PbFamilyProxy()
root_family.name = 'root'
entire_workflow.family_proxies.extend([root_family])
return entire_workflow
def generate_ghost_family(self, fp_id, child_fam=None, child_task=None):
"""Generate the family-point elements from given ID if non-existent.
Adds the ID of the child proxy that called for it's creation. Also
generates parents recursively to root if they don't exist.
Args:
fp_id (str):
Family proxy ID
child_fam (str):
Family proxy ID
child_task (str):
Task proxy ID
Returns:
None
"""
update_time = time()
families = self.data[self.workflow_id][FAMILIES]
if not families:
families = self.added[FAMILIES]
fp_data = self.data[self.workflow_id][FAMILY_PROXIES]
fp_added = self.added[FAMILY_PROXIES]
fp_updated = self.updated[FAMILY_PROXIES]
if fp_id in fp_data:
fp_delta = fp_data[fp_id]
fp_parent = fp_updated.setdefault(fp_id, PbFamilyProxy(id=fp_id))
elif fp_id in fp_added:
fp_delta = fp_added[fp_id]
fp_parent = fp_added.setdefault(fp_id, PbFamilyProxy(id=fp_id))
else:
_, _, point_string, name = fp_id.split(ID_DELIM)
fam = families[f'{self.workflow_id}{ID_DELIM}{name}']
fp_delta = PbFamilyProxy(
stamp=f'{fp_id}@{update_time}',
id=fp_id,
cycle_point=point_string,
name=fam.name,
family=fam.id,
depth=fam.depth,
)
fp_delta.ancestors[:] = [
f'{self.workflow_id}{ID_DELIM}{point_string}{ID_DELIM}{a_name}'
for a_name in self.ancestors[fam.name] if a_name != fam.name
]
if fp_delta.ancestors:
fp_delta.first_parent = fp_delta.ancestors[0]
self.added[FAMILY_PROXIES][fp_id] = fp_delta
fp_parent = fp_delta
# Add ref ID to family element
f_delta = PbFamily(id=fam.id, stamp=f'{fam.id}@{update_time}')
f_delta.proxies.append(fp_id)
self.updated[FAMILIES].setdefault(
fam.id, PbFamily(id=fam.id)).MergeFrom(f_delta)
# Add ref ID to workflow element
getattr(self.updated[WORKFLOW], FAMILY_PROXIES).append(fp_id)
# Generate this families parent if it not root.
if fp_delta.first_parent:
self.generate_ghost_family(fp_delta.first_parent,
child_fam=fp_id)
if child_fam is None:
fp_parent.child_tasks.append(child_task)
elif child_fam not in fp_parent.child_families:
fp_parent.child_families.append(child_fam)
def generate_ghost_families(self, cycle_points=None):
"""Generate the family-point elements from tasks in cycle points.
Args:
cycle_points (set):
a set of cycle points.
Returns:
list: [cylc.flow.data_messages_pb2.PbFamilyProxy]
list of populated family proxy data elements.
"""
update_time = time()
families = self.data[self.workflow_id][FAMILIES]
if not families:
families = self.updates[FAMILIES]
family_proxies = self.data[self.workflow_id][FAMILY_PROXIES]
for point_string, tasks in self.cycle_states.items():
# construct family tree based on the
# first-parent single-inheritance tree
if not cycle_points or point_string not in cycle_points:
continue
cycle_first_parents = set()
for key in tasks:
for parent in self.ancestors.get(key, []):
if parent == key:
continue
cycle_first_parents.add(parent)
for f_id in families:
fam = families[f_id].name
fp_id = (
f'{self.workflow_id}{ID_DELIM}'
f'{point_string}{ID_DELIM}{fam}')
if (fp_id in family_proxies or
fp_id in self.updates[FAMILY_PROXIES]):
continue
fp_delta = PbFamilyProxy(
stamp=f'{fp_id}@{update_time}',
id=fp_id,
cycle_point=point_string,
name=fam,
family=f'{self.workflow_id}{ID_DELIM}{fam}',
depth=families[f_id].depth,
)
fp_delta.parents[:] = [
f'{self.workflow_id}{ID_DELIM}'
f'{point_string}{ID_DELIM}{p_name}'
for p_name in self.parents[fam]]
fp_delta.ancestors[:] = [
f'{self.workflow_id}{ID_DELIM}'
f'{point_string}{ID_DELIM}{a_name}'
for a_name in self.ancestors[fam]
if a_name != fam]
if fp_delta.ancestors:
fp_delta.first_parent = fp_delta.ancestors[0]
if fam in cycle_first_parents:
for child_name in self.descendants[fam]:
ch_id = (
f'{self.workflow_id}{ID_DELIM}'
f'{point_string}{ID_DELIM}{child_name}'
)
if self.parents[child_name][0] == fam:
if child_name in cycle_first_parents:
fp_delta.child_families.append(ch_id)
elif child_name in self.schd.config.taskdefs:
fp_delta.child_tasks.append(ch_id)
self.updates[FAMILY_PROXIES][fp_id] = fp_delta
# Add ref ID to family element
f_delta = PbFamily(
id=f_id,
stamp=f'{f_id}@{update_time}')
f_delta.proxies.append(fp_id)
self.updates[FAMILIES].setdefault(
f_id, PbFamily(id=f_id)).MergeFrom(f_delta)
# Add ref ID to workflow element
getattr(self.deltas[WORKFLOW], FAMILY_PROXIES).append(fp_id)