def _family_ascent_point_update(self, fp_id):
"""Updates the given family and children recursively.
First the child families that haven't been checked/updated are acted
on first by calling this function. This recursion ends at the family
first called with this function, which then adds it's first parent
ancestor to the set of families flagged for update.
"""
fp_added = self.added[FAMILY_PROXIES]
fp_data = self.data[self.workflow_id][FAMILY_PROXIES]
if fp_id in fp_data:
fam_node = fp_data[fp_id]
else:
fam_node = fp_added[fp_id]
# Gather child families, then check/update recursively
child_fam_nodes = [
n_id for n_id in fam_node.child_families
if n_id not in self.updated_state_families
]
for child_fam_id in child_fam_nodes:
self._family_ascent_point_update(child_fam_id)
if fp_id in self.state_update_families:
fp_updated = self.updated[FAMILY_PROXIES]
tp_data = self.data[self.workflow_id][TASK_PROXIES]
tp_updated = self.updated[TASK_PROXIES]
# gather child states for count and set is_held
state_counter = Counter({})
is_held_total = 0
for child_id in fam_node.child_families:
child_node = fp_updated.get(child_id, fp_data.get(child_id))
if child_node is not None:
is_held_total += child_node.is_held_total
state_counter += Counter(dict(child_node.state_totals))
task_states = []
for tp_id in fam_node.child_tasks:
tp_node = tp_updated.get(tp_id, tp_data.get(tp_id))
if tp_node is not None:
if tp_node.state:
task_states.append(tp_node.state)
if tp_node.is_held:
is_held_total += 1
state_counter += Counter(task_states)
# created delta data element
fp_delta = PbFamilyProxy(id=fp_id,
stamp=f'{fp_id}@{time()}',
state=extract_group_state(
state_counter.keys()),
is_held=(is_held_total > 0),
is_held_total=is_held_total)
fp_delta.states[:] = state_counter.keys()
for state, state_cnt in state_counter.items():
fp_delta.state_totals[state] = state_cnt
fp_updated.setdefault(fp_id, PbFamilyProxy()).MergeFrom(fp_delta)
# mark as updated in case parent family is updated next
self.updated_state_families.add(fp_id)
# mark parent for update
if fam_node.first_parent:
self.state_update_families.add(fam_node.first_parent)
self.state_update_families.remove(fp_id)
def update_family_proxies(self, cycle_points=None):
"""Update state of family proxies.
Args:
cycle_points (list):
Update family-node state from given list of
valid cycle point strings.
"""
family_proxies = self.data[self.workflow_id][FAMILY_PROXIES]
if cycle_points is None:
cycle_points = self.cycle_states.keys()
if not cycle_points:
return
update_time = time()
for point_string in cycle_points:
# For each cycle point, construct a family state tree
# based on the first-parent single-inheritance tree
c_task_states = self.cycle_states.get(point_string, None)
if c_task_states is None:
continue
c_fam_task_states = {}
c_fam_task_is_held = {}
for key in c_task_states:
state, is_held = c_task_states[key]
if state is None:
continue
for parent in self.ancestors.get(key, []):
if parent == key:
continue
c_fam_task_states.setdefault(parent, set())
c_fam_task_states[parent].add(state)
c_fam_task_is_held.setdefault(parent, False)
if is_held:
c_fam_task_is_held[parent] = is_held
for fam, child_states in c_fam_task_states.items():
state = extract_group_state(child_states)
fp_id = (
f'{self.workflow_id}{ID_DELIM}'
f'{point_string}{ID_DELIM}{fam}')
if state is None or (
fp_id not in family_proxies and
fp_id not in self.updates[FAMILY_PROXIES]):
continue
# Since two fields strings are reassigned,
# it should be safe without copy.
fp_delta = PbFamilyProxy(
id=fp_id,
stamp=f'{fp_id}@{update_time}',
state=state,
is_held=c_fam_task_is_held[fam]
)
self.updates[FAMILY_PROXIES].setdefault(
fp_id, PbFamilyProxy()).MergeFrom(fp_delta)
def update_family_proxies(self, cycle_points=None):
"""Update state of family proxies"""
update_time = time()
# Compute state_counts (total, and per cycle).
all_states = []
state_count_cycles = {}
for point_string, c_task_states in self.cycle_states.items():
# For each cycle point, construct a family state tree
# based on the first-parent single-inheritance tree
if cycle_points and point_string not in cycle_points:
continue
c_fam_task_states = {}
count = {}
for key in c_task_states:
state = c_task_states[key]
if state is None:
continue
try:
count[state] += 1
except KeyError:
count[state] = 1
all_states.append(state)
for parent in self.ancestors.get(key, []):
if parent == key:
continue
c_fam_task_states.setdefault(parent, set([]))
c_fam_task_states[parent].add(state)
state_count_cycles[point_string] = count
for fam, child_states in c_fam_task_states.items():
state = extract_group_state(child_states)
int_id = TaskID.get(fam, point_string)
if state is None or int_id not in self.family_proxies:
continue
fproxy = self.family_proxies[int_id]
fproxy.checksum = f"{int_id}@{update_time}"
fproxy.state = state
self.all_states = all_states
self.state_count_cycles = state_count_cycles
def update(self, schd):
"""Update."""
self.update_time = time()
global_summary = {}
family_summary = {}
task_summary, task_states = self._get_tasks_info(schd)
all_states = []
ancestors_dict = schd.config.get_first_parent_ancestors()
# Compute state_counts (total, and per cycle).
state_count_totals = {}
state_count_cycles = {}
for point_string, c_task_states in task_states.items():
# For each cycle point, construct a family state tree
# based on the first-parent single-inheritance tree
c_fam_task_states = {}
count = {}
for key in c_task_states:
state = c_task_states[key]
if state is None:
continue
try:
count[state] += 1
except KeyError:
count[state] = 1
all_states.append(state)
for parent in ancestors_dict.get(key, []):
if parent == key:
continue
c_fam_task_states.setdefault(parent, set([]))
c_fam_task_states[parent].add(state)
state_count_cycles[point_string] = count
for fam, child_states in c_fam_task_states.items():
f_id = TaskID.get(fam, point_string)
state = extract_group_state(child_states)
if state is None:
continue
try:
famcfg = schd.config.cfg['runtime'][fam]['meta']
except KeyError:
famcfg = {}
description = famcfg.get('description')
title = famcfg.get('title')
family_summary[f_id] = {'name': fam,
'description': description,
'title': title,
'label': point_string,
'state': state}
state_count_totals = {}
for point_string, count in list(state_count_cycles.items()):
for state, state_count in count.items():
state_count_totals.setdefault(state, 0)
state_count_totals[state] += state_count
all_states.sort()
for key, value in (
('oldest cycle point string', schd.pool.get_min_point()),
('newest cycle point string', schd.pool.get_max_point()),
('newest runahead cycle point string',
schd.pool.get_max_point_runahead())):
if value:
global_summary[key] = str(value)
else:
global_summary[key] = None
if get_utc_mode():
global_summary['time zone info'] = TIME_ZONE_UTC_INFO
else:
global_summary['time zone info'] = TIME_ZONE_LOCAL_INFO
global_summary['last_updated'] = self.update_time
global_summary['run_mode'] = schd.config.run_mode()
global_summary['states'] = all_states
global_summary['namespace definition order'] = (
schd.config.ns_defn_order)
global_summary['reloading'] = schd.pool.do_reload
global_summary['state totals'] = state_count_totals
# Extract suite and task URLs from config.
global_summary['suite_urls'] = dict(
(i, j['meta']['URL'])
for (i, j) in schd.config.cfg['runtime'].items())
global_summary['suite_urls']['suite'] = schd.config.cfg['meta']['URL']
# Construct a suite status string for use by monitoring clients.
status, status_msg = get_suite_status(schd)
global_summary['status'] = str(status)
global_summary['status_string'] = status_msg
# Replace the originals (atomic update, for access from other threads).
self.task_summary = task_summary
self.global_summary = global_summary
self.family_summary = family_summary
self.state_count_totals = state_count_totals
self.state_count_cycles = state_count_cycles
def update(self, schd):
"""Update."""
self.update_time = time()
global_summary = {}
family_summary = {}
task_summary, task_states = self._get_tasks_info(schd)
all_states = []
ancestors_dict = schd.config.get_first_parent_ancestors()
# Compute state_counts (total, and per cycle).
state_count_totals = {}
state_count_cycles = {}
for point_string, c_task_states in task_states.items():
# For each cycle point, construct a family state tree
# based on the first-parent single-inheritance tree
c_fam_task_states = {}
count = {}
for key in c_task_states:
state = c_task_states[key]
if state is None:
continue
try:
count[state] += 1
except KeyError:
count[state] = 1
all_states.append(state)
for parent in ancestors_dict.get(key, []):
if parent == key:
continue
c_fam_task_states.setdefault(parent, set([]))
c_fam_task_states[parent].add(state)
state_count_cycles[point_string] = count
for fam, child_states in c_fam_task_states.items():
f_id = TaskID.get(fam, point_string)
state = extract_group_state(child_states)
if state is None:
continue
try:
famcfg = schd.config.cfg['runtime'][fam]['meta']
except KeyError:
famcfg = {}
description = famcfg.get('description')
title = famcfg.get('title')
family_summary[f_id] = {'name': fam,
'description': description,
'title': title,
'label': point_string,
'state': state}
state_count_totals = {}
for point_string, count in list(state_count_cycles.items()):
for state, state_count in count.items():
state_count_totals.setdefault(state, 0)
state_count_totals[state] += state_count
all_states.sort()
for key, value in (
('oldest cycle point string', schd.pool.get_min_point()),
('newest cycle point string', schd.pool.get_max_point()),
('newest runahead cycle point string',
schd.pool.get_max_point_runahead())):
if value:
global_summary[key] = str(value)
else:
global_summary[key] = None
if get_utc_mode():
global_summary['time zone info'] = TIME_ZONE_UTC_INFO
else:
global_summary['time zone info'] = TIME_ZONE_LOCAL_INFO
global_summary['last_updated'] = self.update_time
global_summary['run_mode'] = schd.run_mode
global_summary['states'] = all_states
global_summary['namespace definition order'] = (
schd.config.ns_defn_order)
global_summary['reloading'] = schd.pool.do_reload
global_summary['state totals'] = state_count_totals
# Extract suite and task URLs from config.
global_summary['suite_urls'] = dict(
(i, j['meta']['URL'])
for (i, j) in schd.config.cfg['runtime'].items())
global_summary['suite_urls']['suite'] = schd.config.cfg['meta']['URL']
# Construct a suite status string for use by monitoring clients.
if schd.pool.is_held:
global_summary['status_string'] = SUITE_STATUS_HELD
elif schd.stop_mode is not None:
global_summary['status_string'] = SUITE_STATUS_STOPPING
elif schd.pool.hold_point:
global_summary['status_string'] = (
SUITE_STATUS_RUNNING_TO_HOLD % schd.pool.hold_point)
elif schd.stop_point:
global_summary['status_string'] = (
SUITE_STATUS_RUNNING_TO_STOP % schd.stop_point)
elif schd.stop_clock_time is not None:
global_summary['status_string'] = (
SUITE_STATUS_RUNNING_TO_STOP % schd.stop_clock_time_string)
elif schd.stop_task:
global_summary['status_string'] = (
SUITE_STATUS_RUNNING_TO_STOP % schd.stop_task)
elif schd.final_point:
global_summary['status_string'] = (
SUITE_STATUS_RUNNING_TO_STOP % schd.final_point)
else:
global_summary['status_string'] = SUITE_STATUS_RUNNING
# Replace the originals (atomic update, for access from other threads).
self.task_summary = task_summary
self.global_summary = global_summary
self.family_summary = family_summary
self.state_count_totals = state_count_totals
self.state_count_cycles = state_count_cycles