def group_super_enum(json_blob):
"""Group the nodes by super enum."""
new_children = []
grouping = collections.defaultdict(list)
direct_enum = set()
for child in json_blob['cd']:
if 'se' in child:
# Now only group by one super enum.
assert len(child['se']) == 1
v = list(child['se'].values())[0]
grouping[v].append(child)
else:
new_children.append(child)
direct_enum.add(child['e'])
# Add a layer for super enum
for v, children in grouping.items():
if v in direct_enum:
for child in new_children:
if child['e'] == v:
child['cd'] = children
child['c'] += len(children)
break
else:
direct_enum.add(v)
enum_blob = children[0].copy() # Shallow copy
enum_blob['cd'] = children
enum_blob['l'] = text_format.format_title(v)
enum_blob['c'] = len(children)
enum_blob['t'] = 'p'
del enum_blob['sv']
new_children.append(enum_blob)
json_blob['cd'] = new_children
return json_blob
def json_blob(self, cpvs: Dict[str, str]):
result = {
'populationType': self.pop_type,
'sv': self.dcids,
'l': text_format.format_title(self.name),
't': 'v',
'e': self.name,
'c': 1,
'cd': [],
'sv_set': set(self.dcids),
}
for node in self.leafs:
if node.addLevel:
result['cd'].append(node.json_blob(cpvs))
else:
# add the statsVar dcids to the value ndoe
result['sv'].extend(node.dcids)
result['sv_set'].update(node.dcids)
if not result['sv']:
result['t'] = 'p'
if result['cd']:
result['cd'] = text_format.filter_and_sort(self.name, result['cd'])
if result['t'] == 'v':
result['d'] = find_denominators(self.pos, cpvs, self.stats_vars_all,
result['sv'])
return result
def json_blob(self):
""" Generate the json blob for the property node"""
if self.pos.name:
name = self.pos.name
else:
name = self.prop
result = {
'populationType': self.pos.pop_type,
# mprop is used for top level reorgnize only
'mprop': self.pos.obs_props[0].mprop,
'l': text_format.format_title(name),
't': 'p',
'c': 0,
'cd': [],
'sv_set': set(),
}
return result
def json_blob(self, cpvs: Dict[str, str]):
""" Generate the json blob of a value node, including the node itself
and its leaf children. """
if self.leafs:
pop_type = self.leafs[0].pop_type
elif self.parent:
pop_type = self.parent.pop_type
else:
pop_type = ""
result = {
'populationType': pop_type,
'sv': [],
'l': text_format.format_title(self.value),
't': 'v',
'e': self.value,
'c': 0,
'sv_set': set([]),
'cd': [],
}
for node in self.leafs:
if node.addLevel:
result['cd'].append(node.json_blob(cpvs))
else:
# add the statsVar dcids to the value ndoe
result['sv'].extend(node.dcids)
result['sv_set'].update(node.dcids)
if not result['sv']:
result['t'] = 'p'
if result['cd']:
result['cd'] = text_format.filter_and_sort(self.value,
result['cd'])
if result['t'] == 'v':
result['d'] = []
if self.parent:
result['d'] = find_denominators(self.parent.pos, cpvs,
self.stats_vars_all,
result['sv'])
return result
def test_format_title(self, name, title, expected):
self.assertEqual(text_format.format_title(title), expected)
def build_tree_recursive(pos, level, pop_obs_spec, stat_vars, show_all,
parent=None):
"""Recursively build the ui tree"""
#get the property of the ui node
if parent:
property_diff = (set(pos.properties) - set(parent.pop_obs_spec.
properties)).pop()
parent_pv = parent.pv
else:
property_diff = pos.properties[0] # This is single pv pos
parent_pv = {}
prop_ui_node = util.UiNode(pos, parent_pv, True, property_diff)
result = {
'populationType': prop_ui_node.pop_type,
'show': 'yes',
'selected': 'no',
'expanded': 'no',
'title': text_format.format_title(prop_ui_node.text),
'type': 'Property',
'count': 0,
'children': [],
'sv_set': set(),
'search_count': 0,
'search_sv_set': set(),
}
#get the child specs of the current node
child_pos = []
for c_pos in pop_obs_spec[level+1]:
if (pos.pop_type == c_pos.pop_type and
set(pos.properties) < set(c_pos.properties)):
child_pos.append(c_pos)
for sv in stat_vars[pos.key]:
if sv.match_ui_node(prop_ui_node):
value_ui_pv = collections.OrderedDict()
for prop, val in parent_pv.items():
value_ui_pv[prop] = val
value_ui_pv[property_diff] = sv.pv[property_diff]
value_ui_node = util.UiNode(pos, value_ui_pv, False, property_diff)
in_search = True
for prop, val in value_ui_pv.items():
in_search = (in_search and (value_ui_node.pop_type,
value_ui_node.mprop, prop) in SEARCH_SPECS and val in SEARCH_VALS)
if not in_search:
break
value_blob = {
'populationType': value_ui_node.pop_type,
'show': 'yes',
'selected': 'no',
'expanded': 'no',
'argString': sv.dcid,
'title': text_format.format_title(value_ui_node.text),
'type': 'value',
'enum': value_ui_node.enum,
'count': 1,
'children': [],
'sv_set': set([sv.dcid]),
'search_count': 1 if in_search else 0,
'search_sv_set': set([sv.dcid]) if in_search else set(),
}
# add statistical variables as the child of current node
result['children'].append(value_blob)
if level <= MAX_LEVEL:
#build the branches recursively
for child in child_pos:
branch = build_tree_recursive(child, level + 1,
pop_obs_spec, stat_vars, show_all, value_ui_node)
if branch['children']:
value_blob['children'].append(branch)
value_blob['sv_set'] |= branch['sv_set']
value_blob['search_sv_set'] |= branch['search_sv_set']
del branch['sv_set']
del branch['search_sv_set']
value_blob['count'] = len(value_blob['sv_set'])
value_blob['search_count'] = len(value_blob['search_sv_set'])
result['children'] = text_format.filter_and_sort(property_diff,
result['children'], show_all)
#update the count
if result['children']:
for child in result['children']:
result['sv_set'] |= child['sv_set']
result['search_sv_set'] |= child['search_sv_set']
del child['sv_set']
del child['search_sv_set']
result['count'] = len(result['sv_set'])
result['search_count'] = len(result['search_sv_set'])
return result
def build_tree(v, pop_obs_spec, stat_vars, show_all):
"""Build the tree for each vertical."""
#vertical as the root
root = {
'show': 'yes',
'selected': 'no',
'expanded': 'no',
'argString': 'top',
'title': text_format.format_title(v),
'type': 'Property',
'count': 0, #count of child nodes
'children': [],
'sv_set': set(),#used for counting child nodes
'search_count': 0,
'search_sv_set': set(),
}
# specs with 0 constaints are of type "value",
# as the level 1 children of root
for pos in pop_obs_spec[0]:
search_count = (1 if (pos.pop_type, pos.mprop, '') in SEARCH_SPECS else 0)
ui_node = util.UiNode(pos, {}, False)
root['children'].append({
'populationType': ui_node.pop_type,
'show': 'yes',
'selected': 'no',
'expanded': 'no',
'argString': ui_node.arg_string,
'title': text_format.format_title(ui_node.text),
'type': 'value',
'children': [],
'count': 1,
'search_count': search_count,
})
root['count'] += 1
root['search_count'] += search_count
# build specs with >= 1 constraints recursively
for pos in pop_obs_spec[1]:
child = build_tree_recursive(pos, 1, pop_obs_spec, stat_vars, show_all)
# For certain branch, we would like to put them under 0 pv nodes:
if (pos.pop_type in ['EarthquakeEvent', 'CycloneEvent',
'MortalityEvent']):
for pv0 in root['children']:
# hoist logic will break if multiple 0 pv
if pv0['argString'] == '{},count'.format(pos.pop_type):
pv0['children'].append(child)
if 'sv_set' not in pv0:
pv0['sv_set'] = set()
pv0['search_sv_set'] = set()
pv0['sv_set'] |= child['sv_set']
pv0['search_sv_set'] |= child['search_sv_set']
break
else:
root['children'].append(child)
root['sv_set'] |= child['sv_set']
root['search_sv_set'] |= child['search_sv_set']
del child['sv_set']
del child['search_sv_set']
# update the count
for pv0 in root['children']:
if 'sv_set' in pv0:
pv0['count'] += len(pv0['sv_set'])
pv0['search_count'] += len(pv0['search_sv_set'])
del pv0['sv_set']
del pv0['search_sv_set']
root['count'] += len(root['sv_set'])
root['search_count'] += len(root['search_sv_set'])
del root['sv_set']
del root['search_sv_set']
return root
def build_tree(v, pop_obs_spec, stat_vars, place_mapping, show_all):
"""Build the tree for each vertical."""
#vertical as the root
root = {
'argString': 'top',
'title': text_format.format_title(v),
'type': 'prop',
'count': 0, #count of child nodes
'children': [],
'sv_set': set(), #used for counting child nodes
'search_count': 0,
'search_sv_set': set(),
'placeTypes': [],
}
# specs with 0 constaints are of type "value",
# as the level 1 children of root
for pos in pop_obs_spec[0]:
search_count = (1 if
(pos.pop_type, pos.mprop, '') in SEARCH_SPECS else 0)
ui_node = util.UiNode(pos, {}, False)
for sv in stat_vars[pos.key]:
if pos.cpv == sv.pv:
root['children'].append({
'populationType':
ui_node.pop_type,
'argString':
sv.dcid,
'title':
text_format.format_title(ui_node.text),
'type':
'val',
'children': [],
'count':
1,
'search_count':
search_count,
'mprop':
ui_node.mprop,
'placeTypes':
sorted(place_mapping[sv.dcid]),
})
root['placeTypes'] = sorted(
list(
set(root['placeTypes']) | set(place_mapping[sv.dcid])))
break # to avoid duplicates related to measurementMethod
root['count'] += 1
root['search_count'] += search_count
# build specs with >= 1 constraints recursively
for pos in pop_obs_spec[1]:
child = build_tree_recursive(pos, 1, pop_obs_spec, stat_vars,
place_mapping, show_all)
# For certain branch, we would like to put them under 0 pv nodes:
if (pos.pop_type
in ['EarthquakeEvent', 'CycloneEvent', 'MortalityEvent']):
for pv0 in root['children']:
# hoist logic will break if multiple 0 pv
if (pv0['populationType'] == pos.pop_type
and pv0['mprop'] == 'count'):
pv0['children'].append(child)
if 'sv_set' not in pv0:
pv0['sv_set'] = set()
pv0['search_sv_set'] = set()
pv0['sv_set'] |= child['sv_set']
pv0['search_sv_set'] |= child['search_sv_set']
break
else:
root['children'].append(child)
root['sv_set'] |= child['sv_set']
root['search_sv_set'] |= child['search_sv_set']
del child['sv_set']
del child['search_sv_set']
root['placeTypes'] = sorted(
list(set(root['placeTypes']) | set(child['placeTypes'])))
# update the count
for pv0 in root['children']:
if 'sv_set' in pv0:
pv0['count'] += len(pv0['sv_set'])
pv0['search_count'] += len(pv0['search_sv_set'])
del pv0['sv_set']
del pv0['search_sv_set']
root['count'] += len(root['sv_set'])
root['search_count'] += len(root['search_sv_set'])
del root['sv_set']
del root['search_sv_set']
return traverseTree(root)
def build_tree_recursive(pos, level, pop_obs_spec, stat_vars, parent=None):
"""Recursively build the ui tree"""
# get the property of the ui node
if parent:
property_diff = (set(pos.properties) -
set(parent.pop_obs_spec.properties)).pop()
parent_pv = parent.pv
else:
property_diff = pos.properties[0] # This is single pv pos
parent_pv = {}
prop_ui_node = util.UiNode(pos, parent_pv, True, property_diff)
result = {
'populationType': prop_ui_node.pop_type,
'l': text_format.format_title(prop_ui_node.text),
't': 'p',
'c': 0,
'cd': [],
'sv_set': set(),
}
# get the child specs of the current node
child_pos = []
for c_pos in pop_obs_spec[level + 1]:
if (pos.pop_type == c_pos.pop_type
and set(pos.properties) < set(c_pos.properties)):
child_pos.append(c_pos)
child_prop_vals = []
for sv in stat_vars[pos.key]:
if sv.match_ui_node(prop_ui_node) and sv.pv[property_diff]:
if sv.pv[property_diff] not in child_prop_vals:
# if this is the first statsvar of this node, build the node
# and corresponding child nodes
child_prop_vals.append(sv.pv[property_diff])
value_ui_pv = collections.OrderedDict()
for prop, val in parent_pv.items():
value_ui_pv[prop] = val
value_ui_pv[property_diff] = sv.pv[property_diff]
value_ui_node = util.UiNode(pos, value_ui_pv, False,
property_diff)
value_blob = {
'populationType': value_ui_node.pop_type,
'sv': [sv.dcid],
'l': text_format.format_title(value_ui_node.text),
't': 'v',
'e': value_ui_node.enum,
'c': 1,
'sv_set': set([sv.dcid]),
}
if sv.se:
value_blob['se'] = sv.se
# add statistical variables as the child of current node
result['cd'].append(value_blob)
if level <= MAX_LEVEL:
# build the branches recursively
for child in child_pos:
branch = build_tree_recursive(child, level + 1,
pop_obs_spec, stat_vars,
value_ui_node)
if branch['cd']:
if 'cd' not in value_blob:
value_blob['cd'] = []
value_blob['cd'].append(branch)
value_blob['sv_set'] |= branch['sv_set']
del branch['sv_set']
value_blob['c'] = len(value_blob['sv_set'])
else:
# if this is a duplicated statsVar
# append the statsVar to the right node, increase the count by 1
for child in result['cd']:
if child['e'] == sv.pv[property_diff]:
child['sv'].append(sv.dcid)
result['cd'] = text_format.filter_and_sort(property_diff, result['cd'],
False)
# update the count
if result['cd']:
for child in result['cd']:
result['sv_set'] |= child['sv_set']
del child['sv_set']
result['c'] = len(result['sv_set'])
result = group_super_enum(result)
return result
def build_tree(v, pop_obs_spec, stat_vars, vertical_idx):
"""Build the tree for each vertical."""
# vertical as the root
root = {
'sv': ['top'],
'l': text_format.format_title(v),
't': 'p',
'c': 0, # count of child nodes
'cd': [],
'sv_set': set(), # used for counting child nodes
}
# specs with 0 constaints are of type "value",
# as the level 1 cd of root
for pos in pop_obs_spec[0]:
ui_node = util.UiNode(pos, {}, False)
childStatsVars = []
# find all the statsvars belong to the node
for sv in stat_vars[pos.key]:
if pos.cpv == sv.pv:
childStatsVars.append(sv.dcid)
root['c'] += 1
if len(childStatsVars) > 0:
root['cd'].append({
'populationType': ui_node.pop_type,
'sv': childStatsVars,
'l': text_format.format_title(ui_node.text),
't': 'v',
'c': len(childStatsVars),
'mprop': ui_node.mprop,
})
# build specs with >= 1 constraints recursively
for pos in pop_obs_spec[1]:
child = build_tree_recursive(
pos,
1,
pop_obs_spec,
stat_vars,
)
# For certain branch, we would like to put them under 0 pv nodes:
if (pos.pop_type
in ['EarthquakeEvent', 'CycloneEvent', 'MortalityEvent']):
for pv0 in root['cd']:
# hoist logic will break if multiple 0 pv
if (pv0['populationType'] == pos.pop_type
and pv0['mprop'] == 'count'):
if 'cd' not in pv0:
pv0['cd'] = []
pv0['cd'].append(child)
if 'sv_set' not in pv0:
pv0['sv_set'] = set()
pv0['sv_set'] |= child['sv_set']
break
else:
root['cd'].append(child)
root['sv_set'] |= child['sv_set']
del child['sv_set']
# update the count
for pv0 in root['cd']:
if 'sv_set' in pv0:
pv0['c'] += len(pv0['sv_set'])
del pv0['sv_set']
root['c'] += len(root['sv_set'])
del root['sv_set']
statsvar_path = {}
return traverseTree(root, [vertical_idx], statsvar_path)