def group_super_enum(json_blob): """Group the nodes by super enum.""" new_children = [] grouping = collections.defaultdict(list) direct_enum = set() for child in json_blob['cd']: if 'se' in child: # Now only group by one super enum. assert len(child['se']) == 1 v = list(child['se'].values())[0] grouping[v].append(child) else: new_children.append(child) direct_enum.add(child['e']) # Add a layer for super enum for v, children in grouping.items(): if v in direct_enum: for child in new_children: if child['e'] == v: child['cd'] = children child['c'] += len(children) break else: direct_enum.add(v) enum_blob = children[0].copy() # Shallow copy enum_blob['cd'] = children enum_blob['l'] = text_format.format_title(v) enum_blob['c'] = len(children) enum_blob['t'] = 'p' del enum_blob['sv'] new_children.append(enum_blob) json_blob['cd'] = new_children return json_blob
def json_blob(self, cpvs: Dict[str, str]): result = { 'populationType': self.pop_type, 'sv': self.dcids, 'l': text_format.format_title(self.name), 't': 'v', 'e': self.name, 'c': 1, 'cd': [], 'sv_set': set(self.dcids), } for node in self.leafs: if node.addLevel: result['cd'].append(node.json_blob(cpvs)) else: # add the statsVar dcids to the value ndoe result['sv'].extend(node.dcids) result['sv_set'].update(node.dcids) if not result['sv']: result['t'] = 'p' if result['cd']: result['cd'] = text_format.filter_and_sort(self.name, result['cd']) if result['t'] == 'v': result['d'] = find_denominators(self.pos, cpvs, self.stats_vars_all, result['sv']) return result
def json_blob(self): """ Generate the json blob for the property node""" if self.pos.name: name = self.pos.name else: name = self.prop result = { 'populationType': self.pos.pop_type, # mprop is used for top level reorgnize only 'mprop': self.pos.obs_props[0].mprop, 'l': text_format.format_title(name), 't': 'p', 'c': 0, 'cd': [], 'sv_set': set(), } return result
def json_blob(self, cpvs: Dict[str, str]): """ Generate the json blob of a value node, including the node itself and its leaf children. """ if self.leafs: pop_type = self.leafs[0].pop_type elif self.parent: pop_type = self.parent.pop_type else: pop_type = "" result = { 'populationType': pop_type, 'sv': [], 'l': text_format.format_title(self.value), 't': 'v', 'e': self.value, 'c': 0, 'sv_set': set([]), 'cd': [], } for node in self.leafs: if node.addLevel: result['cd'].append(node.json_blob(cpvs)) else: # add the statsVar dcids to the value ndoe result['sv'].extend(node.dcids) result['sv_set'].update(node.dcids) if not result['sv']: result['t'] = 'p' if result['cd']: result['cd'] = text_format.filter_and_sort(self.value, result['cd']) if result['t'] == 'v': result['d'] = [] if self.parent: result['d'] = find_denominators(self.parent.pos, cpvs, self.stats_vars_all, result['sv']) return result
def test_format_title(self, name, title, expected): self.assertEqual(text_format.format_title(title), expected)
def build_tree_recursive(pos, level, pop_obs_spec, stat_vars, show_all, parent=None): """Recursively build the ui tree""" #get the property of the ui node if parent: property_diff = (set(pos.properties) - set(parent.pop_obs_spec. properties)).pop() parent_pv = parent.pv else: property_diff = pos.properties[0] # This is single pv pos parent_pv = {} prop_ui_node = util.UiNode(pos, parent_pv, True, property_diff) result = { 'populationType': prop_ui_node.pop_type, 'show': 'yes', 'selected': 'no', 'expanded': 'no', 'title': text_format.format_title(prop_ui_node.text), 'type': 'Property', 'count': 0, 'children': [], 'sv_set': set(), 'search_count': 0, 'search_sv_set': set(), } #get the child specs of the current node child_pos = [] for c_pos in pop_obs_spec[level+1]: if (pos.pop_type == c_pos.pop_type and set(pos.properties) < set(c_pos.properties)): child_pos.append(c_pos) for sv in stat_vars[pos.key]: if sv.match_ui_node(prop_ui_node): value_ui_pv = collections.OrderedDict() for prop, val in parent_pv.items(): value_ui_pv[prop] = val value_ui_pv[property_diff] = sv.pv[property_diff] value_ui_node = util.UiNode(pos, value_ui_pv, False, property_diff) in_search = True for prop, val in value_ui_pv.items(): in_search = (in_search and (value_ui_node.pop_type, value_ui_node.mprop, prop) in SEARCH_SPECS and val in SEARCH_VALS) if not in_search: break value_blob = { 'populationType': value_ui_node.pop_type, 'show': 'yes', 'selected': 'no', 'expanded': 'no', 'argString': sv.dcid, 'title': text_format.format_title(value_ui_node.text), 'type': 'value', 'enum': value_ui_node.enum, 'count': 1, 'children': [], 'sv_set': set([sv.dcid]), 'search_count': 1 if in_search else 0, 'search_sv_set': set([sv.dcid]) if in_search else set(), } # add statistical variables as the child of current node result['children'].append(value_blob) if level <= MAX_LEVEL: #build the branches recursively for child in child_pos: branch = build_tree_recursive(child, level + 1, pop_obs_spec, stat_vars, show_all, value_ui_node) if branch['children']: value_blob['children'].append(branch) value_blob['sv_set'] |= branch['sv_set'] value_blob['search_sv_set'] |= branch['search_sv_set'] del branch['sv_set'] del branch['search_sv_set'] value_blob['count'] = len(value_blob['sv_set']) value_blob['search_count'] = len(value_blob['search_sv_set']) result['children'] = text_format.filter_and_sort(property_diff, result['children'], show_all) #update the count if result['children']: for child in result['children']: result['sv_set'] |= child['sv_set'] result['search_sv_set'] |= child['search_sv_set'] del child['sv_set'] del child['search_sv_set'] result['count'] = len(result['sv_set']) result['search_count'] = len(result['search_sv_set']) return result
def build_tree(v, pop_obs_spec, stat_vars, show_all): """Build the tree for each vertical.""" #vertical as the root root = { 'show': 'yes', 'selected': 'no', 'expanded': 'no', 'argString': 'top', 'title': text_format.format_title(v), 'type': 'Property', 'count': 0, #count of child nodes 'children': [], 'sv_set': set(),#used for counting child nodes 'search_count': 0, 'search_sv_set': set(), } # specs with 0 constaints are of type "value", # as the level 1 children of root for pos in pop_obs_spec[0]: search_count = (1 if (pos.pop_type, pos.mprop, '') in SEARCH_SPECS else 0) ui_node = util.UiNode(pos, {}, False) root['children'].append({ 'populationType': ui_node.pop_type, 'show': 'yes', 'selected': 'no', 'expanded': 'no', 'argString': ui_node.arg_string, 'title': text_format.format_title(ui_node.text), 'type': 'value', 'children': [], 'count': 1, 'search_count': search_count, }) root['count'] += 1 root['search_count'] += search_count # build specs with >= 1 constraints recursively for pos in pop_obs_spec[1]: child = build_tree_recursive(pos, 1, pop_obs_spec, stat_vars, show_all) # For certain branch, we would like to put them under 0 pv nodes: if (pos.pop_type in ['EarthquakeEvent', 'CycloneEvent', 'MortalityEvent']): for pv0 in root['children']: # hoist logic will break if multiple 0 pv if pv0['argString'] == '{},count'.format(pos.pop_type): pv0['children'].append(child) if 'sv_set' not in pv0: pv0['sv_set'] = set() pv0['search_sv_set'] = set() pv0['sv_set'] |= child['sv_set'] pv0['search_sv_set'] |= child['search_sv_set'] break else: root['children'].append(child) root['sv_set'] |= child['sv_set'] root['search_sv_set'] |= child['search_sv_set'] del child['sv_set'] del child['search_sv_set'] # update the count for pv0 in root['children']: if 'sv_set' in pv0: pv0['count'] += len(pv0['sv_set']) pv0['search_count'] += len(pv0['search_sv_set']) del pv0['sv_set'] del pv0['search_sv_set'] root['count'] += len(root['sv_set']) root['search_count'] += len(root['search_sv_set']) del root['sv_set'] del root['search_sv_set'] return root
def build_tree(v, pop_obs_spec, stat_vars, place_mapping, show_all): """Build the tree for each vertical.""" #vertical as the root root = { 'argString': 'top', 'title': text_format.format_title(v), 'type': 'prop', 'count': 0, #count of child nodes 'children': [], 'sv_set': set(), #used for counting child nodes 'search_count': 0, 'search_sv_set': set(), 'placeTypes': [], } # specs with 0 constaints are of type "value", # as the level 1 children of root for pos in pop_obs_spec[0]: search_count = (1 if (pos.pop_type, pos.mprop, '') in SEARCH_SPECS else 0) ui_node = util.UiNode(pos, {}, False) for sv in stat_vars[pos.key]: if pos.cpv == sv.pv: root['children'].append({ 'populationType': ui_node.pop_type, 'argString': sv.dcid, 'title': text_format.format_title(ui_node.text), 'type': 'val', 'children': [], 'count': 1, 'search_count': search_count, 'mprop': ui_node.mprop, 'placeTypes': sorted(place_mapping[sv.dcid]), }) root['placeTypes'] = sorted( list( set(root['placeTypes']) | set(place_mapping[sv.dcid]))) break # to avoid duplicates related to measurementMethod root['count'] += 1 root['search_count'] += search_count # build specs with >= 1 constraints recursively for pos in pop_obs_spec[1]: child = build_tree_recursive(pos, 1, pop_obs_spec, stat_vars, place_mapping, show_all) # For certain branch, we would like to put them under 0 pv nodes: if (pos.pop_type in ['EarthquakeEvent', 'CycloneEvent', 'MortalityEvent']): for pv0 in root['children']: # hoist logic will break if multiple 0 pv if (pv0['populationType'] == pos.pop_type and pv0['mprop'] == 'count'): pv0['children'].append(child) if 'sv_set' not in pv0: pv0['sv_set'] = set() pv0['search_sv_set'] = set() pv0['sv_set'] |= child['sv_set'] pv0['search_sv_set'] |= child['search_sv_set'] break else: root['children'].append(child) root['sv_set'] |= child['sv_set'] root['search_sv_set'] |= child['search_sv_set'] del child['sv_set'] del child['search_sv_set'] root['placeTypes'] = sorted( list(set(root['placeTypes']) | set(child['placeTypes']))) # update the count for pv0 in root['children']: if 'sv_set' in pv0: pv0['count'] += len(pv0['sv_set']) pv0['search_count'] += len(pv0['search_sv_set']) del pv0['sv_set'] del pv0['search_sv_set'] root['count'] += len(root['sv_set']) root['search_count'] += len(root['search_sv_set']) del root['sv_set'] del root['search_sv_set'] return traverseTree(root)
def build_tree_recursive(pos, level, pop_obs_spec, stat_vars, parent=None): """Recursively build the ui tree""" # get the property of the ui node if parent: property_diff = (set(pos.properties) - set(parent.pop_obs_spec.properties)).pop() parent_pv = parent.pv else: property_diff = pos.properties[0] # This is single pv pos parent_pv = {} prop_ui_node = util.UiNode(pos, parent_pv, True, property_diff) result = { 'populationType': prop_ui_node.pop_type, 'l': text_format.format_title(prop_ui_node.text), 't': 'p', 'c': 0, 'cd': [], 'sv_set': set(), } # get the child specs of the current node child_pos = [] for c_pos in pop_obs_spec[level + 1]: if (pos.pop_type == c_pos.pop_type and set(pos.properties) < set(c_pos.properties)): child_pos.append(c_pos) child_prop_vals = [] for sv in stat_vars[pos.key]: if sv.match_ui_node(prop_ui_node) and sv.pv[property_diff]: if sv.pv[property_diff] not in child_prop_vals: # if this is the first statsvar of this node, build the node # and corresponding child nodes child_prop_vals.append(sv.pv[property_diff]) value_ui_pv = collections.OrderedDict() for prop, val in parent_pv.items(): value_ui_pv[prop] = val value_ui_pv[property_diff] = sv.pv[property_diff] value_ui_node = util.UiNode(pos, value_ui_pv, False, property_diff) value_blob = { 'populationType': value_ui_node.pop_type, 'sv': [sv.dcid], 'l': text_format.format_title(value_ui_node.text), 't': 'v', 'e': value_ui_node.enum, 'c': 1, 'sv_set': set([sv.dcid]), } if sv.se: value_blob['se'] = sv.se # add statistical variables as the child of current node result['cd'].append(value_blob) if level <= MAX_LEVEL: # build the branches recursively for child in child_pos: branch = build_tree_recursive(child, level + 1, pop_obs_spec, stat_vars, value_ui_node) if branch['cd']: if 'cd' not in value_blob: value_blob['cd'] = [] value_blob['cd'].append(branch) value_blob['sv_set'] |= branch['sv_set'] del branch['sv_set'] value_blob['c'] = len(value_blob['sv_set']) else: # if this is a duplicated statsVar # append the statsVar to the right node, increase the count by 1 for child in result['cd']: if child['e'] == sv.pv[property_diff]: child['sv'].append(sv.dcid) result['cd'] = text_format.filter_and_sort(property_diff, result['cd'], False) # update the count if result['cd']: for child in result['cd']: result['sv_set'] |= child['sv_set'] del child['sv_set'] result['c'] = len(result['sv_set']) result = group_super_enum(result) return result
def build_tree(v, pop_obs_spec, stat_vars, vertical_idx): """Build the tree for each vertical.""" # vertical as the root root = { 'sv': ['top'], 'l': text_format.format_title(v), 't': 'p', 'c': 0, # count of child nodes 'cd': [], 'sv_set': set(), # used for counting child nodes } # specs with 0 constaints are of type "value", # as the level 1 cd of root for pos in pop_obs_spec[0]: ui_node = util.UiNode(pos, {}, False) childStatsVars = [] # find all the statsvars belong to the node for sv in stat_vars[pos.key]: if pos.cpv == sv.pv: childStatsVars.append(sv.dcid) root['c'] += 1 if len(childStatsVars) > 0: root['cd'].append({ 'populationType': ui_node.pop_type, 'sv': childStatsVars, 'l': text_format.format_title(ui_node.text), 't': 'v', 'c': len(childStatsVars), 'mprop': ui_node.mprop, }) # build specs with >= 1 constraints recursively for pos in pop_obs_spec[1]: child = build_tree_recursive( pos, 1, pop_obs_spec, stat_vars, ) # For certain branch, we would like to put them under 0 pv nodes: if (pos.pop_type in ['EarthquakeEvent', 'CycloneEvent', 'MortalityEvent']): for pv0 in root['cd']: # hoist logic will break if multiple 0 pv if (pv0['populationType'] == pos.pop_type and pv0['mprop'] == 'count'): if 'cd' not in pv0: pv0['cd'] = [] pv0['cd'].append(child) if 'sv_set' not in pv0: pv0['sv_set'] = set() pv0['sv_set'] |= child['sv_set'] break else: root['cd'].append(child) root['sv_set'] |= child['sv_set'] del child['sv_set'] # update the count for pv0 in root['cd']: if 'sv_set' in pv0: pv0['c'] += len(pv0['sv_set']) del pv0['sv_set'] root['c'] += len(root['sv_set']) del root['sv_set'] statsvar_path = {} return traverseTree(root, [vertical_idx], statsvar_path)