def get_direct_children(self): """ Parse etlMapping file and return a list of direct children from the root """ children = self.mapping["flatten_props"] nodes = [] bypass = self.mapping.get("settings", {}).get("bypass_multiplicity_check") for child in children: child_label, edge = get_edge_table(self.model, self.root, child["path"]) child_name, is_child = get_child_table(self.model, self.root, child["path"]) multiplicity = (get_multiplicity(self.dictionary, self.root, child_label) if is_child else get_multiplicity( self.dictionary, child_label, self.root)) sorted_by, desc_order = self.parse_sorting(child) if (not bypass and sorted_by is None and multiplicity != "one_to_one" and multiplicity != "one_to_many"): raise Exception( "something bad has just happened\n" "the properties '{}' for '{}'\n" "for parent '{}'\n" "has multiplicity '{}' that cannot be used on in 'flatten_props'" "\n".format(child["props"], child["path"], child_label, multiplicity)) props = self.create_props_from_json(self.doc_type, child["props"], node_label=child_label) nodes.append( DirectNode(child_name, edge, props, sorted_by, desc_order, is_child)) return nodes
def json_to_parent_node(self, path): words = path.split(".") nodes = [ tuple([_f for _f in re.split("[\[\]]", w) if _f]) for w in words ] first = None prev = None prev_label = self.root for nd in nodes: n = nd[0] p = nd[1] if len(nd) > 1 else None parent_name, edge_tbl = get_edge_table(self.model, prev_label, n) parent_tbl = get_node_table_name(self.model, parent_name) if p is not None: json_props = [{ "name": p[0], "src": p[1] } for p in self.get_src_name(p.split(","))] props = self.create_props_from_json(self.doc_type, json_props, node_label=parent_name) else: props = [] cur = ParentNode(parent_name, parent_tbl, edge_tbl, props) if prev is not None: prev.child = cur else: first = cur prev_label = parent_name prev = cur return first
def json_to_special_node(self, path): """ Create node in the path of special aggregation :param path: path define the node and the prop to be aggregated :return: """ words = path.split(".") nodes = [ tuple([_f for _f in re.split("[\[\]]", w) if _f]) for w in words ] first = None prev = None prev_label = self.root for (n, str_p) in nodes: child_name, edge_tbl = get_edge_table(self.model, prev_label, n) child_tbl = get_node_table_name(self.model, child_name) json_props = [{"name": p, "src": p} for p in str_p.split(",")] props = self.create_props_from_json(self.doc_type, json_props, node_label=child_name) cur = SpecialNode(child_name, child_tbl, edge_tbl, props) if prev is not None: prev.child = cur else: first = cur prev_label = child_name prev = cur return first
def json_to_parent_node(self, path): words = path.split('.') nodes = [tuple(filter(None, re.split('[\[\]]', w))) for w in words] first = None prev = None prev_label = self.root for nd in nodes: n = nd[0] p = nd[1] if len(nd) > 1 else None parent_name, edge_tbl = get_edge_table(self.model, prev_label, n) parent_tbl = get_node_table_name(self.model, parent_name) if p is not None: json_props = [{ 'name': p[0], 'src': p[1] } for p in self.get_src_name(p.split(','))] props = self.create_props_from_json(self.doc_type, json_props, node_label=parent_name) else: props = [] cur = ParentNode(parent_name, parent_tbl, edge_tbl, props) if prev is not None: prev.child = cur else: first = cur prev_label = parent_name prev = cur return first
def add_collecting_node(self, child, collectors, fst): parent_name = get_node_label( self.model, get_parent_name(self.model, child.name, fst)) _, edge_up_tbl = get_edge_table(self.model, child.name, fst) collecting_node = collectors[parent_name] if parent_name in collectors \ else CollectingNode(parent_name) collecting_node.add_child(child) child.add_parent(collecting_node.name, edge_up_tbl) collectors[parent_name] = collecting_node return collecting_node
def construct_auth_path_tree(self, flat_paths): collectors = {} root = self.create_auth_path_root() for p in flat_paths: segments = list(p.path) _, edge_up_tbl = get_edge_table(self.model, p.src, segments[0]) if p.src not in collectors: collectors[p.src] = CollectingNode(p.src, edge_up_tbl) child = collectors[p.src] if len(segments) > 1: for node in segments[0:len(segments) - 2]: child = self.add_collecting_node(child, collectors, node) _, edge_up_tbl = get_edge_table(self.model, child.name, segments[-2]) elif len(segments) == 1: _, edge_up_tbl = get_edge_table(self.model, child.name, segments[-1]) root.add_child(child) child.add_parent('auth_path_root', edge_up_tbl) return collectors.values(), root
def create_tree_from_generated_edges(self, flat_paths, nodes_with_props, roots): collectors = nodes_with_props checking_set = set(self.generated_edges) for p in flat_paths: segments = list(p.path) _, edge_up_tbl = get_edge_table(self.model, p.src, segments[0]) if edge_up_tbl not in checking_set: continue if p.src not in collectors: tbl_name = get_node_table_name(self.model, p.src) collectors[p.src] = CollectingNode(p.src, tbl_name) child = collectors[p.src] if len(segments) > 1: for fst in segments[0:len(segments) - 1]: _, edge_up_tbl = get_edge_table(self.model, p.src, segments[0]) if edge_up_tbl not in checking_set: break child = self.add_collecting_node(child, collectors, fst) self.add_root_node(child, roots, segments[-1]) return list(collectors.values()), list(roots.values())
def construct_aggregation_tree(self, flat_paths): reversed_index = {} list_nodes = [] for path in flat_paths: n_name = self.mapping["root"] current_parent_edge = None level = 0 for i, p in enumerate(path.path): if (n_name, current_parent_edge) in reversed_index: n_current = list_nodes[reversed_index[( n_name, current_parent_edge)]] else: n_current = AggregatedNode( n_name, get_node_table_name(self.model, n_name), current_parent_edge, level, ) list_nodes.append(n_current) reversed_index[(n_name, current_parent_edge)] = len(list_nodes) - 1 child_name, edge_tbl = get_edge_table(self.model, n_name, p) n_child = (list_nodes[reversed_index[(child_name, edge_tbl)]] if (child_name, edge_tbl) in reversed_index else AggregatedNode( child_name, get_node_table_name(self.model, child_name), edge_tbl, level + 1, )) n_child.parent = n_current if i == len(path.path) - 1: for reducer in path.reducers: prop = self.create_prop_from_json( self.doc_type, reducer, None) n_child.reducers.append(Reducer(prop, reducer["fn"])) n_current.add_child(n_child) if (child_name, edge_tbl) not in reversed_index: list_nodes.append(n_child) reversed_index[(child_name, edge_tbl)] = len(list_nodes) - 1 n_name = child_name current_parent_edge = edge_tbl level += 1 return list_nodes, Parser.get_leaves(list_nodes)
def get_table_list_from_path(self, p, root, path): r = [] splitted_path = path.split(".") if path else [] node = get_node_table_name(p.model, root) r.append(node) for i in splitted_path: root, node = get_edge_table(p.model, root, i) r.append(node) node = get_node_table_name(p.model, root) r.append(node) return r
def construct_reversed_collection_tree(self, flat_paths): collectors = {} roots = {} for p in flat_paths: segments = list(p.path) _, edge_up_tbl = get_edge_table(self.model, p.src, segments[0]) if p.src not in collectors: collectors[p.src] = CollectingNode(p.src, edge_up_tbl) child = collectors[p.src] if len(segments) > 1: for fst in segments[0:len(segments) - 1]: child = self.add_collecting_node(child, collectors, fst) self.add_root_node(child, roots, segments[-1]) return collectors.values(), roots.values()
def add_root_node(self, child, roots, segment): root_name = get_node_label( self.model, get_parent_name(self.model, child.name, segment)) _, edge_up_tbl = get_edge_table(self.model, child.name, segment) root_tbl_name = get_node_table_name( self.model, get_parent_label(self.model, child.name, segment)) top_node = roots[root_name] if root_name in roots \ else RootNode(root_name, root_tbl_name, self.create_props_from_json(self.doc_type, self.mapping['injecting_props'][root_name]['props'], node_label=root_name)) child.add_parent(top_node.name, edge_up_tbl) top_node.add_child(child) roots[root_name] = top_node
def create_auth_path_root(self): program_table_name = get_node_table_name(self.model, 'program') project_table_name = get_node_table_name(self.model, 'project') _, edge_up_tbl = get_edge_table(self.model, 'project', 'programs') root_program = RootNode( 'auth_path_root', program_table_name, self.create_props_from_json(self.doc_type, [{ 'name': 'program_name', 'src': 'name' }], node_label='program')) root_project = RootNode( 'project', project_table_name, self.create_props_from_json(self.doc_type, [{ 'name': 'project_code', 'src': 'code' }], node_label='project'), edge_up_tbl) root_program.root_child = root_project return root_program
def add_root_node(self, child, roots, segment): root_name = get_node_label( self.model, get_parent_name(self.model, child.name, segment)) _, edge_up_tbl = get_edge_table(self.model, child.name, segment) root_tbl_name = get_node_table_name( self.model, get_parent_label(self.model, child.name, segment)) top_node = (roots[root_name] if root_name in roots else RootNode( root_name, root_tbl_name, self.create_props_from_json( self.doc_type, [{ "name": "program_name", "src": "name" }], node_label=root_name, is_additional=True, ), )) child.add_parent(top_node.name, edge_up_tbl) top_node.add_child(child) roots[root_name] = top_node