def get_direct_children(self):
"""
Parse etlMapping file and return a list of direct children from the root
"""
children = self.mapping["flatten_props"]
nodes = []
bypass = self.mapping.get("settings",
{}).get("bypass_multiplicity_check")
for child in children:
child_label, edge = get_edge_table(self.model, self.root,
child["path"])
child_name, is_child = get_child_table(self.model, self.root,
child["path"])
multiplicity = (get_multiplicity(self.dictionary, self.root,
child_label)
if is_child else get_multiplicity(
self.dictionary, child_label, self.root))
sorted_by, desc_order = self.parse_sorting(child)
if (not bypass and sorted_by is None
and multiplicity != "one_to_one"
and multiplicity != "one_to_many"):
raise Exception(
"something bad has just happened\n"
"the properties '{}' for '{}'\n"
"for parent '{}'\n"
"has multiplicity '{}' that cannot be used on in 'flatten_props'"
"\n".format(child["props"], child["path"], child_label,
multiplicity))
props = self.create_props_from_json(self.doc_type,
child["props"],
node_label=child_label)
nodes.append(
DirectNode(child_name, edge, props, sorted_by, desc_order,
is_child))
return nodes
def json_to_parent_node(self, path):
words = path.split(".")
nodes = [
tuple([_f for _f in re.split("[\[\]]", w) if _f]) for w in words
]
first = None
prev = None
prev_label = self.root
for nd in nodes:
n = nd[0]
p = nd[1] if len(nd) > 1 else None
parent_name, edge_tbl = get_edge_table(self.model, prev_label, n)
parent_tbl = get_node_table_name(self.model, parent_name)
if p is not None:
json_props = [{
"name": p[0],
"src": p[1]
} for p in self.get_src_name(p.split(","))]
props = self.create_props_from_json(self.doc_type,
json_props,
node_label=parent_name)
else:
props = []
cur = ParentNode(parent_name, parent_tbl, edge_tbl, props)
if prev is not None:
prev.child = cur
else:
first = cur
prev_label = parent_name
prev = cur
return first
def json_to_special_node(self, path):
"""
Create node in the path of special aggregation
:param path: path define the node and the prop to be aggregated
:return:
"""
words = path.split(".")
nodes = [
tuple([_f for _f in re.split("[\[\]]", w) if _f]) for w in words
]
first = None
prev = None
prev_label = self.root
for (n, str_p) in nodes:
child_name, edge_tbl = get_edge_table(self.model, prev_label, n)
child_tbl = get_node_table_name(self.model, child_name)
json_props = [{"name": p, "src": p} for p in str_p.split(",")]
props = self.create_props_from_json(self.doc_type,
json_props,
node_label=child_name)
cur = SpecialNode(child_name, child_tbl, edge_tbl, props)
if prev is not None:
prev.child = cur
else:
first = cur
prev_label = child_name
prev = cur
return first
def json_to_parent_node(self, path):
words = path.split('.')
nodes = [tuple(filter(None, re.split('[\[\]]', w))) for w in words]
first = None
prev = None
prev_label = self.root
for nd in nodes:
n = nd[0]
p = nd[1] if len(nd) > 1 else None
parent_name, edge_tbl = get_edge_table(self.model, prev_label, n)
parent_tbl = get_node_table_name(self.model, parent_name)
if p is not None:
json_props = [{
'name': p[0],
'src': p[1]
} for p in self.get_src_name(p.split(','))]
props = self.create_props_from_json(self.doc_type,
json_props,
node_label=parent_name)
else:
props = []
cur = ParentNode(parent_name, parent_tbl, edge_tbl, props)
if prev is not None:
prev.child = cur
else:
first = cur
prev_label = parent_name
prev = cur
return first
def add_collecting_node(self, child, collectors, fst):
parent_name = get_node_label(
self.model, get_parent_name(self.model, child.name, fst))
_, edge_up_tbl = get_edge_table(self.model, child.name, fst)
collecting_node = collectors[parent_name] if parent_name in collectors \
else CollectingNode(parent_name)
collecting_node.add_child(child)
child.add_parent(collecting_node.name, edge_up_tbl)
collectors[parent_name] = collecting_node
return collecting_node
def construct_auth_path_tree(self, flat_paths):
collectors = {}
root = self.create_auth_path_root()
for p in flat_paths:
segments = list(p.path)
_, edge_up_tbl = get_edge_table(self.model, p.src, segments[0])
if p.src not in collectors:
collectors[p.src] = CollectingNode(p.src, edge_up_tbl)
child = collectors[p.src]
if len(segments) > 1:
for node in segments[0:len(segments) - 2]:
child = self.add_collecting_node(child, collectors, node)
_, edge_up_tbl = get_edge_table(self.model, child.name,
segments[-2])
elif len(segments) == 1:
_, edge_up_tbl = get_edge_table(self.model, child.name,
segments[-1])
root.add_child(child)
child.add_parent('auth_path_root', edge_up_tbl)
return collectors.values(), root
def create_tree_from_generated_edges(self, flat_paths, nodes_with_props,
roots):
collectors = nodes_with_props
checking_set = set(self.generated_edges)
for p in flat_paths:
segments = list(p.path)
_, edge_up_tbl = get_edge_table(self.model, p.src, segments[0])
if edge_up_tbl not in checking_set:
continue
if p.src not in collectors:
tbl_name = get_node_table_name(self.model, p.src)
collectors[p.src] = CollectingNode(p.src, tbl_name)
child = collectors[p.src]
if len(segments) > 1:
for fst in segments[0:len(segments) - 1]:
_, edge_up_tbl = get_edge_table(self.model, p.src,
segments[0])
if edge_up_tbl not in checking_set:
break
child = self.add_collecting_node(child, collectors, fst)
self.add_root_node(child, roots, segments[-1])
return list(collectors.values()), list(roots.values())
def construct_aggregation_tree(self, flat_paths):
reversed_index = {}
list_nodes = []
for path in flat_paths:
n_name = self.mapping["root"]
current_parent_edge = None
level = 0
for i, p in enumerate(path.path):
if (n_name, current_parent_edge) in reversed_index:
n_current = list_nodes[reversed_index[(
n_name, current_parent_edge)]]
else:
n_current = AggregatedNode(
n_name,
get_node_table_name(self.model, n_name),
current_parent_edge,
level,
)
list_nodes.append(n_current)
reversed_index[(n_name,
current_parent_edge)] = len(list_nodes) - 1
child_name, edge_tbl = get_edge_table(self.model, n_name, p)
n_child = (list_nodes[reversed_index[(child_name,
edge_tbl)]] if
(child_name,
edge_tbl) in reversed_index else AggregatedNode(
child_name,
get_node_table_name(self.model, child_name),
edge_tbl,
level + 1,
))
n_child.parent = n_current
if i == len(path.path) - 1:
for reducer in path.reducers:
prop = self.create_prop_from_json(
self.doc_type, reducer, None)
n_child.reducers.append(Reducer(prop, reducer["fn"]))
n_current.add_child(n_child)
if (child_name, edge_tbl) not in reversed_index:
list_nodes.append(n_child)
reversed_index[(child_name,
edge_tbl)] = len(list_nodes) - 1
n_name = child_name
current_parent_edge = edge_tbl
level += 1
return list_nodes, Parser.get_leaves(list_nodes)
def get_table_list_from_path(self, p, root, path):
r = []
splitted_path = path.split(".") if path else []
node = get_node_table_name(p.model, root)
r.append(node)
for i in splitted_path:
root, node = get_edge_table(p.model, root, i)
r.append(node)
node = get_node_table_name(p.model, root)
r.append(node)
return r
def construct_reversed_collection_tree(self, flat_paths):
collectors = {}
roots = {}
for p in flat_paths:
segments = list(p.path)
_, edge_up_tbl = get_edge_table(self.model, p.src, segments[0])
if p.src not in collectors:
collectors[p.src] = CollectingNode(p.src, edge_up_tbl)
child = collectors[p.src]
if len(segments) > 1:
for fst in segments[0:len(segments) - 1]:
child = self.add_collecting_node(child, collectors, fst)
self.add_root_node(child, roots, segments[-1])
return collectors.values(), roots.values()
def add_root_node(self, child, roots, segment):
root_name = get_node_label(
self.model, get_parent_name(self.model, child.name, segment))
_, edge_up_tbl = get_edge_table(self.model, child.name, segment)
root_tbl_name = get_node_table_name(
self.model, get_parent_label(self.model, child.name, segment))
top_node = roots[root_name] if root_name in roots \
else RootNode(root_name, root_tbl_name,
self.create_props_from_json(self.doc_type,
self.mapping['injecting_props'][root_name]['props'],
node_label=root_name))
child.add_parent(top_node.name, edge_up_tbl)
top_node.add_child(child)
roots[root_name] = top_node
def create_auth_path_root(self):
program_table_name = get_node_table_name(self.model, 'program')
project_table_name = get_node_table_name(self.model, 'project')
_, edge_up_tbl = get_edge_table(self.model, 'project', 'programs')
root_program = RootNode(
'auth_path_root', program_table_name,
self.create_props_from_json(self.doc_type, [{
'name': 'program_name',
'src': 'name'
}],
node_label='program'))
root_project = RootNode(
'project', project_table_name,
self.create_props_from_json(self.doc_type, [{
'name': 'project_code',
'src': 'code'
}],
node_label='project'), edge_up_tbl)
root_program.root_child = root_project
return root_program
def add_root_node(self, child, roots, segment):
root_name = get_node_label(
self.model, get_parent_name(self.model, child.name, segment))
_, edge_up_tbl = get_edge_table(self.model, child.name, segment)
root_tbl_name = get_node_table_name(
self.model, get_parent_label(self.model, child.name, segment))
top_node = (roots[root_name] if root_name in roots else RootNode(
root_name,
root_tbl_name,
self.create_props_from_json(
self.doc_type,
[{
"name": "program_name",
"src": "name"
}],
node_label=root_name,
is_additional=True,
),
))
child.add_parent(top_node.name, edge_up_tbl)
top_node.add_child(child)
roots[root_name] = top_node
