def get_tree_direct_children(tree): """ Get only the direct children of a given anytree Node Parameters ---------- tree : Node Returns ------- List[Node] Child-less nodes first, then sorted by name. """ childless = [[node for node in children] for children in LevelOrderGroupIter( tree, filter_=lambda n: not n.children)][1] childed = [ [node for node in children] for children in LevelOrderGroupIter(tree, filter_=lambda n: n.children) ][1] # Sort the lists childless.sort(key=lambda x: x.name) childed.sort(key=lambda x: x.name) return childless + childed
def get_nodes_per_level(tree, skip_leaves=False): if skip_leaves: nodes_per_level = [[ node.name for node in children if not node.is_leaf ] for children in LevelOrderGroupIter(tree)] else: nodes_per_level = [[node.name for node in children] for children in LevelOrderGroupIter(tree)] nodes_per_level = list(filter(lambda x: len(x) > 0, nodes_per_level)) return nodes_per_level
def entity_analysis(node_dictionary): node_dictionary = node_dictionary root = node_dictionary['root'] orders = [[node.name for node in children] for children in LevelOrderGroupIter(root)] order_consumption = [] if len(orders): order_consumption = [len(o) for o in orders] average_reference_length = 0 if len(orders) >= 2: average_reference_length = np.mean( np.array([ grandchildren.height for child in root.children for grandchildren in child.children ])) max_expansion = 0 if len(orders) - 1: prev, cur, level = 1, 1, 1 max_expansion = 0 while cur != prev: cur_level = [ [node.name for node in children] for children in LevelOrderGroupIter(root, maxlevel=level + 1) ][-1] prev_level = [[ node.name for node in children ] for children in LevelOrderGroupIter(root, maxlevel=level)][-1] if (len(cur_level) / len(prev_level)) > max_expansion: max_expansion = len(cur_level) / len(prev_level) prev = len(prev_level) cur = len(cur_level) level += 1 return { 'order_consumption': order_consumption, 'average_reference_length': average_reference_length, 'max_expansion': max_expansion }
def getTimeForLevels(self): group = 0 levelTimes = {} firstRoot = True highestIndex = 0 for root in self.roots: levelIndex = 0 timeGroups = [[node.time for node in children] for children in LevelOrderGroupIter(root)] for level in timeGroups: minOnLevel = self.findMinTimeStamp(level) if firstRoot or highestIndex < levelIndex: levelTimes[levelIndex] = minOnLevel if highestIndex < levelIndex: highestIndex += 1 else: levelTimes[levelIndex] = min(minOnLevel, levelTimes[levelIndex]) levelIndex += 1 group += 1 firstRoot = False times = {} times[0] = 0 index = 1 while index < len(levelTimes): times[index] = self.compareStrippedStamps(levelTimes[0], levelTimes[index]) index += 1 return times
def findNodesOnLevel(self, level): nodeCount = 0 for root in self.roots: nodes = [[node.nodeNr for node in children] for children in LevelOrderGroupIter(root)] if len(nodes) > level: nodeCount += len(nodes[level]) return nodeCount
def getPredecessorsVariables(self, prefixed=False, parentheses=True): lst = [] for children in LevelOrderGroupIter(self): for node in children: lst += node.getVariables(prefixed=prefixed, parentheses=parentheses) return lst
def WavRec(y,shape,decLevel=5,family='haar',randperm=None): if randperm is not None: y=y[inv(randperm)] y=y.reshape(shape) root=AnyNode(id="root",shape=shape) lastParent=root lls=shape#lastLevelShape for i in range(decLevel): lls=tuple([int(x/2) for x in lls]) childA=AnyNode(id="child"+str(i+1)+"A",value=np.zeros(lls),shape=lls,level=i,parent=lastParent) childH=AnyNode(id="child"+str(i+1)+"H",value=np.zeros(lls),shape=lls,level=i,parent=lastParent) childV=AnyNode(id="child"+str(i+1)+"V",value=np.zeros(lls),shape=lls,level=i,parent=lastParent) childD=AnyNode(id="child"+str(i+1)+"D",value=np.zeros(lls),shape=lls,level=i,parent=lastParent) lastParent=childA # dec=np.zeros(y.shape) layers=[children for children in LevelOrderGroupIter(root)] for layer in layers[::-1]: try: lls=tuple([int(x*2) for x in lls]) cA=y[0:int(lls[0]/2),0:int(lls[1]/2)] cV=y[int(lls[0]/2):lls[0],0:int(lls[1]/2)] cH=y[0:int(lls[0]/2),int(lls[1]/2):lls[1]] cD=y[int(lls[0]/2):lls[0],int(lls[1]/2):lls[1]] y[0:lls[0],0:lls[1]]=pywt.idwt2((cA,(cH,cV,cD)),'haar') except: break return y.reshape(-1,1)
def WavDec(x,shape,decLevel=5,family='haar',randperm=None): x=x.reshape(shape) root=AnyNode(id="root",value=x,shape=x.shape) lastParent=root for i in range(decLevel): coeffs = pywt.dwt2(lastParent.value, family) cA, (cH, cV, cD) = coeffs childA=AnyNode(id="child"+str(i+1)+"A",value=cA,shape=cA.shape,level=i,parent=lastParent) childH=AnyNode(id="child"+str(i+1)+"H",value=cH,shape=cH.shape,level=i,parent=lastParent) childV=AnyNode(id="child"+str(i+1)+"V",value=cV,shape=cV.shape,level=i,parent=lastParent) childD=AnyNode(id="child"+str(i+1)+"D",value=cD,shape=cD.shape,level=i,parent=lastParent) lastParent=childA dec=np.zeros(x.shape) layers=[children for children in LevelOrderGroupIter(root)] for layer in layers[::-1]: try: layerImg=np.vstack((np.hstack((layer[0].value,layer[1].value)),np.hstack((layer[2].value,layer[3].value)))) layer[0].parent.value=layerImg except: break #plt.imshow(np.log(1+np.abs(root.value))) y=root.value.reshape((-1,1)) if randperm is not None: y=y[randperm] return y
def tree_of_ImageNet(): labels = np.array(os.listdir('/data/local/datasets/ImageNet/train')) freq = np.zeros(len(labels), np.float) for i, label in enumerate(labels): freq[i] = len( os.listdir('/data/local/datasets/ImageNet/train/' + label)) freq /= np.sum(freq) tree = get_hierarchy_of_label(labels=labels, freqs=freq, xml='imagenet_structure.xml', population_code=1000, class_sel=1) array = [] for level, children in enumerate(LevelOrderGroupIter(tree)): for node in children: name = node.name if type(name) is tuple: name = name[0] leafs = len([0 for child in PreOrderIter(node) if child.is_leaf]) array.append((name, node.freq, level, node.is_leaf, leafs)) array = np.array(array, dtype=([('label', 'U128'), ('frequency', np.float), ('level', np.int), ('is_leaf', np.bool), ('leafs_contained', np.int)])) pd.DataFrame(array).to_csv('FrequencyOfImageNetClasses.csv')
def get_aminos_perc_dict(self, root, aminos_list, unknown_aminos): ### convert list of aminos to dict (amino -> its perc) # get list of aminos levels aminos_levels = [[node.amino for node in children] for children in LevelOrderGroupIter( root, filter_=lambda n: n.amino != 'LocRootAmino') ] # clear any empty lists (parent roots) aminos_levels = [x for x in aminos_levels if x] # run through levels deepables = [*unknown_aminos] deepables.remove('?') # '?' is not searchable deepables.remove('-') # '-' is not searchable aminos_dict = {} for target_amino in aminos_list: total_perc = 0 deepness_perc = 1 for level in aminos_levels: amino_perc_on_lvl = (level.count(target_amino) / len(level)) total_perc = total_perc + (amino_perc_on_lvl) * deepness_perc num_of_deepables = sum(level.count(x) for x in deepables) deepables_perc_on_lvl = (num_of_deepables / len(level)) deepness_perc = deepness_perc * deepables_perc_on_lvl aminos_dict[target_amino] = total_perc return aminos_dict
def get_codons_perc_dict(self, root, codons_list): ### convert list of codons to dict (codon -> its perc) # get list of codons levels codons_levels = [[node.codon for node in children] for children in LevelOrderGroupIter( root, filter_=lambda n: n.codon != 'LocRootCodon') ] # clear any empty lists (parent roots) codons_levels = [x for x in codons_levels if x] codons_dict = {} for target_codon in codons_list: total_perc = 0 deepness_perc = 1 for level in codons_levels: codon_perc_on_lvl = (level.count(target_codon) / len(level)) total_perc = total_perc + (codon_perc_on_lvl) * deepness_perc num_of_deepables = sum([ len(set(codon).difference(set(['A', 'C', 'T', 'G', '-']))) != 0 for codon in level ]) deepables_perc_on_lvl = (num_of_deepables / len(level)) deepness_perc = deepness_perc * deepables_perc_on_lvl codons_dict[target_codon] = total_perc return codons_dict
def show_tree(self, level=-1): """ Print the current state of the tree along with some statistics on nodes :param level: Max level to print. If -1 print full tree :return: Tree representation as a string or nothing if printed """ result = [] # From list of tuples of nodes to list of nodes if level > 0: nodes_selections = [ e for sub in list(LevelOrderGroupIter(self.root))[:level + 1] for e in sub ] else: nodes_selections = [] # Iterate through the Tree and construct the Output for indent, _, node in RenderTree(self.root, childiter=self._sort_by_move): if level == -1 or node in nodes_selections: result.append( (self.OUT % (indent, node.ACTION, node.GAME.current_player.display, node.N_WINS, node.N_PLAYS, node.V, node.Q, node.U, node.PRIOR, node.Q + node.U))) # Display the result print('\n'.join(result))
def build_linkage(self): # get a tuple of node at each level levels = [] for group in LevelOrderGroupIter(self): levels.append(group) # just find how many nodes are leaves # this is necessary only because we need to add n to non-leaf clusters num_leaves = 0 for node in PostOrderIter(self): if not node.children: num_leaves += 1 link_count = 0 node_index = 0 linkages = [] labels = [] for g, group in enumerate( levels[::-1][:-1]): # reversed and skip the last for i in range(len(group) // 2): # get partner nodes left_node = group[2 * i] right_node = group[2 * i + 1] # just double check that these are always partners assert leftsibling(right_node) == left_node # check if leaves, need to add some new fields to track for linkage if not left_node.children: left_node._ind = node_index left_node._n_clusters = 1 node_index += 1 labels.append(left_node.name) if not right_node.children: right_node._ind = node_index right_node._n_clusters = 1 node_index += 1 labels.append(right_node.name) # find the parent, count samples parent_node = left_node.parent n_clusters = left_node._n_clusters + right_node._n_clusters parent_node._n_clusters = n_clusters # assign an ind to this cluster for the dendrogram parent_node._ind = link_count + num_leaves link_count += 1 distance = g + 1 # equal height for all links # add a row to the linkage matrix linkages.append( [left_node._ind, right_node._ind, distance, n_clusters]) labels = np.array(labels) linkages = np.array(linkages, dtype=np.double) # needs to be a double for scipy return (linkages, labels)
def get_lowest_level(self): level_it = LevelOrderGroupIter(self) last = next(level_it) nxt = last while nxt is not None: last = nxt nxt = next(level_it, None) return last
def textpart_levels(tree): levels, current_level = [], [] for group in filter(None, LevelOrderGroupIter(tree, filter_=is_textpart)): current_level.append(group[0]) levels.append(copy(current_level)) return levels
def max_leaf_decision(self, r): for children in LevelOrderGroupIter(r, maxlevel=2): for node in children: if node in r.children: if node.name.grade == r.name.grade: if not node.children: return node.name.decision[0] else: return self.max_leaf_decision(node)
def tree_search(r): if not r.children: #if node is a leaf return max = -1001 for children in LevelOrderGroupIter(r, maxlevel=2): for node in children: if node in r.children: tree_search(node) if node.name.grade > max: max = node.name.grade r.name.grade = max #set current node's eval to max of children
def compute_summing_matrix(tree): bottom_nodes = [leaf.name for leaf in tree.leaves] tree_nodes = list() matrix_rows = list() for level in LevelOrderGroupIter(tree): for node in level: node_leaves = [leaf.name for leaf in node.leaves] matrix_rows.append( [int(leaf in node_leaves) for leaf in bottom_nodes]) tree_nodes.append(node.name) summing_matrix = np.asarray(matrix_rows) return summing_matrix, tree_nodes, bottom_nodes
def _cascade_tree_helper(start_ind, probs, stop_inds, max_depth, node_hist_mat): root = Node(start_ind) root = generate_cascade_tree(root, probs, 1, stop_inds=stop_inds, visited=[], max_depth=max_depth) for level, children in enumerate(LevelOrderGroupIter(root)): for node in children: node_hist_mat[node.name, level] += 1
def test_levelordergroup(): """LevelOrderGroupIter.""" f = Node("f") b = Node("b", parent=f) a = Node("a", parent=b) d = Node("d", parent=b) c = Node("c", parent=d) e = Node("e", parent=d) g = Node("g", parent=f) i = Node("i", parent=g) h = Node("h", parent=i) eq_(list(LevelOrderGroupIter(f)), [(f,), (b, g), (a, d, i), (c, e, h)]) eq_(list(LevelOrderGroupIter(f, maxlevel=0)), []) eq_(list(LevelOrderGroupIter(f, maxlevel=3)), [(f,), (b, g), (a, d, i)]) eq_(list(LevelOrderGroupIter(f, filter_=lambda n: n.name not in ('e', 'g'))), [(f,), (b,), (a, d, i), (c, h)]) eq_(list(LevelOrderGroupIter(f, stop=lambda n: n.name == 'd')), [(f,), (b, g), (a, i), (h, )]) it = LevelOrderGroupIter(f) eq_(next(it), (f, )) eq_(next(it), (b, g)) eq_(list(it), [(a, d, i), (c, e, h)])
def test_it_builds_the_tree(self): expected_tree = Node("root") a = Node("A", parent=expected_tree) b = Node("B", parent=expected_tree) _ = Node("A1", parent=a) _ = Node("A2", parent=a) _ = Node("B1", parent=b) _ = Node("B2", parent=b) _ = Node("B3", parent=b) actual_tree = build_tree(self.hierarchy) nodes_expected_tree = [ node.name for children in LevelOrderGroupIter(expected_tree) for node in children ] nodes_actual_tree = [ node.name for children in LevelOrderGroupIter(actual_tree) for node in children ] assert nodes_expected_tree == nodes_actual_tree, \ "Actual tree is not equal to expected tree."
def get_neighbors(self, product, level=0): if isinstance(product, str): product = self.get_node(product) print("Getting neighbors of " + product.name) """ Getting parents for a specific product at a specific level, level 0 is the leafs of the tree (ie. The products themselves) """ parent = product for i in range(0, level): # Getting the right parent parent = parent.parent # Returning the corresponding children at specific level similarity ([-1] is fo getting only product IDs) return [[node.name for node in children] for children in LevelOrderGroupIter(parent)][-1]
def getTimeStampOnLevel(self, maxLevel): stamps = [] #print("Max") #print(maxLevel) for root in self.roots: timeGroups = [[node.time for node in children] for children in LevelOrderGroupIter(root, maxlevel=maxLevel)] #print("Len") #print(len(timeGroups)) if len(timeGroups) + 1 >= maxLevel: #print(timeGroups[-1]) minOnLevel = min(timeGroups[-1]) #print(minOnLevel) stamps.append(minOnLevel) # print(min(stamps)) return min(stamps)
def recommended_play(self, scoring_func=average_wins): """ Move recommended by the Monte Carlo Tree Search :return: tuple corresponding to the recommended move """ nodes = list(LevelOrderGroupIter(self.root)) if nodes: level1_nodes = nodes[1] scores = [ scoring_func(plays=n.n_plays, wins=n.n_wins, ties=n.n_ties) for n in level1_nodes ] best_node = level1_nodes[np.argmax(scores)] return best_node.game.last_play
def _get_lowest_dirs(self, remote_dirs=False): """ Get list of lowest local directory paths (absolute) as mapped by the initial transfer of depth self.max_depth :return: list :rtype: list of strings """ dir_list = [[node.name for node in children] for children in LevelOrderGroupIter(self.top_node)][-1] if remote_dirs: dir_list_remote = [] for dir_path in dir_list: absolute_path = dir_path.replace(self.local_path, self.remote_path) dir_list_remote.append(absolute_path) dir_list = dir_list_remote return dir_list
def diff(self, compare_root): if compare_root.parent != None: raise ValueError('The parameter you specified is not a root node!') # Variables to hold number of layer and relationship differences between two trees layer_count = 0 relationship_count = 0 self.display() # Relationship groupings declare how each node is related to its parent [[1], [2,3], [4, 5]] this is how tree layers and child structure is evaluated root_grouping = np.array([[node.parent for node in children] for children in LevelOrderGroupIter(self.root)]) comparison_grouping = np.array([[node.parent for node in children] for children in LevelOrderGroupIter(compare_root)]) layer_chunk = [] comparison_chunk = [] # Count the nodes at each layer of the tree for node in root_grouping: layer_chunk.append(len(node)) for node in comparison_grouping: comparison_chunk.append(len(node)) # Create numpy arrays out of the lists layer_chunk = np.array(layer_chunk) comparison_chunk = np.array(comparison_chunk) # Pad shorter array with blank values todo check for equal chunking if len(layer_chunk) > len(comparison_chunk): comparison_chunk = np.pad(comparison_chunk, (0, len(layer_chunk) - len(comparison_chunk)), 'constant') else: layer_chunk = np.pad(layer_chunk, (0, len(comparison_chunk) - len(layer_chunk)), 'constant') print layer_chunk print comparison_chunk # Compute difference between serialized arrays (just use layer chunk bc they are the same length) layer_count = functools.reduce(lambda a, b: a + b, [abs(a_i - b_i) for a_i, b_i in zip(layer_chunk, comparison_chunk)]) percent = (functools.reduce(lambda a,b: a + b, layer_chunk) / layer_count) * 100 return layer_count, percent
def get_alphas(self): """ Create a sequence of trees by weakest link pruning, and calculate alpha values. """ # Create a deepcopy of self.root. root = copy.deepcopy(self.root) # Include all levels except leaves. levels = [[node for node in level] for level in LevelOrderGroupIter(root)][::-1][1:] alpha_values = [0] for level in levels: while level: node = level.pop() # Each time we prune a height==1 tree with 2 leaves. alpha = difference(node) / 2 node.children = () if alpha > 0: alpha_values.append(alpha) alpha_values = list(set(alpha_values)) return sorted(alpha_values)
def test_backpropagate2(self): # create fake wins and plays self.update_nodes(nodes=list(PreOrderIter(self.tree.root)), n_plays=0, n_wins=0) # select a node from which backpropagation start node_name = '0_3_6' node = findall(self.tree.root, filter_=lambda n: n.name == node_name)[0] self.tree.backpropagate(node, n_plays=20, n_wins=6, n_ties=0) # root node stats root_plays = self.tree.root.n_plays root_wins = self.tree.root.n_wins # level 1 nodes stats level1_nodes = list(LevelOrderGroupIter(self.tree.root))[1] level1_plays = sum([node.n_plays for node in level1_nodes]) level1_wins = sum([node.n_wins for node in level1_nodes]) self.assertEquals(root_plays, level1_plays) self.assertEquals(root_wins, level1_wins)
def recommended_play(self, train=True): """ Move recommended by the Monte Carlo Tree Search :return: A tuple corresponding to the recommended move """ nodes = list(LevelOrderGroupIter(self.root)) if nodes: action_prob = np.zeros((1, 9)) for n in nodes[1]: action_prob[0, n.ACTION] = n.PRIOR if train: logging.debug("Using a stochastic action selection") return self.stochastic_action( nodes[1]).GAME.last_play, action_prob else: logging.debug("Using the U + Q strategy used in AlphaZero") return self.deterministic_action( nodes[1]).GAME.last_play, action_prob
def _init_fields(self, tree_obj): tree_obj = node_util.add_max_depth_att(tree_obj) tree_obj = node_util.tree2maxdepth(tree_obj) tree_obj = self.hasher.add_hash_att(tree_obj, self._type_dict) hash_gold_tree = tree_obj.hash hash_gold_levelorder = [] for tree_list in LevelOrderGroupIter(tree_obj): hash_gold_levelorder.append([tree.hash for tree in tree_list]) pad_el = hash_gold_levelorder[0] for i in range(self._decoder_timesteps - len(hash_gold_levelorder) + 2): hash_gold_levelorder.insert(0, pad_el) hash_gold_levelorder = hash_gold_levelorder[::-1] max_size = max(len(level) for level in hash_gold_levelorder) for level in hash_gold_levelorder: level.extend([-1] * (max_size - len(level))) hash_gold_levelorder = np.array(hash_gold_levelorder) return ( hash_gold_levelorder, hash_gold_tree, )