def get_tree_direct_children(tree):
"""
Get only the direct children of a given anytree Node
Parameters
----------
tree : Node
Returns
-------
List[Node]
Child-less nodes first, then sorted by name.
"""
childless = [[node for node in children]
for children in LevelOrderGroupIter(
tree, filter_=lambda n: not n.children)][1]
childed = [
[node for node in children]
for children in LevelOrderGroupIter(tree, filter_=lambda n: n.children)
][1]
# Sort the lists
childless.sort(key=lambda x: x.name)
childed.sort(key=lambda x: x.name)
return childless + childed
def get_nodes_per_level(tree, skip_leaves=False):
if skip_leaves:
nodes_per_level = [[
node.name for node in children if not node.is_leaf
] for children in LevelOrderGroupIter(tree)]
else:
nodes_per_level = [[node.name for node in children]
for children in LevelOrderGroupIter(tree)]
nodes_per_level = list(filter(lambda x: len(x) > 0, nodes_per_level))
return nodes_per_level
def entity_analysis(node_dictionary):
node_dictionary = node_dictionary
root = node_dictionary['root']
orders = [[node.name for node in children]
for children in LevelOrderGroupIter(root)]
order_consumption = []
if len(orders):
order_consumption = [len(o) for o in orders]
average_reference_length = 0
if len(orders) >= 2:
average_reference_length = np.mean(
np.array([
grandchildren.height for child in root.children
for grandchildren in child.children
]))
max_expansion = 0
if len(orders) - 1:
prev, cur, level = 1, 1, 1
max_expansion = 0
while cur != prev:
cur_level = [
[node.name for node in children]
for children in LevelOrderGroupIter(root, maxlevel=level + 1)
][-1]
prev_level = [[
node.name for node in children
] for children in LevelOrderGroupIter(root, maxlevel=level)][-1]
if (len(cur_level) / len(prev_level)) > max_expansion:
max_expansion = len(cur_level) / len(prev_level)
prev = len(prev_level)
cur = len(cur_level)
level += 1
return {
'order_consumption': order_consumption,
'average_reference_length': average_reference_length,
'max_expansion': max_expansion
}
def getTimeForLevels(self):
group = 0
levelTimes = {}
firstRoot = True
highestIndex = 0
for root in self.roots:
levelIndex = 0
timeGroups = [[node.time for node in children] for children in LevelOrderGroupIter(root)]
for level in timeGroups:
minOnLevel = self.findMinTimeStamp(level)
if firstRoot or highestIndex < levelIndex:
levelTimes[levelIndex] = minOnLevel
if highestIndex < levelIndex:
highestIndex += 1
else:
levelTimes[levelIndex] = min(minOnLevel, levelTimes[levelIndex])
levelIndex += 1
group += 1
firstRoot = False
times = {}
times[0] = 0
index = 1
while index < len(levelTimes):
times[index] = self.compareStrippedStamps(levelTimes[0], levelTimes[index])
index += 1
return times
def findNodesOnLevel(self, level):
nodeCount = 0
for root in self.roots:
nodes = [[node.nodeNr for node in children] for children in LevelOrderGroupIter(root)]
if len(nodes) > level:
nodeCount += len(nodes[level])
return nodeCount
def getPredecessorsVariables(self, prefixed=False, parentheses=True):
lst = []
for children in LevelOrderGroupIter(self):
for node in children:
lst += node.getVariables(prefixed=prefixed,
parentheses=parentheses)
return lst
def WavRec(y,shape,decLevel=5,family='haar',randperm=None):
if randperm is not None:
y=y[inv(randperm)]
y=y.reshape(shape)
root=AnyNode(id="root",shape=shape)
lastParent=root
lls=shape#lastLevelShape
for i in range(decLevel):
lls=tuple([int(x/2) for x in lls])
childA=AnyNode(id="child"+str(i+1)+"A",value=np.zeros(lls),shape=lls,level=i,parent=lastParent)
childH=AnyNode(id="child"+str(i+1)+"H",value=np.zeros(lls),shape=lls,level=i,parent=lastParent)
childV=AnyNode(id="child"+str(i+1)+"V",value=np.zeros(lls),shape=lls,level=i,parent=lastParent)
childD=AnyNode(id="child"+str(i+1)+"D",value=np.zeros(lls),shape=lls,level=i,parent=lastParent)
lastParent=childA
# dec=np.zeros(y.shape)
layers=[children for children in LevelOrderGroupIter(root)]
for layer in layers[::-1]:
try:
lls=tuple([int(x*2) for x in lls])
cA=y[0:int(lls[0]/2),0:int(lls[1]/2)]
cV=y[int(lls[0]/2):lls[0],0:int(lls[1]/2)]
cH=y[0:int(lls[0]/2),int(lls[1]/2):lls[1]]
cD=y[int(lls[0]/2):lls[0],int(lls[1]/2):lls[1]]
y[0:lls[0],0:lls[1]]=pywt.idwt2((cA,(cH,cV,cD)),'haar')
except:
break
return y.reshape(-1,1)
def WavDec(x,shape,decLevel=5,family='haar',randperm=None):
x=x.reshape(shape)
root=AnyNode(id="root",value=x,shape=x.shape)
lastParent=root
for i in range(decLevel):
coeffs = pywt.dwt2(lastParent.value, family)
cA, (cH, cV, cD) = coeffs
childA=AnyNode(id="child"+str(i+1)+"A",value=cA,shape=cA.shape,level=i,parent=lastParent)
childH=AnyNode(id="child"+str(i+1)+"H",value=cH,shape=cH.shape,level=i,parent=lastParent)
childV=AnyNode(id="child"+str(i+1)+"V",value=cV,shape=cV.shape,level=i,parent=lastParent)
childD=AnyNode(id="child"+str(i+1)+"D",value=cD,shape=cD.shape,level=i,parent=lastParent)
lastParent=childA
dec=np.zeros(x.shape)
layers=[children for children in LevelOrderGroupIter(root)]
for layer in layers[::-1]:
try:
layerImg=np.vstack((np.hstack((layer[0].value,layer[1].value)),np.hstack((layer[2].value,layer[3].value))))
layer[0].parent.value=layerImg
except:
break
#plt.imshow(np.log(1+np.abs(root.value)))
y=root.value.reshape((-1,1))
if randperm is not None:
y=y[randperm]
return y
def tree_of_ImageNet():
labels = np.array(os.listdir('/data/local/datasets/ImageNet/train'))
freq = np.zeros(len(labels), np.float)
for i, label in enumerate(labels):
freq[i] = len(
os.listdir('/data/local/datasets/ImageNet/train/' + label))
freq /= np.sum(freq)
tree = get_hierarchy_of_label(labels=labels,
freqs=freq,
xml='imagenet_structure.xml',
population_code=1000,
class_sel=1)
array = []
for level, children in enumerate(LevelOrderGroupIter(tree)):
for node in children:
name = node.name
if type(name) is tuple:
name = name[0]
leafs = len([0 for child in PreOrderIter(node) if child.is_leaf])
array.append((name, node.freq, level, node.is_leaf, leafs))
array = np.array(array,
dtype=([('label', 'U128'), ('frequency', np.float),
('level', np.int), ('is_leaf', np.bool),
('leafs_contained', np.int)]))
pd.DataFrame(array).to_csv('FrequencyOfImageNetClasses.csv')
def get_aminos_perc_dict(self, root, aminos_list, unknown_aminos):
### convert list of aminos to dict (amino -> its perc)
# get list of aminos levels
aminos_levels = [[node.amino for node in children]
for children in LevelOrderGroupIter(
root, filter_=lambda n: n.amino != 'LocRootAmino')
]
# clear any empty lists (parent roots)
aminos_levels = [x for x in aminos_levels if x]
# run through levels
deepables = [*unknown_aminos]
deepables.remove('?') # '?' is not searchable
deepables.remove('-') # '-' is not searchable
aminos_dict = {}
for target_amino in aminos_list:
total_perc = 0
deepness_perc = 1
for level in aminos_levels:
amino_perc_on_lvl = (level.count(target_amino) / len(level))
total_perc = total_perc + (amino_perc_on_lvl) * deepness_perc
num_of_deepables = sum(level.count(x) for x in deepables)
deepables_perc_on_lvl = (num_of_deepables / len(level))
deepness_perc = deepness_perc * deepables_perc_on_lvl
aminos_dict[target_amino] = total_perc
return aminos_dict
def get_codons_perc_dict(self, root, codons_list):
### convert list of codons to dict (codon -> its perc)
# get list of codons levels
codons_levels = [[node.codon for node in children]
for children in LevelOrderGroupIter(
root, filter_=lambda n: n.codon != 'LocRootCodon')
]
# clear any empty lists (parent roots)
codons_levels = [x for x in codons_levels if x]
codons_dict = {}
for target_codon in codons_list:
total_perc = 0
deepness_perc = 1
for level in codons_levels:
codon_perc_on_lvl = (level.count(target_codon) / len(level))
total_perc = total_perc + (codon_perc_on_lvl) * deepness_perc
num_of_deepables = sum([
len(set(codon).difference(set(['A', 'C', 'T', 'G', '-'])))
!= 0 for codon in level
])
deepables_perc_on_lvl = (num_of_deepables / len(level))
deepness_perc = deepness_perc * deepables_perc_on_lvl
codons_dict[target_codon] = total_perc
return codons_dict
def show_tree(self, level=-1):
""" Print the current state of the tree along with some statistics on nodes
:param level: Max level to print. If -1 print full tree
:return: Tree representation as a string or nothing if printed
"""
result = []
# From list of tuples of nodes to list of nodes
if level > 0:
nodes_selections = [
e for sub in list(LevelOrderGroupIter(self.root))[:level + 1]
for e in sub
]
else:
nodes_selections = []
# Iterate through the Tree and construct the Output
for indent, _, node in RenderTree(self.root,
childiter=self._sort_by_move):
if level == -1 or node in nodes_selections:
result.append(
(self.OUT %
(indent, node.ACTION, node.GAME.current_player.display,
node.N_WINS, node.N_PLAYS, node.V, node.Q, node.U,
node.PRIOR, node.Q + node.U)))
# Display the result
print('\n'.join(result))
def build_linkage(self):
# get a tuple of node at each level
levels = []
for group in LevelOrderGroupIter(self):
levels.append(group)
# just find how many nodes are leaves
# this is necessary only because we need to add n to non-leaf clusters
num_leaves = 0
for node in PostOrderIter(self):
if not node.children:
num_leaves += 1
link_count = 0
node_index = 0
linkages = []
labels = []
for g, group in enumerate(
levels[::-1][:-1]): # reversed and skip the last
for i in range(len(group) // 2):
# get partner nodes
left_node = group[2 * i]
right_node = group[2 * i + 1]
# just double check that these are always partners
assert leftsibling(right_node) == left_node
# check if leaves, need to add some new fields to track for linkage
if not left_node.children:
left_node._ind = node_index
left_node._n_clusters = 1
node_index += 1
labels.append(left_node.name)
if not right_node.children:
right_node._ind = node_index
right_node._n_clusters = 1
node_index += 1
labels.append(right_node.name)
# find the parent, count samples
parent_node = left_node.parent
n_clusters = left_node._n_clusters + right_node._n_clusters
parent_node._n_clusters = n_clusters
# assign an ind to this cluster for the dendrogram
parent_node._ind = link_count + num_leaves
link_count += 1
distance = g + 1 # equal height for all links
# add a row to the linkage matrix
linkages.append(
[left_node._ind, right_node._ind, distance, n_clusters])
labels = np.array(labels)
linkages = np.array(linkages,
dtype=np.double) # needs to be a double for scipy
return (linkages, labels)
def get_lowest_level(self):
level_it = LevelOrderGroupIter(self)
last = next(level_it)
nxt = last
while nxt is not None:
last = nxt
nxt = next(level_it, None)
return last
def textpart_levels(tree):
levels, current_level = [], []
for group in filter(None, LevelOrderGroupIter(tree, filter_=is_textpart)):
current_level.append(group[0])
levels.append(copy(current_level))
return levels
def max_leaf_decision(self, r):
for children in LevelOrderGroupIter(r, maxlevel=2):
for node in children:
if node in r.children:
if node.name.grade == r.name.grade:
if not node.children:
return node.name.decision[0]
else:
return self.max_leaf_decision(node)
def tree_search(r):
if not r.children: #if node is a leaf
return
max = -1001
for children in LevelOrderGroupIter(r, maxlevel=2):
for node in children:
if node in r.children:
tree_search(node)
if node.name.grade > max:
max = node.name.grade
r.name.grade = max #set current node's eval to max of children
def compute_summing_matrix(tree):
bottom_nodes = [leaf.name for leaf in tree.leaves]
tree_nodes = list()
matrix_rows = list()
for level in LevelOrderGroupIter(tree):
for node in level:
node_leaves = [leaf.name for leaf in node.leaves]
matrix_rows.append(
[int(leaf in node_leaves) for leaf in bottom_nodes])
tree_nodes.append(node.name)
summing_matrix = np.asarray(matrix_rows)
return summing_matrix, tree_nodes, bottom_nodes
def _cascade_tree_helper(start_ind, probs, stop_inds, max_depth,
node_hist_mat):
root = Node(start_ind)
root = generate_cascade_tree(root,
probs,
1,
stop_inds=stop_inds,
visited=[],
max_depth=max_depth)
for level, children in enumerate(LevelOrderGroupIter(root)):
for node in children:
node_hist_mat[node.name, level] += 1
def test_levelordergroup():
"""LevelOrderGroupIter."""
f = Node("f")
b = Node("b", parent=f)
a = Node("a", parent=b)
d = Node("d", parent=b)
c = Node("c", parent=d)
e = Node("e", parent=d)
g = Node("g", parent=f)
i = Node("i", parent=g)
h = Node("h", parent=i)
eq_(list(LevelOrderGroupIter(f)),
[(f,), (b, g), (a, d, i), (c, e, h)])
eq_(list(LevelOrderGroupIter(f, maxlevel=0)),
[])
eq_(list(LevelOrderGroupIter(f, maxlevel=3)),
[(f,), (b, g), (a, d, i)])
eq_(list(LevelOrderGroupIter(f, filter_=lambda n: n.name not in ('e', 'g'))),
[(f,), (b,), (a, d, i), (c, h)])
eq_(list(LevelOrderGroupIter(f, stop=lambda n: n.name == 'd')),
[(f,), (b, g), (a, i), (h, )])
it = LevelOrderGroupIter(f)
eq_(next(it), (f, ))
eq_(next(it), (b, g))
eq_(list(it), [(a, d, i), (c, e, h)])
def test_it_builds_the_tree(self):
expected_tree = Node("root")
a = Node("A", parent=expected_tree)
b = Node("B", parent=expected_tree)
_ = Node("A1", parent=a)
_ = Node("A2", parent=a)
_ = Node("B1", parent=b)
_ = Node("B2", parent=b)
_ = Node("B3", parent=b)
actual_tree = build_tree(self.hierarchy)
nodes_expected_tree = [
node.name for children in LevelOrderGroupIter(expected_tree)
for node in children
]
nodes_actual_tree = [
node.name for children in LevelOrderGroupIter(actual_tree)
for node in children
]
assert nodes_expected_tree == nodes_actual_tree, \
"Actual tree is not equal to expected tree."
def get_neighbors(self, product, level=0):
if isinstance(product, str):
product = self.get_node(product)
print("Getting neighbors of " + product.name)
""" Getting parents for a specific product at a specific level, level 0 is the leafs of the tree (ie. The products themselves) """
parent = product
for i in range(0, level):
# Getting the right parent
parent = parent.parent
# Returning the corresponding children at specific level similarity ([-1] is fo getting only product IDs)
return [[node.name for node in children]
for children in LevelOrderGroupIter(parent)][-1]
def getTimeStampOnLevel(self, maxLevel):
stamps = []
#print("Max")
#print(maxLevel)
for root in self.roots:
timeGroups = [[node.time for node in children] for children in LevelOrderGroupIter(root, maxlevel=maxLevel)]
#print("Len")
#print(len(timeGroups))
if len(timeGroups) + 1 >= maxLevel:
#print(timeGroups[-1])
minOnLevel = min(timeGroups[-1])
#print(minOnLevel)
stamps.append(minOnLevel)
# print(min(stamps))
return min(stamps)
def recommended_play(self, scoring_func=average_wins):
"""
Move recommended by the Monte Carlo Tree Search
:return: tuple corresponding to the recommended move
"""
nodes = list(LevelOrderGroupIter(self.root))
if nodes:
level1_nodes = nodes[1]
scores = [
scoring_func(plays=n.n_plays, wins=n.n_wins, ties=n.n_ties)
for n in level1_nodes
]
best_node = level1_nodes[np.argmax(scores)]
return best_node.game.last_play
def _get_lowest_dirs(self, remote_dirs=False):
"""
Get list of lowest local directory paths (absolute) as mapped by the initial transfer of depth self.max_depth
:return: list
:rtype: list of strings
"""
dir_list = [[node.name for node in children]
for children in LevelOrderGroupIter(self.top_node)][-1]
if remote_dirs:
dir_list_remote = []
for dir_path in dir_list:
absolute_path = dir_path.replace(self.local_path,
self.remote_path)
dir_list_remote.append(absolute_path)
dir_list = dir_list_remote
return dir_list
def diff(self, compare_root):
if compare_root.parent != None:
raise ValueError('The parameter you specified is not a root node!')
# Variables to hold number of layer and relationship differences between two trees
layer_count = 0
relationship_count = 0
self.display()
# Relationship groupings declare how each node is related to its parent [[1], [2,3], [4, 5]] this is how tree layers and child structure is evaluated
root_grouping = np.array([[node.parent for node in children] for children in LevelOrderGroupIter(self.root)])
comparison_grouping = np.array([[node.parent for node in children] for children in LevelOrderGroupIter(compare_root)])
layer_chunk = []
comparison_chunk = []
# Count the nodes at each layer of the tree
for node in root_grouping:
layer_chunk.append(len(node))
for node in comparison_grouping:
comparison_chunk.append(len(node))
# Create numpy arrays out of the lists
layer_chunk = np.array(layer_chunk)
comparison_chunk = np.array(comparison_chunk)
# Pad shorter array with blank values todo check for equal chunking
if len(layer_chunk) > len(comparison_chunk):
comparison_chunk = np.pad(comparison_chunk, (0, len(layer_chunk) - len(comparison_chunk)), 'constant')
else:
layer_chunk = np.pad(layer_chunk, (0, len(comparison_chunk) - len(layer_chunk)), 'constant')
print layer_chunk
print comparison_chunk
# Compute difference between serialized arrays (just use layer chunk bc they are the same length)
layer_count = functools.reduce(lambda a, b: a + b, [abs(a_i - b_i) for a_i, b_i in zip(layer_chunk, comparison_chunk)])
percent = (functools.reduce(lambda a,b: a + b, layer_chunk) / layer_count) * 100
return layer_count, percent
def get_alphas(self):
"""
Create a sequence of trees by weakest link pruning, and calculate alpha values.
"""
# Create a deepcopy of self.root.
root = copy.deepcopy(self.root)
# Include all levels except leaves.
levels = [[node for node in level]
for level in LevelOrderGroupIter(root)][::-1][1:]
alpha_values = [0]
for level in levels:
while level:
node = level.pop()
# Each time we prune a height==1 tree with 2 leaves.
alpha = difference(node) / 2
node.children = ()
if alpha > 0:
alpha_values.append(alpha)
alpha_values = list(set(alpha_values))
return sorted(alpha_values)
def test_backpropagate2(self):
# create fake wins and plays
self.update_nodes(nodes=list(PreOrderIter(self.tree.root)), n_plays=0, n_wins=0)
# select a node from which backpropagation start
node_name = '0_3_6'
node = findall(self.tree.root, filter_=lambda n: n.name == node_name)[0]
self.tree.backpropagate(node, n_plays=20, n_wins=6, n_ties=0)
# root node stats
root_plays = self.tree.root.n_plays
root_wins = self.tree.root.n_wins
# level 1 nodes stats
level1_nodes = list(LevelOrderGroupIter(self.tree.root))[1]
level1_plays = sum([node.n_plays for node in level1_nodes])
level1_wins = sum([node.n_wins for node in level1_nodes])
self.assertEquals(root_plays, level1_plays)
self.assertEquals(root_wins, level1_wins)
def recommended_play(self, train=True):
""" Move recommended by the Monte Carlo Tree Search
:return: A tuple corresponding to the recommended move
"""
nodes = list(LevelOrderGroupIter(self.root))
if nodes:
action_prob = np.zeros((1, 9))
for n in nodes[1]:
action_prob[0, n.ACTION] = n.PRIOR
if train:
logging.debug("Using a stochastic action selection")
return self.stochastic_action(
nodes[1]).GAME.last_play, action_prob
else:
logging.debug("Using the U + Q strategy used in AlphaZero")
return self.deterministic_action(
nodes[1]).GAME.last_play, action_prob
def _init_fields(self, tree_obj):
tree_obj = node_util.add_max_depth_att(tree_obj)
tree_obj = node_util.tree2maxdepth(tree_obj)
tree_obj = self.hasher.add_hash_att(tree_obj, self._type_dict)
hash_gold_tree = tree_obj.hash
hash_gold_levelorder = []
for tree_list in LevelOrderGroupIter(tree_obj):
hash_gold_levelorder.append([tree.hash for tree in tree_list])
pad_el = hash_gold_levelorder[0]
for i in range(self._decoder_timesteps - len(hash_gold_levelorder) +
2):
hash_gold_levelorder.insert(0, pad_el)
hash_gold_levelorder = hash_gold_levelorder[::-1]
max_size = max(len(level) for level in hash_gold_levelorder)
for level in hash_gold_levelorder:
level.extend([-1] * (max_size - len(level)))
hash_gold_levelorder = np.array(hash_gold_levelorder)
return (
hash_gold_levelorder,
hash_gold_tree,
)
