class AVLOpenHash(BaseHash):
def __init__(self, th):
super().__init__(th, None)
self.tree = AVLTree()
def add(self, value):
pos = self.hash_function(value)
self.table[pos] = self.tree.add(value, root=self.table[pos])
def search(self, value):
pos = self.hash_function(value)
return pos, self.tree.search(self.table[pos], value)
def delete(self, value):
pos, root = self.search(value)
if (root is not None):
self.table[pos].delete(self.table[pos], value)
@property
def load_factor(self):
load = 0
for root in self.table:
if (self.tree.get_height(root) > load):
load = self.tree.get_height(root)
return load
@property
def balance_factor(self):
soma = 0
for root in self.table:
soma += self.tree.get_height(root)
return soma / (self.th * self.load_factor)
def build_rr_tree(subtree_placement='root'):
tr = AVLTree()
tr.root = AVLNode('placeholder')
node_a = AVLNode('A')
node_x = AVLNode('X')
node_y = AVLNode('Y')
node_a.left = node_x
node_a.right = node_y
node_b = AVLNode('B')
node_z = AVLNode('Z')
node_b.left = node_a
node_b.right = node_z
if subtree_placement == 'right':
tr.root.right = node_b
elif subtree_placement == 'left':
tr.root.left = node_b
else: # tr.root is b
tr.root = node_b
return tr
def test_minmax():
tree = AVLTree()
rvals = [10, 5, 3, 18, 2, 100, 4123, 4393014, 49310]
for i, rval in enumerate(rvals):
print('({}) Inserting {}'.format(i,rval))
tree.insert(rval)
assert tree.min() == 2
assert tree.max() == 4393014
emtree = AVLTree()
def main():
for tree_size in (100, 1000, 10000):
tree = AVLTree(range(tree_size))
print(f"Ordered traversal of a tree size {tree_size}")
for name, test_func_creator in ORDERED_TRAVERSERS:
test_func = test_func_creator(tree)
assert test_func() == sorted(range(tree_size))
best = min(timeit.repeat(test_func, number=1000, repeat=3))
print(f" * [{name}]: {best * 1000:.1f}μs")
def test_del():
rvals = [48, 23, 74, 3, 44, 64, 98, 41, 56, 91]
# Delete leaf
tree = AVLTree.from_keys([10,5,25,3,8])
assert tree.size() == 5
assert tree.root.key == 10
assert tree.root.left.key, tree.root.right.key == (5,25)
assert tree.root.right.is_leaf()
tree.delete(8)
assert tree.root.key == 10
assert tree.root.left.key, tree.root.right.key == (5,25)
assert tree.root.right.is_leaf()
assert tree.root.left.left.key == 3
assert tree.root.left.right is None
assert 8 not in {x.key for x in tree.traverse()}
# Delete node with one child
tree = AVLTree.from_keys(rvals)
tree.delete(64)
assert tree.root.right.left.key == 56
# Symmetric one child case (we need to add another node)
tree = AVLTree.from_keys(rvals)
tree.insert(45)
tree.delete(45)
assert tree.root.left.right.right is None
# Delete two nodes
tree = AVLTree.from_keys([10, 5, 25, 3, 8])
assert tree.root.key == 10
assert tree.root.left.key == 5
assert tree.root.right.key == 25
assert tree.root.left.left.key == 3
tree.delete(5)
assert tree.root.key == 10
assert tree.root.left.key == 3
assert_raises(KeyError, tree.delete, -900)
emtree = AVLTree()
assert_raises(KeyError, emtree.delete, 1)
def test_path():
tree = AVLTree.from_keys([69, 60, 22, 91, 19, 71, 96, 27, 84, 43])
assert tree.root.key == 60
expected = [84,91,71,60]
observed = [x.key for x in tree.path_to_root(84)]
assert observed == expected
expected = expected[::-1]
observed = [x.key for x in tree.path_to_node(84)]
assert observed == expected
def __init__(self):
super(Window, self).__init__()
self.setupUi(self)
self.setWindowTitle("Demchenko - Self Balanced AVLTree")
self.axes = self.mpl_figure.add_subplot(111)
self.axes.axis('off')
self.canvas = FigureCanvas(self.mpl_figure)
self.mplVL.addWidget(self.canvas)
self.canvas.draw()
self.toolbar = NavigationToolbar(self.canvas, self, coordinates=True)
self.addToolBar(self.toolbar)
self.add_button.clicked.connect(self.add_node)
self.tree = AVLTree()
self.nodes = []
self.values = set()
self.show()
def mix_dumps_results_retAVL(files) -> (AVLTree, int):
'''
Create an AVL tree that represents the mix of the dumps
'''
mix_tree = AVLTree()
_mpages, _totalpages, _modulename = get_inorder_pagelist(files)
# iterate on the list of memory pages in reversed order and get results of new nodes added in each step
_nodes_after = 0
for key in reversed(sorted(_mpages)):
for f in _mpages[key]:
process_new_item(f, mix_tree)
return mix_tree, _totalpages
def mix_dumps_results_AVL(files) -> (AVLTree, int):
'''
Create an AVL tree that represents the mix of the dumps
'''
mix_tree = AVLTree()
_reversed_pages, _totalpages, _modulename = get_inorder_pagelist_AVL(files)
# iterate on the list of memory pages in reversed order and get results of new nodes added in each step
for itm in _reversed_pages:
first_occurrence = itm.find('[') + 1
itm = ast.literal_eval(itm[first_occurrence:-1])
if type(itm) is tuple:
process_new_item(itm, mix_tree)
elif type(itm) is list:
for aux in itm:
process_new_item(aux, mix_tree)
return mix_tree, _totalpages
def test_special():
# __contains__
tree = AVLTree.from_keys([10, 5, 8, 3, 20])
assert 20 in tree
assert 10 in tree
assert 15 not in tree
assert -1000 not in tree
emtree = AVLTree()
assert 1 not in emtree
# __nonzero__
if not tree:
assert False
if emtree:
assert False
# __len__
assert len(emtree) == 0
assert len(tree) == 5
def generate_mixed_module_AVL(dump_folder, tree: AVLTree, out_name,
t_pages: int):
'''
Create a mixed module, considering the info stored in the given AVL tree
'''
# create output file
outfile = os.path.join(output_folder, out_name)
fo = open(outfile, "wb")
in_order = tree.in_order(False)
logging.debug(f'[+] Content of the dict (in-order): {in_order}')
in_order_list = in_order.split(';')[:-1]
# iterate on the items, reading each file and writing the content to fo
last_page = -1 # initial page is 0
for item in in_order_list:
data = item.split('[')
n_page = int(data[0])
filename = data[1][:-1] # remove the ] at the end of the filename
# fill missing pages with zeros, if needed
if n_page - last_page != 1:
logger.debug('[+] Filling with zeros from {0} to {1}'.format(
last_page, n_page))
fill_with_zeros(fo, last_page, n_page - 1)
# now read a page from filename and write it to fo
read_and_write_page(fo, filename, n_page)
# update last_page appropriately for next iteration
last_page = n_page
# fill last pages, if needed
if t_pages - last_page != 1:
logger.debug('[+] Filling with zeros from {0} to {1}'.format(
last_page, t_pages))
fill_with_zeros(fo, last_page, t_pages - 1)
logger.debug('[+] Closing mixed file {0}'.format(out_name))
# close output file
fo.close()
return
if ta is not None:
sleeptime = 1.0 - (parsetime + storetime + requesttime)
print("[REQ] {0:.2f} - [PAR] {1:.2f} - [STR] {2:.2f} - [SLP] {3:.2f}".format(requesttime,
parsetime,
storetime,
sleeptime))
if sleeptime <= 1:
time.sleep(1 - sleeptime)
if (urls_crawled % 100) == 0:
frontOutput = open("./frontier", 'wb')
visitedOutput = open("./visited", 'wb')
pickle.dump(visited, visitedOutput)
pickle.dump(frontier, frontOutput)
frontOutput.close()
visitedOutput.close()
dumpOutlinks(outlinks)
if __name__ == '__main__':
seed_urls = ['http://en.wikipedia.org/wiki/Harvard_University',
'http://www.harvard.edu',
'http://colleges.usnews.rankingsandreviews.com/best-colleges/harvard-university-166027/overall-rankings',
'http://colleges.usnews.rankingsandreviews.com/best-colleges/harvard-university-2155',
'http://www.webometrics.info/en/world',
'http://www.timeshighereducation.co.uk/world-university-rankings/2014-15/world-ranking']
frontier = AVLTree()
for u in seed_urls:
frontier.insert(Key(u, 100000000))
startCrawl(frontier)
def test_selfbalancing():
tree = AVLTree()
rvals = [7680, 1027, 2564, 4103, 6671, 4118, 5143, 6680, 5144, 5146, 6682,
1560, 31, 2079, 546, 4643, 3625, 6188, 4656, 9776, 7732, 1591,
8760, 65, 6722, 68, 71, 8265, 8778, 1103, 81, 2642, 9299, 4692,
7251, 9809, 2136, 5720, 4700, 7260, 613, 6762, 4203, 5739, 3694,
5238, 4727, 6262, 2680, 5240, 4219, 2173, 1661, 8838, 4744, 1161,
9866, 4234, 1676, 6799, 9875, 3740, 7837, 5629, 9892, 1700, 9386,
1197, 2734, 7855, 6651, 8384, 2242, 1222, 200, 4816, 7382, 3803,
4317, 6366, 7389, 1251, 4839, 8938, 7403, 748, 751, 4848, 1268,
5878, 247, 4343, 2298, 9978, 6908, 5886, 256, 3845, 1798, 263,
8968, 6921, 6410, 3849, 5393, 2325, 6421, 4374, 3352, 6935, 282,
6939, 1815, 8987, 5409, 3875, 1315, 5418, 1323, 1838, 5426, 7988,
9015, 7481, 826, 4924, 6973, 1859, 6468, 3911, 4424, 1871, 1879,
3415, 7513, 3932, 5469, 8546, 1890, 9061, 7018, 7531, 2926, 4462,
9587, 7544, 893, 383, 5505, 8065, 7058, 7571, 2967, 1944, 6553,
9625, 2968, 5016, 1431, 6559, 5535, 6048, 6562, 1955, 9122, 4518,
2982, 3500, 3508, 6072, 7103, 6082, 963, 6084, 3523, 455, 7627,
8654, 979, 7637, 6101, 8150, 8664, 7642, 3035, 2525, 480, 2020,
5092, 4071, 1512, 3567, 2032, 6132, 4086, 7161, 4091, 1533]
for i, rval in enumerate(rvals):
tree.insert(rval)
for node in tree.traverse():
assert node.verify()
assert tree.find_node(31).key == 31
assert tree.find_node(6973).key == 6973
assert tree.find_node(9015).key == 9015
assert tree.size() == len(rvals)
print(tree.size())
for k in [5409, 3875, 1315, 5418, 1323, 1838, 7103, 6082, 963, 6084]:
tree.delete(k)
for x in tree.traverse():
assert x.verify()
def test_union():
t1 = AVLTree.from_keys([1, 3, 5, 7, 9])
t2 = AVLTree.from_keys([3,6,7])
t_union = t1.union(t2)
assert [x.key for x in t_union.traverse()] == [1,3,5,6,7,9]
def test_intersect():
t1 = AVLTree.from_keys([1, 3, 5, 7, 9])
t2 = AVLTree.from_keys([3,6,7])
t_intersect = t1.intersection(t2)
assert [x.key for x in t_intersect.traverse()] == [3,7]
def _main():
def build_rr_tree(subtree_placement='root'):
tr = AVLTree()
tr.root = AVLNode('placeholder')
node_a = AVLNode('A')
node_x = AVLNode('X')
node_y = AVLNode('Y')
node_a.left = node_x
node_a.right = node_y
node_b = AVLNode('B')
node_z = AVLNode('Z')
node_b.left = node_a
node_b.right = node_z
if subtree_placement == 'right':
tr.root.right = node_b
elif subtree_placement == 'left':
tr.root.left = node_b
else: # tr.root is b
tr.root = node_b
return tr
def build_rl_tree(subtree_placement='root'):
tr = AVLTree()
tr.root = AVLNode('placeholder')
node_a = AVLNode('A')
node_x = AVLNode('X')
node_y = AVLNode('Y')
node_b = AVLNode('B')
node_z = AVLNode('Z')
node_b.right = node_z
node_b.left = node_y
node_a.right = node_b
node_a.left = node_x
if subtree_placement == 'right':
tr.root.right = node_a
elif subtree_placement == 'left':
tr.root.left = node_a
else: # tr.root is a
tr.root = node_a
return tr
"""
print 'rotate_right test'
rr_tree = build_rr_tree()
#rr_tree.rotate_right(rr_tree.root)
print rr_tree
#print colored('rotate_left test', 'blue')
print 'rotate_left test'
#rl_tree = build_rl_tree()
#print rl_tree
#rl_tree.rotate_left(rl_tree.root)
#print rl_tree
#print rl_tree.delete_node(rl_tree.root)
#print rl_tree.insert('Q')
#print rl_tree
"""
tr = AVLTree()
import random, sys
sys.setrecursionlimit(100)
numbers = random.sample(xrange(90), 10)
random.shuffle(numbers)
for i in numbers:
#for i in range(15):
print termcolor.colored('test is inserting %d' % i, 'blue', attrs=['bold'])
tr.insert(i)
print tr
pick = i
print termcolor.colored('test is deleting %d' % 11, 'red', attrs=['bold'])
tr.delete_node(tr.root)
print tr
print 'balance factor at root %d' % tr.balance_factor(tr.root)
def gen_new(self):
"""Create new command and initialize driver"""
self.tree = AVLTree(preferred_erase=self.params["preferred_erase"],
preferred_dir=self.params["preferred_dir"])
self.write("new")
class Window(QMainWindow, form_class):
mpl_figure = Figure()
lines_pic = {}
lines = {}
def __init__(self):
super(Window, self).__init__()
self.setupUi(self)
self.setWindowTitle("Demchenko - Self Balanced AVLTree")
self.axes = self.mpl_figure.add_subplot(111)
self.axes.axis('off')
self.canvas = FigureCanvas(self.mpl_figure)
self.mplVL.addWidget(self.canvas)
self.canvas.draw()
self.toolbar = NavigationToolbar(self.canvas, self, coordinates=True)
self.addToolBar(self.toolbar)
self.add_button.clicked.connect(self.add_node)
self.tree = AVLTree()
self.nodes = []
self.values = set()
self.show()
def add_node(self):
data = self.key.text()
if len(data) > 3:
print(
"This value is too long, tree will look ugly, use numbers lower then 999"
)
return
self.key.clear()
value = None
try:
value = int(data)
except ValueError:
print("Wrong node value inserted. %s is not integer" % (data))
return
if value not in self.values:
self.values.add(value)
else:
print("%d is already in tree" % value)
return
self.tree.add(value)
# clean up old annotations
for node in self.nodes:
node.remove()
self.nodes.clear()
new_tree = self.tree.nodes_as_annotations()
for node, options in new_tree.items():
if not (node.left or node.right):
annotation = self.axes.annotate(node.value, **options)
self.nodes.append(annotation)
continue
if node.left:
left_data = options.copy()
left_node = new_tree.get(node.left)
left_data.update(xy=left_node['xytext'])
annotation = self.axes.annotate(node.value, **left_data)
self.nodes.append(annotation)
if node.right:
right_data = options.copy()
right_node = new_tree.get(node.right)
right_data.update(xy=right_node['xytext'])
annotation = self.axes.annotate(node.value, **right_data)
self.nodes.append(annotation)
self.canvas.draw()
def test_insertion():
tree = AVLTree()
tree.insert(10)
assert tree.root.key == 10
tree.insert(5)
assert tree.root.left.key == 5
tree.insert(8)
assert tree.root.key == 8
assert tree.root.left.key == 5
assert tree.root.right.key == 10
tree.insert(7)
assert tree.root.left.right.key == 7
tree.insert(6)
assert tree.root.left.key == 6
assert tree.root.left.left.key == 5
assert tree.root.left.right.key == 7
#!/usr/bin/env python3
import traverse
from avl import AVLTree
tree = AVLTree()
root = None
root = tree.insert(root, 10)
root = tree.insert(root, 20)
root = tree.insert(root, 30)
root = tree.insert(root, 40)
root = tree.insert(root, 50)
root = tree.insert(root, 25)
print('=========[preorder]==========')
traverse.preorder(root)
print('=========[postorder]==========')
traverse.postorder(root)
print('=========[inorder]==========')
traverse.inorder(root)
import random, sys
from avl import AVLTree
rvals = list({random.randint(0, sys.maxsize) for x in range(5000)})
tree = AVLTree()
for val in rvals:
tree.insert(val)
def __init__(self, th):
super().__init__(th, None)
self.tree = AVLTree()
def __init__(self, th, max_height):
super().__init__(th, None)
self.tree = AVLTree()
self.max_height = max_height
class LambdaHMA(BaseHash):
def __init__(self, th, max_height):
super().__init__(th, None)
self.tree = AVLTree()
self.max_height = max_height
def rehash(self):
# tamanho novo da tabela hash
new_th = next_prime(self.th * 2)
# nova tabela
new_tb = [None for i in range(new_th)]
# percorre a tabela original
for root in self.table:
# se a arvore não for nula:
if (root is not None):
# lista de elementos da avl
items = self.tree.to_list(root)
# para cada elemento na lista
for item in items:
# loop de tentativas
for i in range(int(new_th / 2) + 1):
pos = (item + i**2) % new_th
if (self.tree.get_height(new_tb[pos]) >
self.max_height):
continue
if (self.tree.search(new_tb[pos], item) is not None):
continue
new_tb[pos] = self.tree.add(item, root=new_tb[pos])
break
self.table = new_tb
self.th = new_th
def can_add(self, value):
not_available = 0
for root in self.table:
if (self.tree.get_height(root) > self.max_height):
not_available += 1
continue
if (self.tree.search(root, value) is not None):
not_available += 1
continue
return not_available <= int(self.th / 2)
def add(self, value):
if (not self.can_add(value)):
self.rehash()
for i in range(int(self.th / 2) + 1):
pos = self.hash_function(value + i**2)
if (self.tree.get_height(self.table[pos]) > self.max_height):
continue
if (self.tree.search(self.table[pos], value) is not None):
continue
self.tree.add(value, root=self.table[pos])
break
def search(self, value):
for i in range(int(self.th / 2) + 1):
pos = self.hash_function(value + i**2)
if (self.tree.search(self.table[pos], value) is not None):
return pos, self.tree.search(self.table[pos], value)
def delete(self, value):
pos, root = self.search(value)
if (root is not None):
self.table[pos].delete(self.table[pos], value)
def get_inorder_pagelist_AVL(files) -> (list, int, str):
'''
Create an AVL tree that represents the mix of the dumps
'''
sort_tree = AVLTree()
#sort_tree = {}
total_pages = -1
md5_first_page = -1
_filename = ''
current_file_version = ''
base_address = ''
# iterate on files
for f in files:
logger.info(f"[+] Processing dump log file \"{f}\" ... ")
lines = read_data(f)
current_file_version = ''
base_address = ''
for line in lines:
data = line.split(':')
aux = data[0].split(',')
filename = aux[1]
# get file version
file_version = aux[3]
if current_file_version == '':
current_file_version = file_version
if _filename == '':
_filename = filename.split('-')[-3]
current_base_address = filename.split('-')[-1].split('.')[0]
if base_address == '':
base_address = filename.split('-')[-1].split('.')[0]
# check if the first page is identical, otherwise give a warning
current_md5 = aux[2]
if md5_first_page == -1:
md5_first_page = current_md5
elif md5_first_page != current_md5:
logger.warning(
f'MD5 mismatch in "{filename}" ({current_md5} found, it should be {md5_first_page})'
)
if current_file_version != file_version:
logger.warning(
f'Version of {_filename} DLL differs ({current_file_version} vs. {file_version}), adding DLL considering version ...'
)
filename += '-' + file_version
elif base_address != current_base_address: # XXX occurs by WoW64
logger.warning(
f'Module base address differs ({current_base_address} found, it should be {base_address}), skipping ... '
)
continue
# get total pages, and warn if different
current_total = int(aux[4])
if total_pages == -1: # XXX we consider the first total pages value as the reference value
total_pages = current_total
elif total_pages != current_total:
logger.warning(
f'Total number of memory pages mismatch in "{filename}" ({current_total} found, it should be {total_pages})'
)
list_pages = data[1].split(',')
# store in an auxiliary structure to sort by len(list_pages)
sort_tree.insert(len(list_pages), (filename, list_pages),
duplicated_keys=True)
# then traverse the tree in reversed order
in_order = sort_tree.in_order(False)
in_order = in_order.split(';')[:-1]
return reversed(in_order), total_pages, _filename
def test_traverse():
keys = [10, 5, 3, 18, 2]
tree = AVLTree.from_keys([10, 5, 3, 18, 2])
assert [x.key for x in tree.traverse()] == list(sorted(keys))
assert [x.key for x in tree.traverse(reverse=True)] == list(sorted(keys, reverse=True))
def generate_tree(elems: list[any]) -> AVLTree:
res = AVLTree()
for elem in elems:
res.insert(elem)
return res