def update(utree, freq):
updt_tree = Tree(tree=utree, deep=True)
for item in utree.expand_tree('root'):
if item == 'root':
continue
if updt_tree.contains(item) and utree[item].data < freq:
updt_tree.remove_node(item)
return updt_tree
def test_root_removal(self):
t = Tree()
t.create_node(identifier="root-A")
self.assertEqual(len(t.nodes.keys()), 1)
self.assertEqual(t.root, 'root-A')
t.remove_node(identifier="root-A")
self.assertEqual(len(t.nodes.keys()), 0)
self.assertEqual(t.root, None)
t.create_node(identifier="root-B")
self.assertEqual(len(t.nodes.keys()), 1)
self.assertEqual(t.root, 'root-B')
def _build_tree(self, scores: ndarray, bin_edges: ndarray) -> Tree:
# Build tree with specified number of children at each level
tree = Tree()
tree.add_node(Node()) # root node
nodes_prev = [tree.get_node(tree.root)]
for level in range(self.depth):
nodes_current = []
for node in nodes_prev:
children = []
for _ in range(self.n_children[level]):
child = Node()
tree.add_node(child, parent=node)
children.append(child)
nodes_current.extend(children)
nodes_prev = nodes_current
assignments = np.digitize(scores, bin_edges) - 1
# Store instance ids in leaves
leaves = tree.leaves()
for k, node in enumerate(leaves):
instance_ids = np.where(assignments == k)[0]
if instance_ids.size == 0:
tree.remove_node(node.identifier)
else:
node.data = instance_ids
# Prune empty leaves
check_for_empty_leaves = True
while check_for_empty_leaves:
check_for_empty_leaves = False
leaves = tree.leaves()
for node in leaves:
if node.data is None and len(node.successors(
tree.identifier)) == 0:
# Node is empty and has no siblings
tree.remove_node(node.identifier)
check_for_empty_leaves = True
# Simplify tree: remove nodes that only have one child
for nid in tree.expand_tree(mode=tree.WIDTH):
children = tree.children(nid)
if len(children) == 1:
tree.link_past_node(nid)
return tree
def create_tree(files):
tree = Tree()
root = files[0]
tree.create_node(f"[{root.url.split('/')[0]}]", root.url.split("/")[0])
for item in files:
if not tree.contains(item.url):
pieces = item.url.split("/")
for index, path in enumerate(pieces, 1):
if not tree.contains("/".join(pieces[:index])):
if len(pieces) == index and item.is_file:
path = f"{path} ({item.qty_lines} linhas)"
else:
path = f"[{path}]"
tree.create_node(path, "/".join(pieces[:index]),
parent="/".join(pieces[:index - 1]))
if tree.contains(f'{root.url}/tree'):
tree.remove_node(f'{root.url}/tree')
return tree
class RMQueue(object):
__metaclass__ = Singleton
def __init__(self):
self.tree = Tree()
self.MAX_METRIC_COUNT = 12
self.CAL_INTERVAL_IN_SECOND = 2 * 60 * 60
self.conf = conf.Config("./conf/config.json")
def set_stat_interval(self, interval):
self.CAL_INTERVAL_IN_SECOND = interval
def set_system_memory(self, size):
root = self.get_root()
root.data.set_abs_memory(float(size))
def create_queue(self, name=None, parent=None):
data = QueueData()
self.tree.create_node(name, name, parent, data)
def display(self):
self.tree.show()
def display_score(self, queue=None, depth=0, table=None, printer=None):
flag = False
if queue is None:
queue = self.get_root()
flag = True
table = PrettyTable([
"QUEUE", "PENDING AVG", "PENDING DIV", "MEMORY USAGE AVG(Q)",
"MEMORY USAGE AVG(C)", "MEMORY USAGE DIV", "ABS CAPACITY"
])
if table is not None:
table.add_row([
queue.tag, 0 if queue.data.get_pending() == 0 else "%.3f" %
queue.data.get_pending(),
0 if queue.data.get_pending_div() == 0 else "%.3f" %
queue.data.get_pending_div(),
0 if queue.data.get_mem_usage() == 0 else "%.3f" %
queue.data.get_mem_usage(),
0 if queue.data.cal_queue_memory_usage() == 0 else "%.3f" %
queue.data.cal_queue_memory_usage(),
0 if queue.data.get_mem_usage_div() == 0 else "%.3f" %
queue.data.get_mem_usage_div(),
str(0 if queue.data.get_abs_capacity() == 0 else "%.3f" %
queue.data.get_abs_capacity()) + " %"
])
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.display_score(child, depth + 1, table)
if flag:
if printer is None:
print('------------' + utils.get_str_time() +
' SCORE ----------')
print table
else:
printer.write('\n------------' + utils.get_str_time() +
' SCORE ----------\n')
printer.write(str(table))
def display_prediction(self,
queue=None,
depth=0,
table=None,
printer=None):
flag = False
if queue is None:
queue = self.get_root()
flag = True
table = PrettyTable([
"QUEUE", "DESIRED CAPACITY(Q)", "DESIRED CAPACITY(C)",
"ABS CAPACITY"
])
if table is not None:
table.add_row([
queue.tag,
str(0 if queue.data.wish.capacity == 0 else "%.3f" %
(100 * queue.data.wish.capacity)) + " %",
0 if queue.data.wish.abs_capacity == 0 else "%.3f" %
queue.data.wish.abs_capacity,
str(0 if queue.data.config.abs_capacity == 0 else "%.3f" %
queue.data.config.abs_capacity) + " %"
])
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.display_prediction(child, depth + 1, table)
if flag:
if printer is None:
print('------------' + utils.get_str_time() +
' PREDICTION ----------')
print table
else:
printer.write('\n------------' + utils.get_str_time() +
' PREDICTION ----------\n')
printer.write(str(table))
def write_score(self, path):
FileOperator.touch(path)
with open(path, 'a') as f:
self.display_score(printer=f)
def request_score(self, queue=None):
if queue is None:
queue = self.get_root()
postData = {
'queue': queue.tag,
'pending': queue.data.get_pending(),
'pending_div': queue.data.get_pending_div(),
'memory_usage': queue.data.get_mem_usage(),
'memory_usage_div': queue.data.get_mem_usage_div(),
'abs_capacity': queue.data.get_abs_capacity()
}
requests.post(str(self.conf.es_rest_address) +
str(self.conf.es_index) + "score",
data=json.dumps(postData))
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.request_score(child)
def request_prediction(self, queue=None):
if queue is None:
queue = self.get_root()
postData = {
'queue': queue.tag,
'wish_capacity': queue.data.wish.capacity,
'wish_abs_capacity': queue.data.wish.abs_capacity,
'abs_capacity': queue.data.config.abs_capacity
}
requests.post(str(self.conf.es_rest_address) +
str(self.conf.es_index) + "prediction",
data=json.dumps(postData))
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.request_prediction(child)
def write_prediction(self, path):
FileOperator.touch(path)
with open(path, 'a') as f:
self.display_prediction(printer=f)
def add_job(self, job, qname):
queue = self.tree.get_node(qname)
if queue.is_leaf():
queue.data.add_job(job)
else:
print("Cannot add jobs to parent queue", queue.tag,
queue.identifier)
def add_metric(self, qname):
queue = self.tree.get_node(qname)
queue.data.add_metric(queue.cur_metric)
if len(queue.data.metrics) > RMQueue.MAX_METRIC_COUNT:
del queue.data.metrics[0]
def remove_queue(self, qname):
self.tree.remove_node(qname)
def move_queue(self, src, dest):
self.tree.move_node(src, dest)
def get_queue(self, qname):
return self.tree.get_node(qname)
def get_root(self):
return self.get_queue('root')
def is_leaf(self, qname):
queue = self.tree.get_node(qname)
return queue.is_leaf()
def cal_slowdown(self, queue=None):
if queue is None:
queue = self.get_root()
avg_slowdown = 0.0
if queue.is_leaf():
job_count = len(queue.data.jobs)
for i in list(range(job_count)):
job = queue.data.jobs[i]
slowdown = (job.wait_time + job.run_time) / job.run_time
avg_slowdown += slowdown / job_count
queue.data.set_job_count(job_count)
queue.data.cur_metric.slowdown = avg_slowdown
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_slowdown(child)
job_count = 0
for child in children:
job_count += child.data.get_job_count()
queue.data.set_job_count(job_count)
if job_count == 0:
queue.data.cur_metric.slowdown = avg_slowdown
return avg_slowdown
for child in children:
avg_slowdown += child.data.get_job_count(
) * child.data.get_slowdown() / job_count
queue.data.cur_metric.slowdown = avg_slowdown
return queue.data.get_slowdown()
def cal_pending(self, queue=None):
if queue is None:
queue = self.get_root()
if queue.is_leaf():
if len(queue.data.pendings) > 0:
queue.data.cur_metric.pending = np.mean(queue.data.pendings)
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_pending(child)
queue.data.cur_metric.pending += child.data.get_pending()
return queue.data.get_pending()
def cal_pending_division(self, queue=None):
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_pending_division(child)
count = len(children)
avg_pending = queue.data.get_pending() * 1.0 / count
square_sum = 0.0
for child in children:
square_sum += np.square(child.data.get_pending() - avg_pending)
division = np.sqrt(square_sum / count)
queue.data.cur_metric.pending_div = division
return division
def cal_slowdown_division(self, queue=None):
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_slowdown_division(child)
square_sum = 0.0
count = len(children)
for child in children:
square_sum += np.square(child.data.get_slowdown() -
queue.data.get_slowdown())
division = np.sqrt(square_sum / count)
queue.data.cur_metric.slowdown_div = division
return division
def cal_memory_usage(self, queue=None):
if queue is None:
queue = self.get_root()
if queue.is_leaf():
capacity = queue.data.get_abs_capacity()
memory_usage = 0.0
if capacity != 0:
memory_usage = 100.0 * queue.data.cal_queue_memory_usage(
) / capacity
queue.data.set_mem_usage(memory_usage)
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_memory_usage(child)
abs_memory_usage = 0
for child in children:
abs_memory_usage += child.data.get_abs_memory_usage()
queue.data.set_mem_usage(100.0 * abs_memory_usage /
queue.data.get_abs_capacity())
return queue.data.get_mem_usage()
def cal_mem_usage_division(self, queue=None):
if queue is None:
queue = self.get_root()
std_division = 0.0
if self.is_leaf(queue.tag):
queue.data.cur_metric.mem_usage_div = std_division
return std_division
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_mem_usage_division(child)
count = len(children)
total_mem_usage = 0
for child in children:
total_mem_usage += child.data.get_mem_usage()
avg_mem_usage = total_mem_usage / count
square_sum = 0
for child in children:
square_sum += np.square(child.data.get_mem_usage() -
avg_mem_usage)
std_division = np.sqrt(square_sum / count)
queue.data.cur_metric.mem_usage_div = std_division
return std_division
def cal_abs_capacity_bottom_up(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = 0.0
for child in children:
self.cal_abs_capacity_bottom_up(child)
abs_capacity += child.data.get_abs_capacity()
queue.data.set_abs_capacity(abs_capacity)
def cal_desired_abs_capacity_bottom_up(self, queue=None, delim=None):
if queue is None:
queue = self.get_root()
delim = 1
if self.is_leaf(queue.tag):
queue.data.wish.capacity = queue.data.wish.abs_capacity / delim
else:
children = self.tree.children(queue.tag)
abs_capacity = 0.0
for child in children:
self.cal_desired_abs_capacity_bottom_up(
child, queue.data.config.abs_capacity * delim / 100)
abs_capacity += child.data.wish.abs_capacity
queue.data.wish.capacity = abs_capacity / delim
def cal_abs_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.set_abs_capacity(100.0)
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_capacity(queue.data.get_abs_capacity() *
child.data.get_capacity() / 100.0)
self.cal_abs_capacity_top_down(child)
def cal_desired_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.wish.capacity = 100.0
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.wish.abs_capacity
for child in children:
child.data.wish.capacity = child.data.config.capacity
if abs_capacity == 0:
child.data.wish.capacity = 0
else:
child.data.wish.capacity = child.data.wish.abs_capacity / abs_capacity * 100.0
self.cal_desired_capacity_top_down(child)
def cal_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.get_abs_capacity()
for child in children:
if abs_capacity == 0:
child.data.set_capacity(0)
else:
child.data.set_capacity(child.data.get_abs_capacity() /
abs_capacity * 100)
self.cal_capacity_top_down(child)
def cal_abs_memory_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.cal_totalMb_mean()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_memory(queue.data.get_abs_memory() *
child.data.get_capacity() / 100)
self.cal_abs_memory_top_down(child)
def clear_mus_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_queue_memory_usage()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_mus_top_down(child)
def clear_jobs_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_jobs()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_jobs_top_down(child)
def clear_pendings_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_pendings()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_pendings_top_down(child)
def score(self):
# self.cal_abs_capacity_bottom_up()
# self.cal_capacity_top_down()
# self.cal_abs_memory_top_down()
# self.cal_slowdown()
# self.cal_slowdown_division()
self.cal_pending()
self.cal_pending_division()
self.cal_memory_usage()
self.cal_mem_usage_division()
# self.clear_jobs_top_down()
# self.clear_pendings_top_down()
# self.clear_mus_top_down()
def predict(self):
self.cal_desired_abs_capacity_bottom_up()
sub_t.show()
print(sep + "Children of Diane")
for child in tree.is_branch("diane"):
print(tree[child].tag)
print(sep + "OOhh~ new members join Jill's family:")
new_tree = Tree()
new_tree.create_node("n1", 1) # root node
new_tree.create_node("n2", 2, parent=1)
new_tree.create_node("n3", 3, parent=1)
tree.paste("jill", new_tree)
tree.show()
print(sep + "They leave after a while:")
tree.remove_node(1)
tree.show()
print(sep + "Now Jill moves to live with Grand-x-father Harry:")
tree.move_node("jill", "harry")
tree.show()
print(sep + "A big family for George to send message to the oldest Harry:")
for node in tree.rsearch("george"):
print(tree[node].tag)
########NEW FILE########
__FILENAME__ = folder_tree
#!/usr/bin/env python
# A file folder scanner contributed by @holger
#
# You can spicify the scanned folder and file pattern by changing rootPath
class RMQueue(metaclass=Singleton):
MAX_METRIC_COUNT = 12
CAL_INTERVAL_IN_SECOND = 2 * 60 * 60 # 2hours
def __init__(self):
self.tree = Tree()
def set_stat_interval(self, interval):
RMQueue.CAL_INTERVAL_IN_SECOND = interval
def set_system_memory(self, size):
root = self.get_root()
root.data.set_abs_memory(float(size))
def create_queue(self, name=None, parent=None):
data = QueueData()
self.tree.create_node(name, name, parent, data)
def display(self):
self.tree.show()
def display_score_old(self, queue=None, depth=0):
if queue is None:
queue = self.get_root()
print('------------' + utils.get_str_time() + ' SCORE ----------')
print(24 * ' ' + ' SLOWDOWN MEMORY USAGE ')
print('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV')
if depth >= 0:
print(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_slowdown(), \
' %8.3f ' % queue.data.get_slowdown_div(), \
' %8.3f' % queue.data.get_mem_usage(), \
' %8.3f' % queue.data.get_mem_usage_div())
"""
print(queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \
'div: %.3f)' % queue.data.get_slowdown_div(), \
'(mem usage: %.3f' % queue.data.get_mem_usage(), \
'div: %.3f)' % queue.data.get_mem_usage_div())
"""
else:
print('-'*depth + queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \
'div: %.3f)' % queue.data.get_slowdown_div(), \
'(mem usage: %.3f' % queue.data.get_mem_usage(), \
'div: %.3f)' % queue.data.get_mem_usage_div())
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.display_score(child, depth + 2)
def display_score(self, queue=None, depth=0):
if queue is None:
queue = self.get_root()
print('------------' + utils.get_str_time() + ' SCORE ----------')
print(24 * ' ' + ' PENDING MEMORY USAGE ')
print('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV')
if depth >= 0:
print(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_pending(), \
' %8.3f ' % queue.data.get_pending_div(), \
' %8.3f' % queue.data.get_mem_usage(), \
' %8.3f' % queue.data.get_mem_usage_div())
else:
print('-'*depth + queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \
'div: %.3f)' % queue.data.get_slowdown_div(), \
'(mem usage: %.3f' % queue.data.get_mem_usage(), \
'div: %.3f)' % queue.data.get_mem_usage_div())
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.display_score(child, depth + 2)
def display_prediction(self, queue=None, depth=0):
if queue is None:
queue = self.get_root()
print('------------' + utils.get_str_time() +
' PREDICTION ----------')
print('QUEUE NAME DESIRED CAPACITY')
if depth >= 0:
print(queue.tag + (22 - len(queue.tag)) * ' ',
' %8.3f' % queue.data.wish.capacity)
# print(queue.tag, 'desired capacity: %.3f' % queue.data.wish.capacity)
else:
print('-' * depth + queue.tag,
'desired capacity: %.3f' % queue.data.wish.capacity)
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.display_prediction(child, depth + 2)
def write_score(self, path):
with open(path, 'a') as f:
self.write_score_top_down(output=f)
def write_score_top_down_old(self, queue=None, depth=0, output=None):
if queue is None:
queue = self.get_root()
output.writelines(
('\n---------', utils.get_str_time(), ' SCORE ---------\n'))
output.writelines(24 * ' ' +
' SLOWDOWN MEMORY USAGE\n')
output.writelines('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV\n')
if depth >= 0:
output.writelines(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_slowdown() + \
' %8.3f ' % queue.data.get_slowdown_div() + \
' %8.3f' % queue.data.get_mem_usage() + \
' %8.3f' % queue.data.get_mem_usage_div() + '\n')
"""
output.writelines( (queue.tag, ' (slowdown: %.3f' % queue.data.get_slowdown(), \
' div: %.3f)' % queue.data.get_slowdown_div(), \
' (mem usage: %.3f' % queue.data.get_mem_usage(), \
' div: %.3f)' % queue.data.get_mem_usage_div(), '\n'))
"""
else:
output.writelines(('-'*depth + queue.tag, ' (slowdown: %.3f' % queue.data.get_slowdown(), \
' div: %.3f)' % queue.data.get_slowdown_div(), \
' (mem usage: %.3f' % queue.data.get_mem_usage(), \
' div: %.3f)' % queue.data.get_mem_usage_div(), '\n'))
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.write_score_top_down(child, depth + 2, output)
def write_score_top_down(self, queue=None, depth=0, output=None):
if queue is None:
queue = self.get_root()
output.writelines(
('\n---------', utils.get_str_time(), ' SCORE ---------\n'))
output.writelines(24 * ' ' +
' PENDING MEMORY USAGE\n')
output.writelines('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV\n')
output.writelines(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_pending() + \
' %8.3f ' % queue.data.get_pending_div() + \
' %8.3f' % queue.data.get_mem_usage() + \
' %8.3f' % queue.data.get_mem_usage_div() + '\n')
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.write_score_top_down(child, depth + 2, output)
def write_prediction(self, path):
with open(path, 'a') as f:
self.write_prediction_top_down(output=f)
def write_prediction_top_down(self, queue=None, depth=0, output=None):
if queue is None:
queue = self.get_root()
output.writelines(('\n---------', utils.get_str_time(),
' PREDICTION---------\n'))
output.writelines('QUEUE NAME DESIRED CAPACITY\n')
if depth >= 0:
output.writelines(queue.tag + (22 - len(queue.tag)) * ' ' +
' %8.3f' % queue.data.wish.capacity + '\n')
# output.writelines( (queue.tag, ' desired capacity: %.3f' % queue.data.wish.capacity, '\n'))
else:
output.writelines(('-'*depth + queue.tag, \
' desired capacity: %.3f' % queue.data.wish.capacity, '\n'))
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.write_prediction_top_down(child, depth + 2, output)
def add_job(self, job, qname):
queue = self.tree.get_node(qname)
if queue.is_leaf():
queue.data.add_job(job)
else:
print("Canot add jobs to parent queue", queue.tag,
queue.identifier)
def add_metric(self, qname):
queue = self.tree.get_node(qname)
queue.data.add_metric(queue.cur_metric)
if len(queue.data.metrics) > RMQueue.MAX_METRIC_COUNT:
del queue.data.metrics[0]
def remove_queue(self, qname):
"""
Remove a queue indicated by 'qname'; all the successors are
removed as well.
Return the number of removed nodes.
"""
self.tree.remove_node(qname)
def move_queue(self, src, dest):
"""
Move a queue indicated by @src parameter to be a child of
@dest.
"""
self.tree.move_node(src, dest)
def get_queue(self, qname):
return self.tree.get_node(qname)
def get_root(self):
return self.get_queue('root')
def is_leaf(self, qname):
queue = self.tree.get_node(qname)
return queue.is_leaf()
def cal_slowdown(self, queue=None):
"""
if current queue is a leaf queue:
calculate the average slowdown in is jobs.
else:
calculate the average slowdown of its chilren;
calculate the average slowdown of current queue through its chilren's average slowdown.
"""
if queue is None:
queue = self.get_root()
avg_slowdown = 0.0
if queue.is_leaf():
job_count = len(queue.data.jobs)
for i in list(range(job_count)):
job = queue.data.jobs[i]
slowdown = (job.wait_time + job.run_time) / job.run_time
avg_slowdown += slowdown / job_count
queue.data.set_job_count(job_count)
queue.data.cur_metric.slowdown = avg_slowdown
else: # parent queue
# First, get its all chilren queue, and call each child's cal_slowdown function
children = self.tree.children(queue.tag)
for child in children:
self.cal_slowdown(child)
# Second, get the job count
job_count = 0
for child in children:
job_count += child.data.get_job_count()
queue.data.set_job_count(job_count)
# Finally, calculate the average slowdown of the queue
if job_count == 0:
queue.data.cur_metric.slowdown = avg_slowdown
return avg_slowdown
for child in children:
avg_slowdown += child.data.get_job_count(
) * child.data.get_slowdown() / job_count
queue.data.cur_metric.slowdown = avg_slowdown
return queue.data.get_slowdown()
def cal_pending(self, queue=None):
"""
if current queue is a leaf queue:
calculate the average pending count in is pendings.
else:
calculate the average pending of its chilren;
calculate the pending of current queue through the sum all of its chilren's pending.
"""
if queue is None:
queue = self.get_root()
if queue.is_leaf():
queue.data.cal_leaf_pending()
else: # parent queue
# First, get its all chilren queue, and call each child's cal_pending function
# Second, get the sum of all its children pending
children = self.tree.children(queue.tag)
for child in children:
self.cal_pending(child)
queue.data.cur_metric.pending += child.data.get_pending()
return queue.data.get_pending()
def cal_pending_division(self, queue=None):
"""
if current queue is a leaf queue:
stdDivision is zero.
else:
calculate the standard division of its chilren;
calculate the standard division of current queue through its chilren's average pending.
"""
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else: # parent queue
children = self.tree.children(queue.tag)
# First, get its all chilren queue, and call each child's calSlowDown function
for child in children:
self.cal_pending_division(child)
# Second, calculate the square sum of division
count = len(children)
avg_pending = queue.data.get_pending() * 1.0 / count
squareSum = 0.0
for child in children:
squareSum += np.square(child.data.get_pending() - avg_pending)
# Finally, calculate the standard division of the queue
# if count == 0:
# queue.data.cur_metric.slowdown_div = division
# return division
division = np.sqrt(squareSum / count)
queue.data.cur_metric.pending_div = division
return division
def cal_slowdown_division(self, queue=None):
"""
if current queue is a leaf queue:
stdDivision is zero.
else:
calculate the standard division of its chilren;
calculate the standard division of current queue through its chilren's average slowdown.
"""
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else: # parent queue
children = self.tree.children(queue.tag)
# First, get its all chilren queue, and call each child's calSlowDown function
for child in children:
self.cal_slowdown_division(child)
# Second, calculate the square sum of division
squareSum = 0.0
count = len(children)
for child in children:
squareSum += np.square(child.data.get_slowdown() -
queue.data.get_slowdown())
# Finally, calculate the standard division of the queue
# if count == 0:
# queue.data.cur_metric.slowdown_div = division
# return division
division = np.sqrt(squareSum / count)
queue.data.cur_metric.slowdown_div = division
return division
def cal_memory_usage_old(self, queue=None):
"""
if current queue is a leaf queue:
MemoryUsage is the (sum of job memorySeconds )/(self.absMemory * CAL_INTERVAL_IN_SECOND)
Get absUsedMemory by self.memoryUsage * self.absMemory
else:
calculate the memory usage of its chilren;
calculate the absolute used memory of the queue.
MemoryUsage = absUsedMemory / absMemory
"""
if queue is None:
queue = self.get_root()
memory_usage = 0.0
if queue.is_leaf():
total_memory_seconds = queue.data.cal_leaf_mem_second()
total_memory_capacity = queue.data.get_abs_memory(
) * RMQueue.CAL_INTERVAL_IN_SECOND
memory_usage = 1.0 * total_memory_seconds / total_memory_capacity
queue.data.set_mem_usage(memory_usage)
queue.data.cal_abs_used_memory()
else: # parent queue
# First, get its all chilren queue, and call each child's calMemoryUsage function
children = self.tree.children(queue.tag)
for child in children:
self.cal_memory_usage_old(child)
# Second, calculate the absUsedMemory of current queue
abs_used_memory = 0
for child in children:
abs_used_memory += child.data.get_abs_used_memory()
queue.data.set_abs_used_memory(abs_used_memory)
# Finally, calculate the memory usage of the queue
queue.data.set_mem_usage(1.0 * queue.data.get_abs_used_memory() /
queue.data.get_abs_memory())
return queue.data.get_mem_usage()
def cal_memory_usage(self, queue=None):
"""
if current queue is a leaf queue:
MemoryUsage is the abs_memoryusage/self.absMemoryCapacity
else:
calculate the memory usage of its chilren;
calculate the absolute memory of the queue.
MemoryUsage = absUsedMemory / absMemory
"""
if queue is None:
queue = self.get_root()
memory_usage = 0.0
if queue.is_leaf():
abs_memory_usage = queue.data.cal_queue_memory_usage()
abs_memory_capacity = queue.data.get_abs_capacity()
memory_usage = 100.0 * abs_memory_usage / abs_memory_capacity
queue.data.set_mem_usage(memory_usage)
queue.data.set_abs_memory_usage(abs_memory_usage)
else: # parent queue
# First, get its all chilren queue, and call each child's calMemoryUsage function
children = self.tree.children(queue.tag)
for child in children:
self.cal_memory_usage(child)
# Second, calculate the absUsedMemoryUsage of current queue
abs_memory_usage = 0
for child in children:
abs_memory_usage += child.data.get_abs_memory_usage()
queue.data.set_abs_memory_usage(abs_memory_usage)
# Finally, calculate the memory usage of the queue
queue.data.set_mem_usage(100.0 *
queue.data.get_abs_memory_usage() /
queue.data.get_abs_capacity())
return queue.data.get_mem_usage()
def cal_mem_usage_division(self, queue=None):
"""
if current queue is a leaf queue:
memUsageDivision is zero.
else:
calculate the standard division of its chilren;
calculate the standard division of current queue through its chilren's average memoryUsage
"""
if queue is None:
queue = self.get_root()
std_division = 0.0
if self.is_leaf(queue.tag):
queue.data.cur_metric.mem_usage_div = std_division
return std_division
else: # parent queue
# First, get its all chilren queue, and call each child's calSlowDown function
children = self.tree.children(queue.tag)
for child in children:
self.cal_mem_usage_division(child)
# Second, calculate the average memory usage of all its children
count = len(children)
total_mem_usage = 0
for child in children:
total_mem_usage += child.data.get_mem_usage()
# print(child.data.get_mem_usage())
avg_mem_usage = total_mem_usage / count
# Finally, calculate the standard division of the queue
squareSum = 0
for child in children:
squareSum += np.square(child.data.get_mem_usage() -
avg_mem_usage)
std_division = np.sqrt(squareSum / count)
queue.data.cur_metric.mem_usage_div = std_division
return std_division
def cal_abs_capacity_bottom_up(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = 0
for child in children:
# print("Queue name: %s, abs_capacity: %.2f" %(child.tag, child.data.get_abs_capacity()))
self.cal_abs_capacity_bottom_up(child)
abs_capacity += child.data.get_abs_capacity()
queue.data.set_abs_capacity(abs_capacity)
def cal_desired_abs_capacity_bottom_up(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = 0.0
fixed_capacity = 0.0
for child in children:
self.cal_desired_abs_capacity_bottom_up(child)
if child.data.config.fixed:
# print("FIXED")
# print(child.data.config.capacity)
# print(child.data.config.abs_capacity)
fixed_capacity += child.data.config.capacity
else:
abs_capacity += child.data.wish.abs_capacity
for child in children:
if child.data.config.fixed:
child.data.wish.abs_capacity = abs_capacity / (
100.0 - fixed_capacity) * child.data.config.capacity
queue.data.wish.abs_capacity = abs_capacity * 100.0 / (
100.0 - fixed_capacity)
def clear_desired_abs_capacity(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.wish.abs_capacity = 0
if self.is_leaf(queue.tag):
return
else:
queue.data.cur_metric.pending = 0.0
children = self.tree.children(queue.tag)
for child in children:
self.clear_desired_abs_capacity(child)
def cal_abs_capacity_top_down(self, queue=None):
"""
This function calculate the abs capacity of each queue by its capacity.
This function should only be called once at the start time.
"""
if queue is None:
queue = self.get_root()
queue.data.set_abs_capacity(100.0)
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_capacity(queue.data.get_abs_capacity() *
child.data.get_capacity() / 100.0)
# print(child.data.get_abs_capacity())
self.cal_capacity_top_down(child)
def cal_desired_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.wish.capacity = 100.0
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.wish.abs_capacity
remain_capaciy = 100.0
for child in children:
if child.data.config.fixed:
child.data.wish.capacity = child.data.config.capacity
elif abs_capacity == 0:
child.data.wish.capacity = 0
else:
child.data.wish.capacity = child.data.wish.abs_capacity / abs_capacity * 100.0
self.cal_desired_capacity_top_down(child)
def cal_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.get_abs_capacity()
for child in children:
if abs_capacity == 0:
child.data.set_capacity(0)
else:
child.data.set_capacity(child.data.get_abs_capacity() /
abs_capacity * 100)
self.cal_capacity_top_down(child)
def cal_abs_memory_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.cal_totalMb_mean()
queue.data.clear_totalMb()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_memory(queue.data.get_abs_memory() *
child.data.get_capacity() / 100)
self.cal_abs_memory_top_down(child)
def clear_mus_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_queue_memory_usage()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_mus_top_down(child)
def clear_jobs_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_jobs()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_jobs_top_down(child)
def clear_pendings_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_pendings()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_pendings_top_down(child)
def before_scoring(self):
self.cal_abs_capacity_bottom_up()
self.cal_capacity_top_down()
self.cal_abs_memory_top_down()
def after_scoreing(self):
self.clear_jobs_top_down()
self.clear_pendings_top_down()
self.clear_mus_top_down()
def score(self):
self.before_scoring()
# self.cal_slowdown()
# self.cal_slowdown_division()
self.cal_pending()
self.cal_pending_division()
self.cal_memory_usage()
self.cal_mem_usage_division()
self.after_scoreing()
def before_predict(self):
self.cal_desired_abs_capacity_bottom_up()
def after_predict(self):
self.clear_desired_abs_capacity()
def predict(self):
self.before_predict()
self.cal_desired_capacity_top_down()
self.after_predict()
if __name__ == '__main__':
tree = Tree()
node_id = 0
root_node = (3, 3, 1, node_id)
all_nodes = [root_node]
tree.create_node(tuple_to_string(root_node), root_node)
depth = int(input("Enter the depth: "))
for node in all_nodes:
if node[2] == 1:
temp_node = (node[0] - 1, node[1] - 1, 0, node_id)
place_in_tree(temp_node)
if tree.depth() > depth:
tree.remove_node(temp_node)
break
temp_node = (node[0] - 2, node[1], 0, node_id)
place_in_tree(temp_node)
temp_node = (node[0], node[1] - 2, 0, node_id)
place_in_tree(temp_node)
temp_node = (node[0] - 1, node[1], 0, node_id)
place_in_tree(temp_node)
temp_node = (node[0], node[1] - 1, 0, node_id)
place_in_tree(temp_node)
else:
temp_node = (node[0] + 1, node[1] + 1, 1, node_id)
place_in_tree(temp_node)
if tree.depth() > depth:
tree.remove_node(temp_node)
break
class SAGG_BRIAC():
def __init__(self, min, max, temperature=20): # example --> min: [-1,-1] max: [1,1]
assert len(min) == len(max)
self.maxlen = 200
self.window_cp = 200
self.minlen = self.maxlen / 20
self.maxregions = 80
# init regions' tree
self.tree = Tree()
self.regions_bounds = [Box(min, max, dtype=np.float32)]
self.interest = [0.]
self.tree.create_node('root','root',data=Region(maxlen=self.maxlen,
cps_gs=[deque(maxlen=self.maxlen + 1), deque(maxlen=self.maxlen + 1)],
bounds=self.regions_bounds[-1], interest=self.interest[-1]))
self.nb_dims = len(min)
self.temperature = temperature
self.nb_split_attempts = 50
self.max_difference = 0.2
self.init_size = max - min
self.ndims = len(min)
self.mode_3_noise = 0.1
# book-keeping
self.sampled_tasks = []
self.all_boxes = []
self.all_interests = []
self.update_nb = 0
self.split_iterations = []
def compute_interest(self, sub_region):
if len(sub_region[0]) > self.minlen: # TRICK NB 4
cp_window = min(len(sub_region[0]), self.window_cp) # not completely window
half = int(cp_window / 2)
# print(str(cp_window) + 'and' + str(half))
first_half = np.array(sub_region[0])[-cp_window:-half]
snd_half = np.array(sub_region[0])[-half:]
diff = first_half.mean() - snd_half.mean()
cp = np.abs(diff)
else:
cp = 0
interest = np.abs(cp)
return interest
def split(self, nid):
# try nb_split_attempts splits
reg = self.tree.get_node(nid).data
best_split_score = 0
best_abs_interest_diff = 0
best_bounds = None
best_sub_regions = None
is_split = False
for i in range(self.nb_split_attempts):
sub_reg1 = [deque(maxlen=self.maxlen + 1), deque(maxlen=self.maxlen + 1)]
sub_reg2 = [deque(maxlen=self.maxlen + 1), deque(maxlen=self.maxlen + 1)]
# repeat until the two sub regions contain at least minlen of the mother region TRICK NB 1
while len(sub_reg1[0]) < self.minlen or len(sub_reg2[0]) < self.minlen:
# decide on dimension
dim = np.random.choice(range(self.nb_dims))
threshold = reg.bounds.sample()[dim]
bounds1 = Box(reg.bounds.low, reg.bounds.high, dtype=np.float32)
bounds1.high[dim] = threshold
bounds2 = Box(reg.bounds.low, reg.bounds.high, dtype=np.float32)
bounds2.low[dim] = threshold
bounds = [bounds1, bounds2]
valid_bounds = True
if np.any(bounds1.high - bounds1.low < self.init_size / 15): # to enforce not too small boxes TRICK NB 2
valid_bounds = False
if np.any(bounds2.high - bounds2.low < self.init_size / 15):
valid_bounds = valid_bounds and False
# perform split in sub regions
sub_reg1 = [deque(maxlen=self.maxlen + 1), deque(maxlen=self.maxlen + 1)]
sub_reg2 = [deque(maxlen=self.maxlen + 1), deque(maxlen=self.maxlen + 1)]
for i, task in enumerate(reg.cps_gs[1]):
if bounds1.contains(task):
sub_reg1[1].append(task)
sub_reg1[0].append(reg.cps_gs[0][i])
else:
sub_reg2[1].append(task)
sub_reg2[0].append(reg.cps_gs[0][i])
sub_regions = [sub_reg1, sub_reg2]
# compute interest
interest = [self.compute_interest(sub_reg1), self.compute_interest(sub_reg2)]
# compute score
split_score = len(sub_reg1) * len(sub_reg2) * np.abs(interest[0] - interest[1])
if split_score >= best_split_score and valid_bounds: # TRICK NB 3, max diff #and np.abs(interest[0] - interest[1]) >= self.max_difference / 8
is_split = True
best_abs_interest_diff = np.abs(interest[0] - interest[1])
best_split_score = split_score
best_sub_regions = sub_regions
best_bounds = bounds
if is_split:
if best_abs_interest_diff > self.max_difference:
self.max_difference = best_abs_interest_diff
# add new nodes to tree
for i, (cps_gs, bounds) in enumerate(zip(best_sub_regions, best_bounds)):
self.tree.create_node(parent=nid, data=Region(self.maxlen, cps_gs=cps_gs, bounds=bounds, interest=interest[i]))
else:
#print("abort mission")
# TRICK NB 6, remove old stuff if can't find split
assert len(reg.cps_gs[0]) == (self.maxlen + 1)
reg.cps_gs[0] = deque(islice(reg.cps_gs[0], int(self.maxlen / 4), self.maxlen + 1))
reg.cps_gs[1] = deque(islice(reg.cps_gs[1], int(self.maxlen / 4), self.maxlen + 1))
return is_split
def merge(self, all_nodes):
# get a list of children pairs
parent_children = []
for n in all_nodes:
if not n.is_leaf(): # if node is a parent
children = self.tree.children(n.identifier)
if children[0].is_leaf() and children[1].is_leaf(): # both children must be leaves for an easy remove
parent_children.append([n, children]) # [parent, [child1, child2]]
# sort each pair of children by their summed interest
parent_children.sort(key=lambda x: np.abs(x[1][0].data.interest - x[1][1].data.interest), reverse=False)
# remove useless pair
child1 = parent_children[0][1][0]
child2 = parent_children[0][1][1]
# print("just removed {} and {}, daddy is: {}, childs: {}".format(child1.identifier, child2.identifier,
# parent_children[0][0].identifier,
# self.tree.children(
#
# print("bef") # parent_children[0][0].identifier)))
# print([n.identifier for n in self.tree.all_nodes()])
self.tree.remove_node(child1.identifier)
self.tree.remove_node(child2.identifier)
# print("aff remove {} and {}".format(child1.identifier), child2.identifier)
# print([n.identifier for n in self.tree.all_nodes()])
# remove 1/4 of parent to avoid falling in a splitting-merging loop
dadta = parent_children[0][0].data # hahaha!
dadta.cps_gs[0] = deque(islice(dadta.cps_gs[0], int(self.maxlen / 4), self.maxlen + 1))
dadta.cps_gs[1] = deque(islice(dadta.cps_gs[1], int(self.maxlen / 4), self.maxlen + 1))
self.nodes_to_recompute.append(parent_children[0][0].identifier)
# remove child from recompute list if they where touched when adding the current task
if child1.identifier in self.nodes_to_recompute:
self.nodes_to_recompute.pop(self.nodes_to_recompute.index(child1.identifier))
if child2.identifier in self.nodes_to_recompute:
self.nodes_to_recompute.pop(self.nodes_to_recompute.index(child2.identifier))
def add_task_comp(self, node, task, comp):
reg = node.data
nid = node.identifier
if reg.bounds.contains(task): # task falls within region
self.nodes_to_recompute.append(nid)
children = self.tree.children(nid)
for n in children: # if task in region, task is in one sub-region
self.add_task_comp(n, task, comp)
need_split = reg.add(task, comp, children == []) # COPY ALL MODE
if need_split:
self.nodes_to_split.append(nid)
def update(self, task, continuous_competence, all_raw_rewards):
# add new (task, competence) to regions nodes
self.nodes_to_split = []
self.nodes_to_recompute = []
new_split = False
root = self.tree.get_node('root')
self.add_task_comp(root, task, continuous_competence)
#print(self.nodes_to_split)
assert len(self.nodes_to_split) <= 1
# split a node if needed
need_split = len(self.nodes_to_split) == 1
if need_split:
new_split = self.split(self.nodes_to_split[0])
if new_split:
self.update_nb += 1
#print(self.update_nb)
# update list of regions_bounds
all_nodes = self.tree.all_nodes()
if len(all_nodes) > self.maxregions: # too many regions, lets merge one of them
self.merge(all_nodes)
all_nodes = self.tree.all_nodes()
self.regions_bounds = [n.data.bounds for n in all_nodes]
# recompute interests of touched nodes
#print(self.nodes_to_recompute)
for nid in self.nodes_to_recompute:
#print(nid)
node = self.tree.get_node(nid)
reg = node.data
reg.interest = self.compute_interest(reg.cps_gs)
# collect new interests and new [comp, tasks] lists
all_nodes = self.tree.all_nodes()
self.interest = []
self.cps_gs = []
for n in all_nodes:
self.interest.append(n.data.interest)
self.cps_gs.append(n.data.cps_gs)
# bk-keeping
self.all_boxes.append(copy.copy(self.regions_bounds))
self.all_interests.append(copy.copy(self.interest))
self.split_iterations.append(self.update_nb)
assert len(self.interest) == len(self.regions_bounds)
return new_split, None
def draw_random_task(self):
return self.regions_bounds[0].sample() # first region is root region
def sample_task(self, args):
mode = np.random.rand()
if mode < 0.1: # "mode 3" (10%) -> sample on regions and then mutate lowest-performing task in region
if len(self.sampled_tasks) == 0:
self.sampled_tasks.append(self.draw_random_task())
else:
region_id = proportional_choice(self.interest, eps=0.0)
worst_task_idx = np.argmin(self.cps_gs[region_id][0])
# mutate task by a small amount (i.e a gaussian scaled to the regions range)
task = np.random.normal(self.cps_gs[region_id][1][worst_task_idx].copy(), 0.1)
# clip to stay within region (add small epsilon to avoid falling in multiple regions)
task = np.clip(task, self.regions_bounds[region_id].low + 1e-5, self.regions_bounds[region_id].high - 1e-5)
self.sampled_tasks.append(task)
elif mode < 0.3: # "mode 2" (20%) -> random task
self.sampled_tasks.append(self.draw_random_task())
else: # "mode 1" (70%) -> sampling on regions and then random task in selected region
region_id = proportional_choice(self.interest, eps=0.0)
self.sampled_tasks.append(self.regions_bounds[region_id].sample())
# # sample region
# if np.random.rand() < 0.2:
# region_id = np.random.choice(range(self.nb_regions))
# else:
# region_id = np.random.choice(range(self.nb_regions), p=np.array(self.probas))
# # sample task
# self.sampled_tasks.append(self.regions_bounds[region_id].sample())
#
# return self.sampled_tasks[-1].tolist()
# sample region
# region_id = proportional_choice(self.interest, eps=0.2)
# # sample task
# self.sampled_tasks.append(self.regions_bounds[region_id].sample())
return self.sampled_tasks[-1]
def dump(self, dump_dict):
dump_dict['all_boxes'] = self.all_boxes
dump_dict['split_iterations'] = self.split_iterations
dump_dict['all_interests'] = self.all_interests
return dump_dict
@property
def nb_regions(self):
return len(self.regions_bounds)
@property
def get_regions(self):
return self.regions_bounds
class RAMSTKDataModel(object): # pragma: no cover
"""
This is the meta-class for all RAMSTK Data Models.
:ivar tree: the :class:`treelib.Tree` that will contain the
structure of the RAMSTK module being modeled..
:ivar dao: the :class:`ramstk.dao.DAO` object used to communicate
with the RAMSTK Program database.
"""
def __init__(self, dao):
"""
Initialize an RAMSTK data model instance.
:param dao: the data access object for communicating with the
RAMSTK Program database.
:type dao: :class:`ramstk.dao.DAO.DAO`
"""
# Initialize private dictionary attributes.
# Initialize private list attributes.
# Initialize private scalar attributes.
self._last_id = None
# Initialize public dictionary attributes.
# Initialize public list attributes.
# Initialize public scalar attributes.
self.dao = dao
self.tree = Tree()
self.last_id = None
# Add the root to the Tree(). This is neccessary to allow multiple
# entries at the top level as there can only be one root in a treelib
# Tree(). Manipulation and viewing of a RAMSTK module tree needs to
# ignore the root of the tree.
try:
self.tree.create_node(tag=self._tag, identifier=0, parent=None)
except (tree.MultipleRootError, tree.NodeIDAbsentError,
tree.DuplicatedNodeIdError):
pass
def do_select(self, node_id, **kwargs): # pylint: disable=unused-argument
"""
Retrieve the instance of the RAMSTK<MODULE> model for the Node ID passed.
:param int node_id: the Node ID of the data package to retrieve.
:return: the instance of the RAMSTK<MODULE> class that was requested
or None if the requested Node ID does not exist.
"""
try:
_entity = self.tree.get_node(node_id).data
except AttributeError:
_entity = None
except tree.NodeIDAbsentError:
_entity = None
return _entity
def do_select_all(self, **kwargs): # pylint: disable=unused-argument
"""
Retrieve and build the RAMSTK Module tree.
:return: an SQLAlchemy session instance.
:rtype:
"""
_root = self.tree.root
for _node in self.tree.children(_root):
self.tree.remove_node(_node.identifier)
return self.dao.RAMSTK_SESSION(
bind=self.dao.engine, autoflush=False, expire_on_commit=False)
def do_insert(self, **kwargs):
"""
Add the list of RAMSTK<MODULE> instance to the RAMSTK Program database.
:param list entities: the list of RAMSTK<MODULE> entities to add to the
RAMSTK Program database.
:return: (_error_code, _msg); the error code and associated message.
:rtype: (int, str)
"""
_entities = kwargs['entities']
_session = self.dao.RAMSTK_SESSION(
bind=self.dao.engine, autoflush=False, expire_on_commit=False)
_error_code, _msg = self.dao.db_add(_entities, _session)
_session.close()
return _error_code, _msg
def do_delete(self, node_id):
"""
Delete the instance of RAMSTK<MODULE> from the RAMSTK Program database.
:param int node_id entity: the ID of the RAMSTK<MODULE> record to be
removed from the RAMSTK Program database.
:return: (_error_code, _msg); the error code and associated message.
:rtype: (int, str)
"""
_msg = ''
_session = self.dao.RAMSTK_SESSION(
bind=self.dao.engine, autoflush=False, expire_on_commit=False)
try:
_entity = self.tree.get_node(node_id).data
_error_code, _msg = self.dao.db_delete(_entity, _session)
if _error_code == 0:
self.tree.remove_node(node_id)
except AttributeError:
_error_code = 2005
_session.close()
return _error_code, _msg
def do_update(self, node_id):
"""
Update the RAMSTK<MODULE> instance in the RAMSTK Program database.
:param entity: the RAMSTK<MODULE> instance to update in the RAMSTK Program
database.
:return: (_error_code, _msg); the error code and associated message.
:rtype: (int, str)
"""
_error_code = 0
_msg = ''
_session = self.dao.RAMSTK_SESSION(
bind=self.dao.engine,
autoflush=True,
autocommit=False,
expire_on_commit=False)
try:
_entity = self.tree.get_node(node_id).data
if _entity is not None:
_session.add(_entity)
_error_code, _msg = self.dao.db_update(_session)
except AttributeError:
_error_code = 1
_msg = ('RAMSTK ERROR: Attempted to save non-existent '
'entity with Node ID {0:s}.').format(str(node_id))
_session.close()
return _error_code, _msg
print tree.is_branch('diane')
print('\n')
print("#"*4 + "OOhh~ new members enter Jill's family")
new_tree = Tree()
new_tree.create_node("n1", 1) # root node
new_tree.create_node("n2", 2, parent=1)
new_tree.create_node("n3", 3, parent=1)
tree.paste('jill', new_tree)
tree.show()
print('\n')
print("#"*4 + "We are sorry they are gone accidently :(")
tree.remove_node(1)
tree.show()
print('\n')
print("#"*4 + "Now Jill moves to live with Grand-x-father Harry")
tree.move_node('jill', 'harry')
tree.show()
print('\n')
print("#"*4 + "A big family for George to talk to Grand-x-father Harry")
for node in tree.rsearch('george', filter=lambda x: x.identifier != 'harry'):
print node
print('harry')
print('\n')
class BasicTree:
def __init__(self, vehsInfo):
self.tree = Tree()
self.root = self.tree.create_node("Root", "root") # root node
self.vehsInfo = vehsInfo
self.vehList = list(vehsInfo.keys())
self.i = 1
def _build(self, currentNode, vehList):
'''
:param vehList: A dict, keys is the set of vehicles, value is a tuple which represents (lane, position)
:param currentNode: The current node in the tree
:return: None
'''
s = [currentNode.tag.find(vid) for vid in vehList] # the quit contidion in recursion
if (np.array(s) >= 0).all():
return
for vehId in vehList:
if vehId not in currentNode.tag:
if currentNode.is_root:
prefix = currentNode.tag.replace("Root", "")
else:
prefix = currentNode.tag
self.tree.create_node(prefix + vehId + "-", prefix + vehId, parent=currentNode)
for node in self.tree.all_nodes():
if node.is_leaf():
self._build(currentNode=node, vehList=vehList)
def _prune(self):
laneId = [value[0] for value in self.vehsInfo.values()]
sortedList = []
for i in list(set(laneId)):
lane_info = {k: v[1] for k, v in self.vehsInfo.items() if v[0] == i}
# Vehicles in front are at the front of the lane
sortedList.append([vid[0] for vid in sorted(lane_info.items(), key=itemgetter(1), reverse=True)])
pruneList = [sublist for sublist in sortedList if len(sublist) > 1]
for subList in pruneList:
for index in range(1, len(subList)):
# first, prune th subtree which begin with illegal vehicle id
self.tree.remove_subtree(subList[index])
# second, delete the nodes which match the illegal pattern
pattern = subList[index] + ".*" + subList[0]
for node in self.tree.all_nodes():
if re.search(pattern, node.tag):
try:
self.tree.remove_node(node.identifier)
except:
pass
def build(self):
self._build(self.root, self.vehList)
self._prune()
def show(self):
self.tree.show()
def _leaves(self):
'''
:return: All the plan for vehicle passing currently.
'''
all_nodes = self.tree.all_nodes()
return [node for node in all_nodes if node.is_leaf()]
def legal_orders(self):
leaves = self._leaves()
orders = []
for pattern in leaves:
# upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3-
tmp = pattern.tag.split("-")
try:
tmp.remove('')
except:
pass
if len(tmp) == self.tree.depth():
orders.append(tmp)
return orders
from treelib import Tree, Node
if __name__ == '__main__':
# 树的创建，每个节点都有唯一的identifier作为标记，可以手动指定
tree = Tree()
# 增加树的节点,tag是树输出时的显示，identifier是唯一标志，根节点可以不指定父
tree.create_node(tag='root', identifier='root', data=0)
tree.create_node(tag='1_child',
identifier='1_child',
data=1,
parent='root')
tree.create_node(tag='2_child',
identifier='2_child',
data=2,
parent='root')
tree.create_node(tag='3_child',
identifier='3_child',
data=3,
parent='1_child')
# 树的粘贴，需要注意的是这个nid是tree的identifier，不是tree2的
tree2 = Tree()
tree2.create_node(tag='tutu', identifier='tutu', data=0)
tree.paste(nid='root', new_tree=tree2)
# 删除树的节点
tree.remove_node('tutu')
# 移动树的节点
tree.move_node('3_child', 'root')
# 打印树的结构
tree.show()
class PathList:
def __init__(self, disk):
self._tree = Tree()
self._disk = disk
self._tree.create_node(tag='root', identifier='root', data=FileInfo(type=False))
self.depth = 3
def update_path_list(self, file_id='root', depth=None, is_fid=True, **kwargs):
if depth is None:
depth = self.depth
kwargs.setdefault('max_depth', depth)
max_depth = kwargs['max_depth']
kwargs.setdefault('get_file_list_bar', GetFileListBar(max_depth))
kwargs.setdefault('ratio', 0)
get_file_list_bar = kwargs['get_file_list_bar']
ratio = kwargs['ratio']
get_file_list_bar.update(refresh_line=False)
if not is_fid:
file_id = self.get_path_fid(file_id, update=False)
file_list = self._disk.get_file_list(file_id)
if not file_list:
if depth == max_depth:
get_file_list_bar.refresh_line()
return False
old_file_list = self._tree.children(file_id)
for i in old_file_list:
if i.identifier not in [j['file_id'] for j in file_list]:
self._tree.remove_node(i.identifier)
for i, info in enumerate(file_list):
if depth == max_depth:
ratio = (i + 1) / len(file_list) if file_list else None
get_file_list_bar.update(depth=max_depth - depth, ratio=ratio, refresh_line=True)
file_info = self.get_file_info(info)[0]
if self._tree.get_node(file_info.id):
self._tree.update_node(file_info.id, data=file_info)
else:
self._tree.create_node(tag=file_info.name, identifier=file_info.id, data=file_info, parent=file_id)
if not file_info.type and depth:
self.update_path_list(file_id=file_info.id, depth=depth - 1, max_depth=max_depth,
get_file_list_bar=get_file_list_bar, ratio=ratio)
if depth == max_depth:
get_file_list_bar.refresh_line()
return True
def check_path_diff(self, local_path, disk_path_list):
p = Path(local_path)
change_file_list = []
for path in p.iterdir():
flag = False
for i, path_ in enumerate(disk_path_list, 1):
name, file_info = list(path_.items())[0]
if p / name not in p.iterdir():
change_file_list.append(p / name)
if Path(path) == p / name:
if Path(path).is_dir() and file_info['data'] and path.is_dir() != file_info['data'].type:
if 'children' in file_info:
children = file_info['children']
change_file_list.extend(self.check_path_diff(p / name, children))
elif list(path.iterdir()):
change_file_list.extend(list(path.iterdir()))
if file_info and file_info['data'] and path.is_file() == file_info['data'].type:
if path.is_file() and get_sha1(path).lower() != file_info['data'].content_hash.lower():
if i == len(disk_path_list):
change_file_list.append(path)
continue
else:
flag = True
if not flag and i == len(disk_path_list):
change_file_list.append(path)
if not len(list(p.iterdir())):
for path_ in disk_path_list:
name, file_info = list(path_.items())[0]
change_file_list.append(p / name)
if not len(disk_path_list):
for path_ in p.iterdir():
change_file_list.append(path_)
return list(set(change_file_list))
@staticmethod
def get_file_info(info):
file_info_list = []
if not isinstance(info, list):
info_list = [info]
else:
info_list = info
for info in info_list:
if info['type'] == 'file':
file_info = FileInfo(name=info['name'], id=info['file_id'], pid=info['parent_file_id'], type=True,
ctime=time.strptime(info['created_at'],
'%Y-%m-%dT%H:%M:%S.%fZ') if 'created_at' in info else time.localtime(),
update_time=time.strptime(info['updated_at'], '%Y-%m-%dT%H:%M:%S.%fZ'),
hidden=info.get('hidden'), category=info['category'],
content_type=info.get('content_type'),
size=info['size'], content_hash_name=info.get('content_hash_name'),
content_hash=info.get('content_hash'),
download_url=info['download_url'] if 'download_url' in info else '',
video_media_metadata=info[
'video_media_metadata'] if 'video_media_metadata' in info else None,
video_preview_metadata=info[
'video_preview_metadata'] if 'video_preview_metadata' in info else None)
else:
file_info = FileInfo(name=info['name'], id=info['file_id'], pid=info['parent_file_id'], type=False,
ctime=time.strptime(info['created_at'],
'%Y-%m-%dT%H:%M:%S.%fZ') if 'created_at' in info else time.time(),
update_time=time.strptime(info['updated_at'], '%Y-%m-%dT%H:%M:%S.%fZ'),
hidden=info.get('hidden'))
file_info_list.append(file_info)
return file_info_list
def tree(self, path='root', stdout=sys.stdout):
file_id = self.get_path_fid(path, update=False)
self.update_path_list(file_id)
if not file_id:
raise FileNotFoundError(path)
return self._tree.show(file_id, stdout=stdout)
def get_path_list(self, path, update=True):
file_id = self.get_path_fid(path, update=update)
try:
return self.get_fid_list(file_id, update=update)
except FileNotFoundError:
raise FileNotFoundError(path)
def get_fid_list(self, file_id, update=True):
if not file_id:
raise FileNotFoundError
try:
self.auto_update_path_list(update, file_id)
except NodeIDAbsentError:
return list(map(self.get_file_info, self._disk.get_file_list(file_id)))
if not self._tree.get_node(file_id):
return []
if file_id != 'root' and self._tree.get_node(file_id).data.type:
return [self._tree.get_node(file_id).data]
return [i.data for i in self._tree.children(file_id)]
def get_path_fid(self, path, file_id='root', update=True):
if str(path) in ('', '/', '\\', '.', 'root'):
return 'root'
path = AliyunpanPath(path)
flag = False
path_list = list(filter(None, path.split()))
if path_list[0] == 'root':
path_list = path_list[1:]
for i in path_list:
flag = False
node_list = self._tree.children(file_id)
if not node_list:
self.auto_update_path_list(update, file_id)
node_list = self._tree.children(file_id)
for j in node_list:
if i == j.tag:
flag = True
file_id = j.identifier
break
if not flag:
return False
if flag:
return file_id
return False
def get_path_node(self, path, update=True):
file_id = self.get_path_fid(path, update=update)
if file_id:
return self._tree.get_node(file_id)
return False
def get_path_parent_node(self, path, update=True):
file_id = self.get_path_fid(path, update=update)
if file_id:
node = self._tree.parent(file_id)
if node:
return node
return False
def auto_update_path_list(self, update=True, file_id=None):
if not update and file_id:
return self.update_path_list(file_id, depth=0)
elif update and len(self._tree) == 1:
return self.update_path_list()