def create_ding_tree():
#global ding_tree, dept_result #debug only
#连接数据库
db, cursor = connect_db('localhost', 'root', 'yoyoball', 'dingtalk')
ding_tree = Tree()
sql = "SELECT `id`, `name`, `parentid` FROM dingding_department_list"
cursor.execute(sql)
dept_result = cursor.fetchall()
#print dept_result #debug only
if dept_result != None and len(dept_result) > 0:
ding_tree.create_node('##ding_root##', '0') #先创建虚拟根
for i in range(len(dept_result)): #向虚拟根填充所有组织
#print dept_result[i] #debug only
#ding_tree.create_node(dept_result[i][1].decode('utf-8'), dept_result[i][0], '0000')
ding_tree.create_node(dept_result[i][1], dept_result[i][0], '0')
for i in range(len(dept_result)): #修改隶属关系
if dept_result[i][0] != '1' : #只要不是实根，就要修改隶属关系【钉钉中实根id为'1'且无上级部门，数据表dingding_department_list中存储id为'1'的部门上级为'0'】
if ding_tree.contains(dept_result[i][2]): #判断上级是否存在
ding_tree.move_node(dept_result[i][0], dept_result[i][2])
else: #没有上级的不修改
#print type(dept_result[i][2]), dept_result[i][2] #debug only
continue
#断开数据库
close_db(db)
#return ding_tree
return ding_tree.subtree('1')
def create_oa_tree():
#global oa_tree #debug only
#连接数据库
db, cursor = connect_db('localhost', 'root', 'yoyoball', 'test')
oa_tree = Tree()
sql = "SELECT `orgid`, `shortname`, `parentorgid` FROM groupinfo"
cursor.execute(sql)
dept_result = cursor.fetchall()
#print dept_result debug only
if dept_result != None and len(dept_result) > 0:
oa_tree.create_node('##oa_root##', '0000') #先创建虚拟根
for i in range(len(dept_result)): #向虚拟根填充所有组织
#print dept_result[i][1].decode('utf-8'), dept_result[i][0], dept_result[i][2]
oa_tree.create_node(dept_result[i][1], dept_result[i][0], '0000')
for i in range(len(dept_result)): #修改隶属关系
if dept_result[i][2] != '0000' : #OA中'001000'等的组织上级为'0000'即OA数据库中存在虚根，所以无需做此步骤
if oa_tree.contains(dept_result[i][2]): #判断上级是否存在
oa_tree.move_node(dept_result[i][0], dept_result[i][2])
else: #没有上级的不修改
continue
#断开数据库
close_db(db)
return oa_tree
def visualizeCtree(c_tree):
#bottom up build tree
tree = Tree()
tree.create_node("root", "root")
levels = sorted(c_tree.keys())
for level in levels:
for node_id, cluster in c_tree[level].items():
node_id = "{}.{}".format(level, node_id)
tree.create_node("{}".format(cluster["pattern"]), node_id, parent="root")
if level == 0:
for data in cluster["data"]:
tree.create_node("log.{}".format(data), "log.{}".format(data), parent=node_id)
else:
for data in cluster["data"]:
tree.move_node("{}.{}".format(level-1, data), node_id)
tree.show()
tree.save2file("./tree")
## How to use
##1. for log in logs:updateCTree, updatePatterns
##2. get the c_tree
##3. train all the logs to get the max level and estimate the best level we keep.
##4. train the cluster on different level
##要考虑到unmerged list的同步问题, 到时production考虑将c_tree存在redis里，然后同时flush到库里
class AcquisitionChain(object):
def __init__(self, parallel_prepare=False):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
self._presets_list = list()
self._parallel_prepare = parallel_prepare
self._device2one_shot_flag = weakref.WeakKeyDictionary()
@property
def nodes_list(self):
nodes_gen = self._tree.expand_tree()
nodes_gen.next() # first node is 'root'
return list(nodes_gen)
def add(self, master, slave):
self._device2one_shot_flag.setdefault(slave, False)
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if (slave_node.bpointer is not self._root_node
and master_node is not slave_node.bpointer):
raise RuntimeError(
"Cannot add acquisition device %s to multiple masters, current master is %s"
% (slave, slave_node._bpointer))
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(tag=master.name,
identifier=master,
parent="root")
if slave_node is None:
slave_node = self._tree.create_node(tag=slave.name,
identifier=slave,
parent=master)
else:
self._tree.move_node(slave, master)
slave.parent = master
def add_preset(self, preset):
self._presets_list.append(preset)
def set_stopper(self, device, stop_flag):
"""
By default any top master device will stop the scan.
In case of several top master, you can define which one won't
stop the scan
"""
self._device2one_shot_flag[device] = not stop_flag
def __iter__(self):
if len(self._tree) > 1:
return AcquisitionChainIter(
self, parallel_prepare=self._parallel_prepare)
else:
return iter(())
class AcquisitionChain(object):
def __init__(self):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
def add(self, master, slave):
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer:
raise RuntimeError(
"Cannot add acquisition device %s to multiple masters, current master is %s"
% (slave, slave_node._bpointer)
)
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root")
if slave_node is None:
slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master)
else:
self._tree.move_node(slave_node, master_node)
def _execute(self, func_name):
tasks = list()
prev_level = None
for dev in reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]):
node = self._tree.get_node(dev)
level = self._tree.depth(node)
if prev_level != level:
gevent.joinall(tasks)
tasks = list()
func = getattr(dev, func_name)
tasks.append(gevent.spawn(func))
gevent.joinall(tasks)
def prepare(self, dm, scan_info):
# self._devices_tree = self._get_devices_tree()
for master in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionMaster)):
del master.slaves[:]
for dev in self._tree.get_node(master).fpointer:
master.slaves.append(dev)
dm_prepare_task = gevent.spawn(dm.prepare, scan_info, self._tree)
self._execute("_prepare")
dm_prepare_task.join()
def start(self):
self._execute("_start")
for acq_dev in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionDevice)):
acq_dev.wait_reading()
dispatcher.send("end", acq_dev)
def test_loop(self):
tree = Tree()
tree.create_node('a', 'a')
tree.create_node('b', 'b', parent='a')
tree.create_node('c', 'c', parent='b')
tree.create_node('d', 'd', parent='c')
try:
tree.move_node('b', 'd')
except LoopError:
pass
def merge(T, alpha, beta):
"""
merge is a function that mergers two nodes of tree T, changing node beta to be a child of node alpha, where alpha and beta are sisters.
param T: the given tree
param alpha: the target node of T
param beta: the merged node of T
return: the new tree after merging operation.
"""
TT = Tree(T.subtree(T.root), deep=True)
TT.move_node(beta, alpha)
return TT
class AcquisitionChain(object):
def __init__(self, parallel_prepare=False):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
self._presets_list = list()
self._parallel_prepare = parallel_prepare
self._device2one_shot_flag = weakref.WeakKeyDictionary()
@property
def nodes_list(self):
nodes_gen = self._tree.expand_tree()
nodes_gen.next() # first node is 'root'
return list(nodes_gen)
def add(self, master, slave):
self._device2one_shot_flag.setdefault(slave, False)
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if(slave_node.bpointer is not self._root_node and
master_node is not slave_node.bpointer):
raise RuntimeError("Cannot add acquisition device %s to multiple masters, current master is %s" % (
slave, slave_node._bpointer))
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(
tag=master.name, identifier=master, parent="root")
if slave_node is None:
slave_node = self._tree.create_node(
tag=slave.name, identifier=slave, parent=master)
else:
self._tree.move_node(slave, master)
slave.parent = master
def add_preset(self, preset):
self._presets_list.append(preset)
def set_stopper(self, device, stop_flag):
"""
By default any top master device will stop the scan.
In case of several top master, you can define which one won't
stop the scan
"""
self._device2one_shot_flag[device] = not stop_flag
def __iter__(self):
if len(self._tree) > 1:
return AcquisitionChainIter(self, parallel_prepare=self._parallel_prepare)
else:
return iter(())
def get_intersection_tree(T1, T2):
T = Tree(tree=T1, deep=True)
T1_bfs = [n for n in T1.expand_tree(mode=1)]
T2_bfs = [n for n in T2.expand_tree(mode=1)]
for nid in T1_bfs:
X = set(get_leaf_node_ids_for_node(T, nid))
diff = min([len(X.symmetric_difference(set( \
get_leaf_node_ids_for_node(T2,i)))) \
for i in T2_bfs])
if diff != 0:
par = T.parent(nid).identifier
for c in T.children(nid):
T.move_node(c.identifier, par)
T.remove_subtree(nid)
return T
def combine(T, alpha, beta):
"""
combine is a function that combines two nodes of tree T, changing nodes alpha and beta to be children of a new node gama, where alpha and beta are sisters.
param T: the given tree
param alpha: the target node of T
param beta: the target node of T
return: the new tree after combining operation.
"""
TT = Tree(T.subtree(T.root), deep=True)
new_id = max(ids_of_all_nodes(T)) + 1
parent_id = TT.get_node(alpha).bpointer
TT.create_node(tag=new_id, identifier=new_id, parent=parent_id, data=None)
TT.move_node(alpha, new_id)
TT.move_node(beta, new_id)
return TT
def build_hierarchy_tree(self, nodes):
"""
Build hierarchy tree from list of nodes.
"""
tree = Tree()
# root node
tree.create_node('root', 0)
# create all nodes
for i in nodes:
name, id, parent = self.parse_hierarchy_node(i)
tree.create_node(name, id, parent=0)
# assign parents
for i in nodes:
name, id, parent = self.parse_hierarchy_node(i)
if parent is not None:
tree.move_node(id, parent)
return tree
for z in d:
path = walkTree(tree, z, path + z)
return path
input = list(map(lambda x: x.strip(), open("test_input.txt").readlines()))
tree = Tree()
tree.create_node("root", "root")
# first figure out how many steps there are and then sort them
# by their name
for lines in input:
(l1, l2) = (lines[5], lines[36])
print(lines)
if tree.contains(l1) and tree.contains(l2):
tree.move_node(l2, l1)
elif tree.contains(l1) and not tree.contains(l2):
tree.create_node(l2, l2, parent=l1)
elif not tree.contains(l1) and tree.contains(l2):
# get the root for l2 and make that the root for l1
# then move l2 under l1
tree.create_node(l1, l1, parent=tree.parent(l2))
tree.move_node(l2, l1)
else:
tree.create_node(l1, l1, parent="root")
tree.create_node(l2, l2, parent=l1)
tree.show()
print(walkTree(tree, 'root', ''))
class RMQueue(object):
__metaclass__ = Singleton
def __init__(self):
self.tree = Tree()
self.MAX_METRIC_COUNT = 12
self.CAL_INTERVAL_IN_SECOND = 2 * 60 * 60
self.conf = conf.Config("./conf/config.json")
def set_stat_interval(self, interval):
self.CAL_INTERVAL_IN_SECOND = interval
def set_system_memory(self, size):
root = self.get_root()
root.data.set_abs_memory(float(size))
def create_queue(self, name=None, parent=None):
data = QueueData()
self.tree.create_node(name, name, parent, data)
def display(self):
self.tree.show()
def display_score(self, queue=None, depth=0, table=None, printer=None):
flag = False
if queue is None:
queue = self.get_root()
flag = True
table = PrettyTable([
"QUEUE", "PENDING AVG", "PENDING DIV", "MEMORY USAGE AVG(Q)",
"MEMORY USAGE AVG(C)", "MEMORY USAGE DIV", "ABS CAPACITY"
])
if table is not None:
table.add_row([
queue.tag, 0 if queue.data.get_pending() == 0 else "%.3f" %
queue.data.get_pending(),
0 if queue.data.get_pending_div() == 0 else "%.3f" %
queue.data.get_pending_div(),
0 if queue.data.get_mem_usage() == 0 else "%.3f" %
queue.data.get_mem_usage(),
0 if queue.data.cal_queue_memory_usage() == 0 else "%.3f" %
queue.data.cal_queue_memory_usage(),
0 if queue.data.get_mem_usage_div() == 0 else "%.3f" %
queue.data.get_mem_usage_div(),
str(0 if queue.data.get_abs_capacity() == 0 else "%.3f" %
queue.data.get_abs_capacity()) + " %"
])
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.display_score(child, depth + 1, table)
if flag:
if printer is None:
print('------------' + utils.get_str_time() +
' SCORE ----------')
print table
else:
printer.write('\n------------' + utils.get_str_time() +
' SCORE ----------\n')
printer.write(str(table))
def display_prediction(self,
queue=None,
depth=0,
table=None,
printer=None):
flag = False
if queue is None:
queue = self.get_root()
flag = True
table = PrettyTable([
"QUEUE", "DESIRED CAPACITY(Q)", "DESIRED CAPACITY(C)",
"ABS CAPACITY"
])
if table is not None:
table.add_row([
queue.tag,
str(0 if queue.data.wish.capacity == 0 else "%.3f" %
(100 * queue.data.wish.capacity)) + " %",
0 if queue.data.wish.abs_capacity == 0 else "%.3f" %
queue.data.wish.abs_capacity,
str(0 if queue.data.config.abs_capacity == 0 else "%.3f" %
queue.data.config.abs_capacity) + " %"
])
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.display_prediction(child, depth + 1, table)
if flag:
if printer is None:
print('------------' + utils.get_str_time() +
' PREDICTION ----------')
print table
else:
printer.write('\n------------' + utils.get_str_time() +
' PREDICTION ----------\n')
printer.write(str(table))
def write_score(self, path):
FileOperator.touch(path)
with open(path, 'a') as f:
self.display_score(printer=f)
def request_score(self, queue=None):
if queue is None:
queue = self.get_root()
postData = {
'queue': queue.tag,
'pending': queue.data.get_pending(),
'pending_div': queue.data.get_pending_div(),
'memory_usage': queue.data.get_mem_usage(),
'memory_usage_div': queue.data.get_mem_usage_div(),
'abs_capacity': queue.data.get_abs_capacity()
}
requests.post(str(self.conf.es_rest_address) +
str(self.conf.es_index) + "score",
data=json.dumps(postData))
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.request_score(child)
def request_prediction(self, queue=None):
if queue is None:
queue = self.get_root()
postData = {
'queue': queue.tag,
'wish_capacity': queue.data.wish.capacity,
'wish_abs_capacity': queue.data.wish.abs_capacity,
'abs_capacity': queue.data.config.abs_capacity
}
requests.post(str(self.conf.es_rest_address) +
str(self.conf.es_index) + "prediction",
data=json.dumps(postData))
if not self.is_leaf(queue.tag):
children = self.tree.children(queue.tag)
for child in children:
self.request_prediction(child)
def write_prediction(self, path):
FileOperator.touch(path)
with open(path, 'a') as f:
self.display_prediction(printer=f)
def add_job(self, job, qname):
queue = self.tree.get_node(qname)
if queue.is_leaf():
queue.data.add_job(job)
else:
print("Cannot add jobs to parent queue", queue.tag,
queue.identifier)
def add_metric(self, qname):
queue = self.tree.get_node(qname)
queue.data.add_metric(queue.cur_metric)
if len(queue.data.metrics) > RMQueue.MAX_METRIC_COUNT:
del queue.data.metrics[0]
def remove_queue(self, qname):
self.tree.remove_node(qname)
def move_queue(self, src, dest):
self.tree.move_node(src, dest)
def get_queue(self, qname):
return self.tree.get_node(qname)
def get_root(self):
return self.get_queue('root')
def is_leaf(self, qname):
queue = self.tree.get_node(qname)
return queue.is_leaf()
def cal_slowdown(self, queue=None):
if queue is None:
queue = self.get_root()
avg_slowdown = 0.0
if queue.is_leaf():
job_count = len(queue.data.jobs)
for i in list(range(job_count)):
job = queue.data.jobs[i]
slowdown = (job.wait_time + job.run_time) / job.run_time
avg_slowdown += slowdown / job_count
queue.data.set_job_count(job_count)
queue.data.cur_metric.slowdown = avg_slowdown
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_slowdown(child)
job_count = 0
for child in children:
job_count += child.data.get_job_count()
queue.data.set_job_count(job_count)
if job_count == 0:
queue.data.cur_metric.slowdown = avg_slowdown
return avg_slowdown
for child in children:
avg_slowdown += child.data.get_job_count(
) * child.data.get_slowdown() / job_count
queue.data.cur_metric.slowdown = avg_slowdown
return queue.data.get_slowdown()
def cal_pending(self, queue=None):
if queue is None:
queue = self.get_root()
if queue.is_leaf():
if len(queue.data.pendings) > 0:
queue.data.cur_metric.pending = np.mean(queue.data.pendings)
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_pending(child)
queue.data.cur_metric.pending += child.data.get_pending()
return queue.data.get_pending()
def cal_pending_division(self, queue=None):
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_pending_division(child)
count = len(children)
avg_pending = queue.data.get_pending() * 1.0 / count
square_sum = 0.0
for child in children:
square_sum += np.square(child.data.get_pending() - avg_pending)
division = np.sqrt(square_sum / count)
queue.data.cur_metric.pending_div = division
return division
def cal_slowdown_division(self, queue=None):
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_slowdown_division(child)
square_sum = 0.0
count = len(children)
for child in children:
square_sum += np.square(child.data.get_slowdown() -
queue.data.get_slowdown())
division = np.sqrt(square_sum / count)
queue.data.cur_metric.slowdown_div = division
return division
def cal_memory_usage(self, queue=None):
if queue is None:
queue = self.get_root()
if queue.is_leaf():
capacity = queue.data.get_abs_capacity()
memory_usage = 0.0
if capacity != 0:
memory_usage = 100.0 * queue.data.cal_queue_memory_usage(
) / capacity
queue.data.set_mem_usage(memory_usage)
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_memory_usage(child)
abs_memory_usage = 0
for child in children:
abs_memory_usage += child.data.get_abs_memory_usage()
queue.data.set_mem_usage(100.0 * abs_memory_usage /
queue.data.get_abs_capacity())
return queue.data.get_mem_usage()
def cal_mem_usage_division(self, queue=None):
if queue is None:
queue = self.get_root()
std_division = 0.0
if self.is_leaf(queue.tag):
queue.data.cur_metric.mem_usage_div = std_division
return std_division
else:
children = self.tree.children(queue.tag)
for child in children:
self.cal_mem_usage_division(child)
count = len(children)
total_mem_usage = 0
for child in children:
total_mem_usage += child.data.get_mem_usage()
avg_mem_usage = total_mem_usage / count
square_sum = 0
for child in children:
square_sum += np.square(child.data.get_mem_usage() -
avg_mem_usage)
std_division = np.sqrt(square_sum / count)
queue.data.cur_metric.mem_usage_div = std_division
return std_division
def cal_abs_capacity_bottom_up(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = 0.0
for child in children:
self.cal_abs_capacity_bottom_up(child)
abs_capacity += child.data.get_abs_capacity()
queue.data.set_abs_capacity(abs_capacity)
def cal_desired_abs_capacity_bottom_up(self, queue=None, delim=None):
if queue is None:
queue = self.get_root()
delim = 1
if self.is_leaf(queue.tag):
queue.data.wish.capacity = queue.data.wish.abs_capacity / delim
else:
children = self.tree.children(queue.tag)
abs_capacity = 0.0
for child in children:
self.cal_desired_abs_capacity_bottom_up(
child, queue.data.config.abs_capacity * delim / 100)
abs_capacity += child.data.wish.abs_capacity
queue.data.wish.capacity = abs_capacity / delim
def cal_abs_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.set_abs_capacity(100.0)
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_capacity(queue.data.get_abs_capacity() *
child.data.get_capacity() / 100.0)
self.cal_abs_capacity_top_down(child)
def cal_desired_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.wish.capacity = 100.0
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.wish.abs_capacity
for child in children:
child.data.wish.capacity = child.data.config.capacity
if abs_capacity == 0:
child.data.wish.capacity = 0
else:
child.data.wish.capacity = child.data.wish.abs_capacity / abs_capacity * 100.0
self.cal_desired_capacity_top_down(child)
def cal_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.get_abs_capacity()
for child in children:
if abs_capacity == 0:
child.data.set_capacity(0)
else:
child.data.set_capacity(child.data.get_abs_capacity() /
abs_capacity * 100)
self.cal_capacity_top_down(child)
def cal_abs_memory_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.cal_totalMb_mean()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_memory(queue.data.get_abs_memory() *
child.data.get_capacity() / 100)
self.cal_abs_memory_top_down(child)
def clear_mus_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_queue_memory_usage()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_mus_top_down(child)
def clear_jobs_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_jobs()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_jobs_top_down(child)
def clear_pendings_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_pendings()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_pendings_top_down(child)
def score(self):
# self.cal_abs_capacity_bottom_up()
# self.cal_capacity_top_down()
# self.cal_abs_memory_top_down()
# self.cal_slowdown()
# self.cal_slowdown_division()
self.cal_pending()
self.cal_pending_division()
self.cal_memory_usage()
self.cal_mem_usage_division()
# self.clear_jobs_top_down()
# self.clear_pendings_top_down()
# self.clear_mus_top_down()
def predict(self):
self.cal_desired_abs_capacity_bottom_up()
print(tree[child].tag)
print(sep + "OOhh~ new members join Jill's family:")
new_tree = Tree()
new_tree.create_node("n1", 1) # root node
new_tree.create_node("n2", 2, parent=1)
new_tree.create_node("n3", 3, parent=1)
tree.paste("jill", new_tree)
tree.show()
print(sep + "They leave after a while:")
tree.remove_node(1)
tree.show()
print(sep + "Now Jill moves to live with Grand-x-father Harry:")
tree.move_node("jill", "harry")
tree.show()
print(sep + "A big family for George to send message to the oldest Harry:")
for node in tree.rsearch("george"):
print(tree[node].tag)
########NEW FILE########
__FILENAME__ = folder_tree
#!/usr/bin/env python
# A file folder scanner contributed by @holger
#
# You can spicify the scanned folder and file pattern by changing rootPath
# and pattern variables
#
__author__ = "holger"
sub_t = tree.subtree('diane')
sub_t.show()
print('\n')
print("#"*4 + "Children of Diane")
print tree.is_branch('diane')
print('\n')
print("#"*4 + "OOhh~ new members enter Jill's family")
new_tree = Tree()
new_tree.create_node("n1", 1) # root node
new_tree.create_node("n2", 2, parent=1)
new_tree.create_node("n3", 3, parent=1)
tree.paste('jill', new_tree)
tree.show()
print('\n')
print("#"*4 + "We are sorry they are gone accidently :(")
tree.remove_node(1)
tree.show()
print('\n')
print("#"*4 + "Now Jill moves to live with Grand-x-father Harry")
tree.move_node('jill', 'harry')
tree.show()
print('\n')
print("#"*4 + "A big family for George to talk to Grand-x-father Harry")
for node in tree.rsearch('george', filter=lambda x: x != 'harry'):
print node
print('\n')
class Parser():
"""
Parser class for parsing a NAL xml file
"""
# All of the roots are coming from the TM queries
# Other items in NAL may be considered phenotypes or chemicals
# But are not being labelled for our purposes
__root2label = {
322:"Phenotype", 156:"Phenotype", 319:"Phenotype",
7812:"Chemical", 8:"Chemical", 264:"Chemical",
858:"Plant"
}
def __init__(self, namespace="usda_nal_thesaurus"):
self.namespace = namespace
self.tree = Tree()
self.tree.create_node("Root","root")
self.name2id = {}
def parse(self, xmlFile):
"""
parsing function that parses an xml file
"""
nodes = {}
tree = ET.parse(xmlFile)
root = tree.getroot() # THESAURUS node
for concept in root: # Parse xml with nodes as concepts
iden = "NAL:"+concept.find("TNR").text # node source ID
nodes[iden], name = self.parseNode(concept) # Parse the node from xml file
self.tree.create_node(tag=name, identifier=iden, parent='root') # create the node in the tree
self.name2id[name] = iden # For mapping edges
# Iterate through twice because parent/child relationships are connected via name not id
# in xml file and file is sorted alphabetically
for nodeID, props in nodes.items(): # Add edges to node as a list of tuples
nodes[nodeID]["edges"] = self.parseEdges(nodeID,props)
for node,label in self.__root2label.items(): # Add specific labels to nodes
sub = self.tree.subtree("NAL:{}".format(node))
for i in sub.all_nodes():
iden = i.identifier.split(".")[0]
nodes[iden]["labels"].add(label)
return nodes
def parseNode(self,node):
"""
Parse each node from the xml file
"""
labels = set()
name = node.find("DESCRIPTOR").text
synonyms = extractElem(node, "UF") # Used for
parents = extractElem(node, "BT") # Broader term
children = extractElem(node, "NT") # Narrow term
# associated = extractElem(node,"RT") # Related term
# categories = extractElem(node, "SC") # Subject category
# for cat in categories:
# item = cat.split(" ")[0]
# if item in self.labels.keys():
# labels.add(self.labels[item])
labels.add(self.namespace)
return {"name": name, "synonyms": list(synonyms), "parents": parents,
"children": children, "labels": labels}, name
def parseEdges(self, nodeID, props):
"""
Add edges to nodes.
Some nodes have multiple parents
"""
edges = []
multiParent = 0
for name in props["children"]:
edges.append(("has_child",self.name2id[name]))
for name in props["parents"]:
parentID = self.name2id[name]
if multiParent == 0:
self.tree.move_node(nodeID,parentID)
else:
self.tree.create_node(tag=props["name"],identifier=nodeID+".{}".format(multiParent),parent=parentID)
edges.append(("is_a",parentID))
multiParent += 1
return edges
def toDEP(self):
###############################
# Etape 1 : construction du head_tree
# parcours en largeur de tree afin de récupérer chaque id_node
# pour chaque profondeur (init à 0) _! sans compter !_ les feuilles (EDUs)
nodes_depth = [-1] * self.tree.size()
for i in xrange(self.tree.size()):
id_nodes = [0]
depth = [999] * self.tree.size()
while id_nodes: # False if empty
id_node = id_nodes.pop(0)
node = self.tree.get_node(id_node)
if node.bpointer != None:
node_parent = self.tree.get_node(node.bpointer)
depth[node.identifier] = depth[node_parent.identifier] + 1
else:
depth[node.identifier] = 0
if id_node == i:
# print 'noeud ',i,' en profondeur', depth[node.identifier]
if node.fpointer:
nodes_depth[i] = depth[i]
break
if node.fpointer:
id_nodes.append(node.fpointer[0])
id_nodes.append(node.fpointer[1])
# print nodes_depth
id_nodes_depth = []
for d in xrange(self.tree.depth()):
id_nodes_depth.append([])
for i in xrange(self.tree.size()):
if nodes_depth[i] == d:
id_nodes_depth[d].append(i)
# print id_nodes_depth
#
# construction du head_tree
head_tree = [-1] * self.treeNS.size()
# pour chaque noeud (non EDU/feuille) en partant de la plus grande profondeur dans l'arbre
for d in range(len(id_nodes_depth) - 1, -1, -1):
for id_node in id_nodes_depth[d]:
node = self.treeNS.get_node(id_node)
node_left = self.treeNS.get_node(node.fpointer[0])
node_right = self.treeNS.get_node(node.fpointer[1])
if node_left.tag == "N":
if head_tree[node_left.identifier] == -1:
identifier = node_left.identifier
else:
identifier = head_tree[node_left.identifier]
else:
if head_tree[node_right.identifier] == -1:
identifier = node_right.identifier
else:
identifier = head_tree[node_right.identifier]
head_tree[id_node] = identifier
# print head_tree
###############################
# Etape 2 : construction du DEP
#
# construction du DEP
# init
# root est le premier noeud de head
# pour chaque EDU son père est le root dans DEP
dep_tree = Tree()
id_root = head_tree[0]
root = self.tree.get_node(id_root)
# dep_tree.create_node(root.tag, root.identifier)
dep_tree.create_node(root.tag, root.identifier)
for id_EDU in xrange(len(head_tree)):
if head_tree[id_EDU] == -1 and id_EDU != id_root:
node = self.tree.get_node(id_EDU)
# dep_tree.create_node(node.tag, node.identifier, parent=id_root)
# dep_tree.create_node(str(id_EDU), node.identifier, parent=id_root)
dep_tree.create_node(node.tag, node.identifier, parent=id_root)
# print '//////////////////////'
# print 'EDU', id_root
# pour chaque EDU
for id_EDU in xrange(len(head_tree)):
if head_tree[id_EDU] == -1 and id_EDU != id_root:
EDU_NS = self.treeNS.get_node(id_EDU)
# print '.......................'
# print 'EDU', id_EDU
# print 'TAG', EDU_NS.tag
if EDU_NS.tag == "N":
# parcours en largeur jusqu'à trouver un S avec un head donc qui soit pas EDU
id_nodes = [EDU_NS.identifier]
visited = [False] * self.treeNS.size()
while id_nodes:
id_node = id_nodes.pop(0)
EDU = self.tree.get_node(id_node)
# print 'visited EDU', EDU.identifier
visited[EDU.identifier] = True
# cas d'arret
head_EDU = head_tree[EDU.identifier] == -1
head_EDU = False
node_tag = self.treeNS.get_node(EDU.identifier).tag
# print ' head_EDU', head_EDU
# print ' node_tag', node_tag
if not head_EDU and node_tag == "S":
break
if EDU.bpointer:
if not visited[EDU.bpointer]:
id_nodes.append(EDU.bpointer)
if EDU.fpointer: # sécurité
if not visited[EDU.fpointer[0]]:
id_nodes.append(EDU.fpointer[0])
if not visited[EDU.fpointer[1]]:
id_nodes.append(EDU.fpointer[1])
# puis ajouter au DEP comme enfant du head du parent du noeud S
id_head = head_tree[EDU.bpointer]
# si parent S
else:
# parcours en largeur des ancêtre jusqu'à trouver un ancêtre avec un head
parent = self.treeNS.get_node(EDU_NS.bpointer)
id_head = head_tree[parent.identifier]
# puis ajouter au DEP comme enfant de ce head
if id_EDU != id_head:
dep_tree.move_node(id_EDU, id_head)
EDU = self.tree.get_node(id_EDU)
# print '---- ajout de',EDU.identifier,' à',id_head
# if id_EDU == id_head:
# dep_tree.show()
return dep_tree
class RMQueue(metaclass=Singleton):
MAX_METRIC_COUNT = 12
CAL_INTERVAL_IN_SECOND = 2 * 60 * 60 # 2hours
def __init__(self):
self.tree = Tree()
def set_stat_interval(self, interval):
RMQueue.CAL_INTERVAL_IN_SECOND = interval
def set_system_memory(self, size):
root = self.get_root()
root.data.set_abs_memory(float(size))
def create_queue(self, name=None, parent=None):
data = QueueData()
self.tree.create_node(name, name, parent, data)
def display(self):
self.tree.show()
def display_score_old(self, queue=None, depth=0):
if queue is None:
queue = self.get_root()
print('------------' + utils.get_str_time() + ' SCORE ----------')
print(24 * ' ' + ' SLOWDOWN MEMORY USAGE ')
print('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV')
if depth >= 0:
print(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_slowdown(), \
' %8.3f ' % queue.data.get_slowdown_div(), \
' %8.3f' % queue.data.get_mem_usage(), \
' %8.3f' % queue.data.get_mem_usage_div())
"""
print(queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \
'div: %.3f)' % queue.data.get_slowdown_div(), \
'(mem usage: %.3f' % queue.data.get_mem_usage(), \
'div: %.3f)' % queue.data.get_mem_usage_div())
"""
else:
print('-'*depth + queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \
'div: %.3f)' % queue.data.get_slowdown_div(), \
'(mem usage: %.3f' % queue.data.get_mem_usage(), \
'div: %.3f)' % queue.data.get_mem_usage_div())
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.display_score(child, depth + 2)
def display_score(self, queue=None, depth=0):
if queue is None:
queue = self.get_root()
print('------------' + utils.get_str_time() + ' SCORE ----------')
print(24 * ' ' + ' PENDING MEMORY USAGE ')
print('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV')
if depth >= 0:
print(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_pending(), \
' %8.3f ' % queue.data.get_pending_div(), \
' %8.3f' % queue.data.get_mem_usage(), \
' %8.3f' % queue.data.get_mem_usage_div())
else:
print('-'*depth + queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \
'div: %.3f)' % queue.data.get_slowdown_div(), \
'(mem usage: %.3f' % queue.data.get_mem_usage(), \
'div: %.3f)' % queue.data.get_mem_usage_div())
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.display_score(child, depth + 2)
def display_prediction(self, queue=None, depth=0):
if queue is None:
queue = self.get_root()
print('------------' + utils.get_str_time() +
' PREDICTION ----------')
print('QUEUE NAME DESIRED CAPACITY')
if depth >= 0:
print(queue.tag + (22 - len(queue.tag)) * ' ',
' %8.3f' % queue.data.wish.capacity)
# print(queue.tag, 'desired capacity: %.3f' % queue.data.wish.capacity)
else:
print('-' * depth + queue.tag,
'desired capacity: %.3f' % queue.data.wish.capacity)
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.display_prediction(child, depth + 2)
def write_score(self, path):
with open(path, 'a') as f:
self.write_score_top_down(output=f)
def write_score_top_down_old(self, queue=None, depth=0, output=None):
if queue is None:
queue = self.get_root()
output.writelines(
('\n---------', utils.get_str_time(), ' SCORE ---------\n'))
output.writelines(24 * ' ' +
' SLOWDOWN MEMORY USAGE\n')
output.writelines('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV\n')
if depth >= 0:
output.writelines(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_slowdown() + \
' %8.3f ' % queue.data.get_slowdown_div() + \
' %8.3f' % queue.data.get_mem_usage() + \
' %8.3f' % queue.data.get_mem_usage_div() + '\n')
"""
output.writelines( (queue.tag, ' (slowdown: %.3f' % queue.data.get_slowdown(), \
' div: %.3f)' % queue.data.get_slowdown_div(), \
' (mem usage: %.3f' % queue.data.get_mem_usage(), \
' div: %.3f)' % queue.data.get_mem_usage_div(), '\n'))
"""
else:
output.writelines(('-'*depth + queue.tag, ' (slowdown: %.3f' % queue.data.get_slowdown(), \
' div: %.3f)' % queue.data.get_slowdown_div(), \
' (mem usage: %.3f' % queue.data.get_mem_usage(), \
' div: %.3f)' % queue.data.get_mem_usage_div(), '\n'))
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.write_score_top_down(child, depth + 2, output)
def write_score_top_down(self, queue=None, depth=0, output=None):
if queue is None:
queue = self.get_root()
output.writelines(
('\n---------', utils.get_str_time(), ' SCORE ---------\n'))
output.writelines(24 * ' ' +
' PENDING MEMORY USAGE\n')
output.writelines('QUEUE NAME' + 16 * ' ' +
' AVG DIV AVG DIV\n')
output.writelines(queue.tag + (22 - len(queue.tag))*' ' + \
'%8.3f' % queue.data.get_pending() + \
' %8.3f ' % queue.data.get_pending_div() + \
' %8.3f' % queue.data.get_mem_usage() + \
' %8.3f' % queue.data.get_mem_usage_div() + '\n')
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.write_score_top_down(child, depth + 2, output)
def write_prediction(self, path):
with open(path, 'a') as f:
self.write_prediction_top_down(output=f)
def write_prediction_top_down(self, queue=None, depth=0, output=None):
if queue is None:
queue = self.get_root()
output.writelines(('\n---------', utils.get_str_time(),
' PREDICTION---------\n'))
output.writelines('QUEUE NAME DESIRED CAPACITY\n')
if depth >= 0:
output.writelines(queue.tag + (22 - len(queue.tag)) * ' ' +
' %8.3f' % queue.data.wish.capacity + '\n')
# output.writelines( (queue.tag, ' desired capacity: %.3f' % queue.data.wish.capacity, '\n'))
else:
output.writelines(('-'*depth + queue.tag, \
' desired capacity: %.3f' % queue.data.wish.capacity, '\n'))
if self.is_leaf(queue.tag) == False:
children = self.tree.children(queue.tag)
for child in children:
self.write_prediction_top_down(child, depth + 2, output)
def add_job(self, job, qname):
queue = self.tree.get_node(qname)
if queue.is_leaf():
queue.data.add_job(job)
else:
print("Canot add jobs to parent queue", queue.tag,
queue.identifier)
def add_metric(self, qname):
queue = self.tree.get_node(qname)
queue.data.add_metric(queue.cur_metric)
if len(queue.data.metrics) > RMQueue.MAX_METRIC_COUNT:
del queue.data.metrics[0]
def remove_queue(self, qname):
"""
Remove a queue indicated by 'qname'; all the successors are
removed as well.
Return the number of removed nodes.
"""
self.tree.remove_node(qname)
def move_queue(self, src, dest):
"""
Move a queue indicated by @src parameter to be a child of
@dest.
"""
self.tree.move_node(src, dest)
def get_queue(self, qname):
return self.tree.get_node(qname)
def get_root(self):
return self.get_queue('root')
def is_leaf(self, qname):
queue = self.tree.get_node(qname)
return queue.is_leaf()
def cal_slowdown(self, queue=None):
"""
if current queue is a leaf queue:
calculate the average slowdown in is jobs.
else:
calculate the average slowdown of its chilren;
calculate the average slowdown of current queue through its chilren's average slowdown.
"""
if queue is None:
queue = self.get_root()
avg_slowdown = 0.0
if queue.is_leaf():
job_count = len(queue.data.jobs)
for i in list(range(job_count)):
job = queue.data.jobs[i]
slowdown = (job.wait_time + job.run_time) / job.run_time
avg_slowdown += slowdown / job_count
queue.data.set_job_count(job_count)
queue.data.cur_metric.slowdown = avg_slowdown
else: # parent queue
# First, get its all chilren queue, and call each child's cal_slowdown function
children = self.tree.children(queue.tag)
for child in children:
self.cal_slowdown(child)
# Second, get the job count
job_count = 0
for child in children:
job_count += child.data.get_job_count()
queue.data.set_job_count(job_count)
# Finally, calculate the average slowdown of the queue
if job_count == 0:
queue.data.cur_metric.slowdown = avg_slowdown
return avg_slowdown
for child in children:
avg_slowdown += child.data.get_job_count(
) * child.data.get_slowdown() / job_count
queue.data.cur_metric.slowdown = avg_slowdown
return queue.data.get_slowdown()
def cal_pending(self, queue=None):
"""
if current queue is a leaf queue:
calculate the average pending count in is pendings.
else:
calculate the average pending of its chilren;
calculate the pending of current queue through the sum all of its chilren's pending.
"""
if queue is None:
queue = self.get_root()
if queue.is_leaf():
queue.data.cal_leaf_pending()
else: # parent queue
# First, get its all chilren queue, and call each child's cal_pending function
# Second, get the sum of all its children pending
children = self.tree.children(queue.tag)
for child in children:
self.cal_pending(child)
queue.data.cur_metric.pending += child.data.get_pending()
return queue.data.get_pending()
def cal_pending_division(self, queue=None):
"""
if current queue is a leaf queue:
stdDivision is zero.
else:
calculate the standard division of its chilren;
calculate the standard division of current queue through its chilren's average pending.
"""
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else: # parent queue
children = self.tree.children(queue.tag)
# First, get its all chilren queue, and call each child's calSlowDown function
for child in children:
self.cal_pending_division(child)
# Second, calculate the square sum of division
count = len(children)
avg_pending = queue.data.get_pending() * 1.0 / count
squareSum = 0.0
for child in children:
squareSum += np.square(child.data.get_pending() - avg_pending)
# Finally, calculate the standard division of the queue
# if count == 0:
# queue.data.cur_metric.slowdown_div = division
# return division
division = np.sqrt(squareSum / count)
queue.data.cur_metric.pending_div = division
return division
def cal_slowdown_division(self, queue=None):
"""
if current queue is a leaf queue:
stdDivision is zero.
else:
calculate the standard division of its chilren;
calculate the standard division of current queue through its chilren's average slowdown.
"""
if queue is None:
queue = self.get_root()
division = 0.0
if self.is_leaf(queue.tag):
return division
else: # parent queue
children = self.tree.children(queue.tag)
# First, get its all chilren queue, and call each child's calSlowDown function
for child in children:
self.cal_slowdown_division(child)
# Second, calculate the square sum of division
squareSum = 0.0
count = len(children)
for child in children:
squareSum += np.square(child.data.get_slowdown() -
queue.data.get_slowdown())
# Finally, calculate the standard division of the queue
# if count == 0:
# queue.data.cur_metric.slowdown_div = division
# return division
division = np.sqrt(squareSum / count)
queue.data.cur_metric.slowdown_div = division
return division
def cal_memory_usage_old(self, queue=None):
"""
if current queue is a leaf queue:
MemoryUsage is the (sum of job memorySeconds )/(self.absMemory * CAL_INTERVAL_IN_SECOND)
Get absUsedMemory by self.memoryUsage * self.absMemory
else:
calculate the memory usage of its chilren;
calculate the absolute used memory of the queue.
MemoryUsage = absUsedMemory / absMemory
"""
if queue is None:
queue = self.get_root()
memory_usage = 0.0
if queue.is_leaf():
total_memory_seconds = queue.data.cal_leaf_mem_second()
total_memory_capacity = queue.data.get_abs_memory(
) * RMQueue.CAL_INTERVAL_IN_SECOND
memory_usage = 1.0 * total_memory_seconds / total_memory_capacity
queue.data.set_mem_usage(memory_usage)
queue.data.cal_abs_used_memory()
else: # parent queue
# First, get its all chilren queue, and call each child's calMemoryUsage function
children = self.tree.children(queue.tag)
for child in children:
self.cal_memory_usage_old(child)
# Second, calculate the absUsedMemory of current queue
abs_used_memory = 0
for child in children:
abs_used_memory += child.data.get_abs_used_memory()
queue.data.set_abs_used_memory(abs_used_memory)
# Finally, calculate the memory usage of the queue
queue.data.set_mem_usage(1.0 * queue.data.get_abs_used_memory() /
queue.data.get_abs_memory())
return queue.data.get_mem_usage()
def cal_memory_usage(self, queue=None):
"""
if current queue is a leaf queue:
MemoryUsage is the abs_memoryusage/self.absMemoryCapacity
else:
calculate the memory usage of its chilren;
calculate the absolute memory of the queue.
MemoryUsage = absUsedMemory / absMemory
"""
if queue is None:
queue = self.get_root()
memory_usage = 0.0
if queue.is_leaf():
abs_memory_usage = queue.data.cal_queue_memory_usage()
abs_memory_capacity = queue.data.get_abs_capacity()
memory_usage = 100.0 * abs_memory_usage / abs_memory_capacity
queue.data.set_mem_usage(memory_usage)
queue.data.set_abs_memory_usage(abs_memory_usage)
else: # parent queue
# First, get its all chilren queue, and call each child's calMemoryUsage function
children = self.tree.children(queue.tag)
for child in children:
self.cal_memory_usage(child)
# Second, calculate the absUsedMemoryUsage of current queue
abs_memory_usage = 0
for child in children:
abs_memory_usage += child.data.get_abs_memory_usage()
queue.data.set_abs_memory_usage(abs_memory_usage)
# Finally, calculate the memory usage of the queue
queue.data.set_mem_usage(100.0 *
queue.data.get_abs_memory_usage() /
queue.data.get_abs_capacity())
return queue.data.get_mem_usage()
def cal_mem_usage_division(self, queue=None):
"""
if current queue is a leaf queue:
memUsageDivision is zero.
else:
calculate the standard division of its chilren;
calculate the standard division of current queue through its chilren's average memoryUsage
"""
if queue is None:
queue = self.get_root()
std_division = 0.0
if self.is_leaf(queue.tag):
queue.data.cur_metric.mem_usage_div = std_division
return std_division
else: # parent queue
# First, get its all chilren queue, and call each child's calSlowDown function
children = self.tree.children(queue.tag)
for child in children:
self.cal_mem_usage_division(child)
# Second, calculate the average memory usage of all its children
count = len(children)
total_mem_usage = 0
for child in children:
total_mem_usage += child.data.get_mem_usage()
# print(child.data.get_mem_usage())
avg_mem_usage = total_mem_usage / count
# Finally, calculate the standard division of the queue
squareSum = 0
for child in children:
squareSum += np.square(child.data.get_mem_usage() -
avg_mem_usage)
std_division = np.sqrt(squareSum / count)
queue.data.cur_metric.mem_usage_div = std_division
return std_division
def cal_abs_capacity_bottom_up(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = 0
for child in children:
# print("Queue name: %s, abs_capacity: %.2f" %(child.tag, child.data.get_abs_capacity()))
self.cal_abs_capacity_bottom_up(child)
abs_capacity += child.data.get_abs_capacity()
queue.data.set_abs_capacity(abs_capacity)
def cal_desired_abs_capacity_bottom_up(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = 0.0
fixed_capacity = 0.0
for child in children:
self.cal_desired_abs_capacity_bottom_up(child)
if child.data.config.fixed:
# print("FIXED")
# print(child.data.config.capacity)
# print(child.data.config.abs_capacity)
fixed_capacity += child.data.config.capacity
else:
abs_capacity += child.data.wish.abs_capacity
for child in children:
if child.data.config.fixed:
child.data.wish.abs_capacity = abs_capacity / (
100.0 - fixed_capacity) * child.data.config.capacity
queue.data.wish.abs_capacity = abs_capacity * 100.0 / (
100.0 - fixed_capacity)
def clear_desired_abs_capacity(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.wish.abs_capacity = 0
if self.is_leaf(queue.tag):
return
else:
queue.data.cur_metric.pending = 0.0
children = self.tree.children(queue.tag)
for child in children:
self.clear_desired_abs_capacity(child)
def cal_abs_capacity_top_down(self, queue=None):
"""
This function calculate the abs capacity of each queue by its capacity.
This function should only be called once at the start time.
"""
if queue is None:
queue = self.get_root()
queue.data.set_abs_capacity(100.0)
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_capacity(queue.data.get_abs_capacity() *
child.data.get_capacity() / 100.0)
# print(child.data.get_abs_capacity())
self.cal_capacity_top_down(child)
def cal_desired_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.wish.capacity = 100.0
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.wish.abs_capacity
remain_capaciy = 100.0
for child in children:
if child.data.config.fixed:
child.data.wish.capacity = child.data.config.capacity
elif abs_capacity == 0:
child.data.wish.capacity = 0
else:
child.data.wish.capacity = child.data.wish.abs_capacity / abs_capacity * 100.0
self.cal_desired_capacity_top_down(child)
def cal_capacity_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
abs_capacity = queue.data.get_abs_capacity()
for child in children:
if abs_capacity == 0:
child.data.set_capacity(0)
else:
child.data.set_capacity(child.data.get_abs_capacity() /
abs_capacity * 100)
self.cal_capacity_top_down(child)
def cal_abs_memory_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
queue.data.cal_totalMb_mean()
queue.data.clear_totalMb()
if self.is_leaf(queue.tag):
return
else:
children = self.tree.children(queue.tag)
for child in children:
child.data.set_abs_memory(queue.data.get_abs_memory() *
child.data.get_capacity() / 100)
self.cal_abs_memory_top_down(child)
def clear_mus_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_queue_memory_usage()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_mus_top_down(child)
def clear_jobs_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_jobs()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_jobs_top_down(child)
def clear_pendings_top_down(self, queue=None):
if queue is None:
queue = self.get_root()
if self.is_leaf(queue.tag):
queue.data.clear_pendings()
else:
children = self.tree.children(queue.tag)
for child in children:
self.clear_pendings_top_down(child)
def before_scoring(self):
self.cal_abs_capacity_bottom_up()
self.cal_capacity_top_down()
self.cal_abs_memory_top_down()
def after_scoreing(self):
self.clear_jobs_top_down()
self.clear_pendings_top_down()
self.clear_mus_top_down()
def score(self):
self.before_scoring()
# self.cal_slowdown()
# self.cal_slowdown_division()
self.cal_pending()
self.cal_pending_division()
self.cal_memory_usage()
self.cal_mem_usage_division()
self.after_scoreing()
def before_predict(self):
self.cal_desired_abs_capacity_bottom_up()
def after_predict(self):
self.clear_desired_abs_capacity()
def predict(self):
self.before_predict()
self.cal_desired_capacity_top_down()
self.after_predict()
inner_bags = get_inner_bags(split_line[1])
existing_bags = find_existing_bag(baggage_claim, outer_bag)
if (len(existing_bags) > 0):
for bag in existing_bags:
for i in inner_bags:
baggage_claim.create_node(i, parent=bag.identifier)
else:
outer_bag_node = baggage_claim.create_node(outer_bag,
parent='baggage claim')
for i in inner_bags:
inner_existing = find_existing_top_level_bag(baggage_claim, i)
print(f'inner_existing {inner_existing}')
if (len(inner_existing) > 0):
#tree.move(node, new_parent)
baggage_claim.move_node(inner_existing[0].identifier,
outer_bag_node.identifier)
# if inner already exists
# move existing
else:
baggage_claim.create_node(i,
parent=outer_bag_node.identifier)
baggage_claim.show()
shiny = list(
baggage_claim.filter_nodes(lambda n: n.tag == 'shiny gold bag'))
print(f'shiny gold bags: {len(shiny)}')
for child in baggage_claim.children('baggage claim'):
pass
# print(f'{child.tag} {baggage_claim.children(child.identifier)[0]}')
if words[0] == 'extmodule':
words = line.split()
extmodule = words[1]
# print(extmodule)
tree.create_node(extmodule + '(ext)', extmodule, parent=circuit)
cur_module = extmodule
if words[
0] == 'inst': # @inst, re-assign the parent of inst_module from top to cur_module
words = line.split()
inst_name = words[1]
inst_module = words[3]
# print(inst_module)
# print(cur_module)
tree.move_node(inst_module, cur_module)
tree.show()
total_parent_cnt = 0
total_children_cnt = 0
total_modules = 0
for node in tree.expand_tree(mode=Tree.DEPTH):
total_modules += 1
if (tree[node].is_leaf()):
continue
total_parent_cnt += 1
total_children_cnt += len(tree.children(node))
tmp_list = []
class Mwrp:
def __init__(self, world, number_of_agent, start_node):
self.number_of_agent = number_of_agent
self.open_list = np.array([start_node])
self.tree = Tree()
self.tree.create_node(start_node.pos.__str__(),
start_node.pos.__str__(),
data=start_node)
self.world = world
self.node_expend_index = 1
self.need_to_see = np.sum(self.world.grid_map == 0)
def insert_to_open_list(self, new_node):
for index, data in enumerate(self.open_list):
if data.f > new_node.f:
self.open_list = np.insert(self.open_list, index, new_node)
return
self.open_list = np.append(self.open_list, new_node)
def pop_open_list(self):
return self.open_list[0].pos
def move_from_open_to_close(self, index=0):
self.open_list = np.delete(self.open_list, index)
def heuristic(self, state):
return 1
# #old
# def fix_g_subtree(self,neighbor,state):
# tmp_new_open = []
# #self.tree.move_node(neighbor.__str__(), state.__str__())
#
# for node in mwrp.tree.subtree(state.data.pos.__str__()).expand_tree(mode=Tree.DEPTH):
# parent = mwrp.tree.parent(node)
#
# new_g = parent.data.g + Utils.n_dim_distance(
# self.tree.get_node(node).data.pos, parent.data.pos)
# self.tree.get_node(node).data.f = self.tree.get_node(node).data.f - self.tree.get_node(node).data.g + new_g
#
# self.tree.get_node(node).data.g = new_g
# tmp_new_open.append(self.tree.get_node(node))
#
# for need_new_open in tmp_new_open:
# for index, data in enumerate(self.open_list):
# if (np.all(data.pos == need_new_open.data.pos)):
# self.open_list = np.delete(self.open_list, index)
# self.insert_to_open_list(self.tree.get_node(need_new_open.data.pos.__str__()).data)
# break
def get_all_seen(self, state):
tmp_node = self.tree.get_node(state.identifier)
tmp_seen = self.world.get_seen(tmp_node)
while not self.tree.get_node(tmp_node.identifier).is_root():
tmp_node = self.tree.get_node(
tmp_node.predecessor(self.tree.identifier))
tmp_seen = np.vstack((tmp_seen, self.world.get_seen(tmp_node)))
all_seen = np.unique(tmp_seen, axis=0)
return all_seen
# def get_all_seen1(self, state):
#
# tmp_node = self.tree.get_node(state.identifier)
# dictOfWords = {i.__str__(): 0 for i in self.world.get_seen(tmp_node)}
#
# while not self.tree.get_node(tmp_node.identifier).is_root():
# tmp_node = self.tree.get_node(tmp_node.predecessor(self.tree.identifier))
# for seen in self.world.get_seen(tmp_node):
# if not seen.__str__() in dictOfWords:
# dictOfWords[seen.__str__()]=0
#
# return dictOfWords
# def expend_all(self,state):
# for neighbor in self.world.get_neighbors(state):
# if self.world.in_bund(neighbor) and self.world.is_obstical(neighbor):
# new_g = mwrp.tree.get_node(state.__str__()).data.g + Utils.n_dim_distance(state, neighbor)
# if not self.tree.get_node(neighbor.__str__()):
# h = self.heuristic(state)
# new_node=Node(neighbor,new_g,new_g+h)
# self.tree.create_node(neighbor.__str__(),neighbor.__str__(),parent=(state.__str__()),data=new_node)
# self.insert_to_open_list(new_node)
# else:
# self.fix_g(neighbor, state)
#
# #self.tree.show()
# self.move_from_open_to_close()
def expend(self, state):
move_index = np.zeros(self.number_of_agent).astype(int)
for i in range(LOS**number_of_agent):
for j in range(number_of_agent):
i, index = divmod(i, LOS)
move_index[j] = index
neighbor = self.world.get_one_neighbor(state, move_index)
if self.world.in_bund(neighbor) and self.world.is_obstical(
neighbor):
new_g = mwrp.tree.get_node(
state.__str__()).data.g + Utils.n_dim_distance(
state, neighbor)
if not self.tree.contains(neighbor.__str__()):
h = self.heuristic(state)
new_node = Node(neighbor, new_g, new_g + h)
self.tree.create_node(neighbor.__str__(),
neighbor.__str__(),
parent=(state.__str__()),
data=new_node)
self.insert_to_open_list(new_node)
else:
if new_g < self.tree.get_node(neighbor.__str__()).data.g:
self.fix_g(neighbor, state)
self.move_from_open_to_close()
def fix_g(self, old_state, new_parent):
state = self.tree.get_node(old_state.__str__())
old_parent = self.tree.get_node(state.predecessor(
self.tree.identifier))
new_parent = self.tree.get_node(new_parent.__str__())
if self.seen_comparison(new_parent, old_parent):
self.tree.move_node(state.identifier, new_parent.identifier)
new_g = new_parent.data.g + \
Utils.n_dim_distance(self.tree.get_node(state.identifier).data.pos, new_parent.data.pos)
self.tree.get_node(state.identifier).data.f = self.tree.get_node(state.identifier).data.f - \
self.tree.get_node(state.identifier).data.g + new_g
self.tree.get_node(state.identifier).data.g = new_g
for index, data in enumerate(self.open_list):
if np.all(data.pos == state.data.pos):
self.open_list = np.delete(self.open_list, index)
self.insert_to_open_list(
self.tree.get_node(state.identifier).data)
break
def goal_test(self, state):
seen_number = self.world.get_seen(state).shape[0]
if (seen_number == self.need_to_see):
return True
return False
def seen_comparison(self, state_new, state_old):
seen_new = self.get_all_seen(state_new).tolist()
seen_old = self.get_all_seen(state_old).tolist()
if seen_new.shape[0] < seen_old.shape[0]:
return False
for one_seen in seen_old:
if one_seen not in seen_new:
return False
return True
from treelib import Tree, Node
if __name__ == '__main__':
# 树的创建，每个节点都有唯一的identifier作为标记，可以手动指定
tree = Tree()
# 增加树的节点,tag是树输出时的显示，identifier是唯一标志，根节点可以不指定父
tree.create_node(tag='root', identifier='root', data=0)
tree.create_node(tag='1_child',
identifier='1_child',
data=1,
parent='root')
tree.create_node(tag='2_child',
identifier='2_child',
data=2,
parent='root')
tree.create_node(tag='3_child',
identifier='3_child',
data=3,
parent='1_child')
# 树的粘贴，需要注意的是这个nid是tree的identifier，不是tree2的
tree2 = Tree()
tree2.create_node(tag='tutu', identifier='tutu', data=0)
tree.paste(nid='root', new_tree=tree2)
# 删除树的节点
tree.remove_node('tutu')
# 移动树的节点
tree.move_node('3_child', 'root')
# 打印树的结构
tree.show()
def TreeSequenceToTreeClass(simulation,
tree_event_sequence,
is_AA_mutation_in_root_node=False):
t1 = time.time()
tree_size = len(tree_event_sequence.tree_sequence)
tree_class_tree = Tree()
root_id = 'Unknown'
array_tree = simulation.GetTree()
for i in range(tree_size):
if array_tree[i] == -1:
root_id = i
tree_class_tree.create_node(root_id, root_id,
data=None) # placeholder on root
break # there can be only one root
if root_id == 'Unknown':
raise ValueError("There is no root in this tree")
for i in range(tree_size):
if i != root_id:
tree_class_tree.create_node(
i, i, parent=root_id, data=None) # placeholder on other places
for i in range(tree_size):
if i != root_id:
tree_class_tree.move_node(i, array_tree[i])
for i in range(tree_size):
noc = len(tree_class_tree.get_node(i).fpointer) # number of children
ni = tree_event_sequence.tree_sequence[i].node_id
iam = tree_event_sequence.tree_sequence[i].is_a_mutation
on = tree_event_sequence.tree_sequence[i].old_nucleotyde
nn = tree_event_sequence.tree_sequence[i].new_nucleotyde
mc = tree_event_sequence.tree_sequence[i].mutation_cite
tti = tree_event_sequence.tree_sequence[i].tree_time
tty = tree_event_sequence.tree_sequence[i].tree_type
if (i == root_id) and (is_AA_mutation_in_root_node == True):
tree_class_tree.update_node(i,
data=TreeEvent(is_a_mutation=True,
number_of_children=noc,
old_nucleotyde=0,
new_nucleotyde=0,
mutation_cite=0,
tree_time=0,
tree_type='coalescence',
node_id=ni))
else:
tree_event = TreeEvent(is_a_mutation=iam,
number_of_children=noc,
old_nucleotyde=on,
new_nucleotyde=nn,
mutation_cite=mc,
tree_time=tti,
tree_type=tty,
node_id=ni)
tree_class_tree.update_node(i, data=tree_event)
t2 = time.time()
print('Time spent on conversion to tree class = ', t2 - t1)
return tree_class_tree
print tree.is_branch('diane')
print('\n')
print("#"*4 + "OOhh~ new members enter Jill's family")
new_tree = Tree()
new_tree.create_node("n1", 1) # root node
new_tree.create_node("n2", 2, parent=1)
new_tree.create_node("n3", 3, parent=1)
tree.paste('jill', new_tree)
tree.show()
print('\n')
print("#"*4 + "We are sorry they are gone accidently :(")
tree.remove_node(1)
tree.show()
print('\n')
print("#"*4 + "Now Jill moves to live with Grand-x-father Harry")
tree.move_node('jill', 'harry')
tree.show()
print('\n')
print("#"*4 + "A big family for George to talk to Grand-x-father Harry")
for node in tree.rsearch('george', filter=lambda x: x.identifier != 'harry'):
print node
print('harry')
print('\n')