def create_ding_tree(): #global ding_tree, dept_result #debug only #连接数据库 db, cursor = connect_db('localhost', 'root', 'yoyoball', 'dingtalk') ding_tree = Tree() sql = "SELECT `id`, `name`, `parentid` FROM dingding_department_list" cursor.execute(sql) dept_result = cursor.fetchall() #print dept_result #debug only if dept_result != None and len(dept_result) > 0: ding_tree.create_node('##ding_root##', '0') #先创建虚拟根 for i in range(len(dept_result)): #向虚拟根填充所有组织 #print dept_result[i] #debug only #ding_tree.create_node(dept_result[i][1].decode('utf-8'), dept_result[i][0], '0000') ding_tree.create_node(dept_result[i][1], dept_result[i][0], '0') for i in range(len(dept_result)): #修改隶属关系 if dept_result[i][0] != '1' : #只要不是实根,就要修改隶属关系【钉钉中实根id为'1'且无上级部门,数据表dingding_department_list中存储id为'1'的部门上级为'0'】 if ding_tree.contains(dept_result[i][2]): #判断上级是否存在 ding_tree.move_node(dept_result[i][0], dept_result[i][2]) else: #没有上级的不修改 #print type(dept_result[i][2]), dept_result[i][2] #debug only continue #断开数据库 close_db(db) #return ding_tree return ding_tree.subtree('1')
def create_oa_tree(): #global oa_tree #debug only #连接数据库 db, cursor = connect_db('localhost', 'root', 'yoyoball', 'test') oa_tree = Tree() sql = "SELECT `orgid`, `shortname`, `parentorgid` FROM groupinfo" cursor.execute(sql) dept_result = cursor.fetchall() #print dept_result debug only if dept_result != None and len(dept_result) > 0: oa_tree.create_node('##oa_root##', '0000') #先创建虚拟根 for i in range(len(dept_result)): #向虚拟根填充所有组织 #print dept_result[i][1].decode('utf-8'), dept_result[i][0], dept_result[i][2] oa_tree.create_node(dept_result[i][1], dept_result[i][0], '0000') for i in range(len(dept_result)): #修改隶属关系 if dept_result[i][2] != '0000' : #OA中'001000'等的组织上级为'0000'即OA数据库中存在虚根,所以无需做此步骤 if oa_tree.contains(dept_result[i][2]): #判断上级是否存在 oa_tree.move_node(dept_result[i][0], dept_result[i][2]) else: #没有上级的不修改 continue #断开数据库 close_db(db) return oa_tree
def visualizeCtree(c_tree): #bottom up build tree tree = Tree() tree.create_node("root", "root") levels = sorted(c_tree.keys()) for level in levels: for node_id, cluster in c_tree[level].items(): node_id = "{}.{}".format(level, node_id) tree.create_node("{}".format(cluster["pattern"]), node_id, parent="root") if level == 0: for data in cluster["data"]: tree.create_node("log.{}".format(data), "log.{}".format(data), parent=node_id) else: for data in cluster["data"]: tree.move_node("{}.{}".format(level-1, data), node_id) tree.show() tree.save2file("./tree") ## How to use ##1. for log in logs:updateCTree, updatePatterns ##2. get the c_tree ##3. train all the logs to get the max level and estimate the best level we keep. ##4. train the cluster on different level ##要考虑到unmerged list的同步问题, 到时production考虑将c_tree存在redis里,然后同时flush到库里
class AcquisitionChain(object): def __init__(self, parallel_prepare=False): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() self._presets_list = list() self._parallel_prepare = parallel_prepare self._device2one_shot_flag = weakref.WeakKeyDictionary() @property def nodes_list(self): nodes_gen = self._tree.expand_tree() nodes_gen.next() # first node is 'root' return list(nodes_gen) def add(self, master, slave): self._device2one_shot_flag.setdefault(slave, False) slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if (slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer): raise RuntimeError( "Cannot add acquisition device %s to multiple masters, current master is %s" % (slave, slave_node._bpointer)) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave, master) slave.parent = master def add_preset(self, preset): self._presets_list.append(preset) def set_stopper(self, device, stop_flag): """ By default any top master device will stop the scan. In case of several top master, you can define which one won't stop the scan """ self._device2one_shot_flag[device] = not stop_flag def __iter__(self): if len(self._tree) > 1: return AcquisitionChainIter( self, parallel_prepare=self._parallel_prepare) else: return iter(())
class AcquisitionChain(object): def __init__(self): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() def add(self, master, slave): slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer: raise RuntimeError( "Cannot add acquisition device %s to multiple masters, current master is %s" % (slave, slave_node._bpointer) ) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave_node, master_node) def _execute(self, func_name): tasks = list() prev_level = None for dev in reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]): node = self._tree.get_node(dev) level = self._tree.depth(node) if prev_level != level: gevent.joinall(tasks) tasks = list() func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) gevent.joinall(tasks) def prepare(self, dm, scan_info): # self._devices_tree = self._get_devices_tree() for master in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] for dev in self._tree.get_node(master).fpointer: master.slaves.append(dev) dm_prepare_task = gevent.spawn(dm.prepare, scan_info, self._tree) self._execute("_prepare") dm_prepare_task.join() def start(self): self._execute("_start") for acq_dev in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionDevice)): acq_dev.wait_reading() dispatcher.send("end", acq_dev)
def test_loop(self): tree = Tree() tree.create_node('a', 'a') tree.create_node('b', 'b', parent='a') tree.create_node('c', 'c', parent='b') tree.create_node('d', 'd', parent='c') try: tree.move_node('b', 'd') except LoopError: pass
def merge(T, alpha, beta): """ merge is a function that mergers two nodes of tree T, changing node beta to be a child of node alpha, where alpha and beta are sisters. param T: the given tree param alpha: the target node of T param beta: the merged node of T return: the new tree after merging operation. """ TT = Tree(T.subtree(T.root), deep=True) TT.move_node(beta, alpha) return TT
class AcquisitionChain(object): def __init__(self, parallel_prepare=False): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() self._presets_list = list() self._parallel_prepare = parallel_prepare self._device2one_shot_flag = weakref.WeakKeyDictionary() @property def nodes_list(self): nodes_gen = self._tree.expand_tree() nodes_gen.next() # first node is 'root' return list(nodes_gen) def add(self, master, slave): self._device2one_shot_flag.setdefault(slave, False) slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if(slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer): raise RuntimeError("Cannot add acquisition device %s to multiple masters, current master is %s" % ( slave, slave_node._bpointer)) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node( tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node( tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave, master) slave.parent = master def add_preset(self, preset): self._presets_list.append(preset) def set_stopper(self, device, stop_flag): """ By default any top master device will stop the scan. In case of several top master, you can define which one won't stop the scan """ self._device2one_shot_flag[device] = not stop_flag def __iter__(self): if len(self._tree) > 1: return AcquisitionChainIter(self, parallel_prepare=self._parallel_prepare) else: return iter(())
def get_intersection_tree(T1, T2): T = Tree(tree=T1, deep=True) T1_bfs = [n for n in T1.expand_tree(mode=1)] T2_bfs = [n for n in T2.expand_tree(mode=1)] for nid in T1_bfs: X = set(get_leaf_node_ids_for_node(T, nid)) diff = min([len(X.symmetric_difference(set( \ get_leaf_node_ids_for_node(T2,i)))) \ for i in T2_bfs]) if diff != 0: par = T.parent(nid).identifier for c in T.children(nid): T.move_node(c.identifier, par) T.remove_subtree(nid) return T
def combine(T, alpha, beta): """ combine is a function that combines two nodes of tree T, changing nodes alpha and beta to be children of a new node gama, where alpha and beta are sisters. param T: the given tree param alpha: the target node of T param beta: the target node of T return: the new tree after combining operation. """ TT = Tree(T.subtree(T.root), deep=True) new_id = max(ids_of_all_nodes(T)) + 1 parent_id = TT.get_node(alpha).bpointer TT.create_node(tag=new_id, identifier=new_id, parent=parent_id, data=None) TT.move_node(alpha, new_id) TT.move_node(beta, new_id) return TT
def build_hierarchy_tree(self, nodes): """ Build hierarchy tree from list of nodes. """ tree = Tree() # root node tree.create_node('root', 0) # create all nodes for i in nodes: name, id, parent = self.parse_hierarchy_node(i) tree.create_node(name, id, parent=0) # assign parents for i in nodes: name, id, parent = self.parse_hierarchy_node(i) if parent is not None: tree.move_node(id, parent) return tree
for z in d: path = walkTree(tree, z, path + z) return path input = list(map(lambda x: x.strip(), open("test_input.txt").readlines())) tree = Tree() tree.create_node("root", "root") # first figure out how many steps there are and then sort them # by their name for lines in input: (l1, l2) = (lines[5], lines[36]) print(lines) if tree.contains(l1) and tree.contains(l2): tree.move_node(l2, l1) elif tree.contains(l1) and not tree.contains(l2): tree.create_node(l2, l2, parent=l1) elif not tree.contains(l1) and tree.contains(l2): # get the root for l2 and make that the root for l1 # then move l2 under l1 tree.create_node(l1, l1, parent=tree.parent(l2)) tree.move_node(l2, l1) else: tree.create_node(l1, l1, parent="root") tree.create_node(l2, l2, parent=l1) tree.show() print(walkTree(tree, 'root', ''))
class RMQueue(object): __metaclass__ = Singleton def __init__(self): self.tree = Tree() self.MAX_METRIC_COUNT = 12 self.CAL_INTERVAL_IN_SECOND = 2 * 60 * 60 self.conf = conf.Config("./conf/config.json") def set_stat_interval(self, interval): self.CAL_INTERVAL_IN_SECOND = interval def set_system_memory(self, size): root = self.get_root() root.data.set_abs_memory(float(size)) def create_queue(self, name=None, parent=None): data = QueueData() self.tree.create_node(name, name, parent, data) def display(self): self.tree.show() def display_score(self, queue=None, depth=0, table=None, printer=None): flag = False if queue is None: queue = self.get_root() flag = True table = PrettyTable([ "QUEUE", "PENDING AVG", "PENDING DIV", "MEMORY USAGE AVG(Q)", "MEMORY USAGE AVG(C)", "MEMORY USAGE DIV", "ABS CAPACITY" ]) if table is not None: table.add_row([ queue.tag, 0 if queue.data.get_pending() == 0 else "%.3f" % queue.data.get_pending(), 0 if queue.data.get_pending_div() == 0 else "%.3f" % queue.data.get_pending_div(), 0 if queue.data.get_mem_usage() == 0 else "%.3f" % queue.data.get_mem_usage(), 0 if queue.data.cal_queue_memory_usage() == 0 else "%.3f" % queue.data.cal_queue_memory_usage(), 0 if queue.data.get_mem_usage_div() == 0 else "%.3f" % queue.data.get_mem_usage_div(), str(0 if queue.data.get_abs_capacity() == 0 else "%.3f" % queue.data.get_abs_capacity()) + " %" ]) if not self.is_leaf(queue.tag): children = self.tree.children(queue.tag) for child in children: self.display_score(child, depth + 1, table) if flag: if printer is None: print('------------' + utils.get_str_time() + ' SCORE ----------') print table else: printer.write('\n------------' + utils.get_str_time() + ' SCORE ----------\n') printer.write(str(table)) def display_prediction(self, queue=None, depth=0, table=None, printer=None): flag = False if queue is None: queue = self.get_root() flag = True table = PrettyTable([ "QUEUE", "DESIRED CAPACITY(Q)", "DESIRED CAPACITY(C)", "ABS CAPACITY" ]) if table is not None: table.add_row([ queue.tag, str(0 if queue.data.wish.capacity == 0 else "%.3f" % (100 * queue.data.wish.capacity)) + " %", 0 if queue.data.wish.abs_capacity == 0 else "%.3f" % queue.data.wish.abs_capacity, str(0 if queue.data.config.abs_capacity == 0 else "%.3f" % queue.data.config.abs_capacity) + " %" ]) if not self.is_leaf(queue.tag): children = self.tree.children(queue.tag) for child in children: self.display_prediction(child, depth + 1, table) if flag: if printer is None: print('------------' + utils.get_str_time() + ' PREDICTION ----------') print table else: printer.write('\n------------' + utils.get_str_time() + ' PREDICTION ----------\n') printer.write(str(table)) def write_score(self, path): FileOperator.touch(path) with open(path, 'a') as f: self.display_score(printer=f) def request_score(self, queue=None): if queue is None: queue = self.get_root() postData = { 'queue': queue.tag, 'pending': queue.data.get_pending(), 'pending_div': queue.data.get_pending_div(), 'memory_usage': queue.data.get_mem_usage(), 'memory_usage_div': queue.data.get_mem_usage_div(), 'abs_capacity': queue.data.get_abs_capacity() } requests.post(str(self.conf.es_rest_address) + str(self.conf.es_index) + "score", data=json.dumps(postData)) if not self.is_leaf(queue.tag): children = self.tree.children(queue.tag) for child in children: self.request_score(child) def request_prediction(self, queue=None): if queue is None: queue = self.get_root() postData = { 'queue': queue.tag, 'wish_capacity': queue.data.wish.capacity, 'wish_abs_capacity': queue.data.wish.abs_capacity, 'abs_capacity': queue.data.config.abs_capacity } requests.post(str(self.conf.es_rest_address) + str(self.conf.es_index) + "prediction", data=json.dumps(postData)) if not self.is_leaf(queue.tag): children = self.tree.children(queue.tag) for child in children: self.request_prediction(child) def write_prediction(self, path): FileOperator.touch(path) with open(path, 'a') as f: self.display_prediction(printer=f) def add_job(self, job, qname): queue = self.tree.get_node(qname) if queue.is_leaf(): queue.data.add_job(job) else: print("Cannot add jobs to parent queue", queue.tag, queue.identifier) def add_metric(self, qname): queue = self.tree.get_node(qname) queue.data.add_metric(queue.cur_metric) if len(queue.data.metrics) > RMQueue.MAX_METRIC_COUNT: del queue.data.metrics[0] def remove_queue(self, qname): self.tree.remove_node(qname) def move_queue(self, src, dest): self.tree.move_node(src, dest) def get_queue(self, qname): return self.tree.get_node(qname) def get_root(self): return self.get_queue('root') def is_leaf(self, qname): queue = self.tree.get_node(qname) return queue.is_leaf() def cal_slowdown(self, queue=None): if queue is None: queue = self.get_root() avg_slowdown = 0.0 if queue.is_leaf(): job_count = len(queue.data.jobs) for i in list(range(job_count)): job = queue.data.jobs[i] slowdown = (job.wait_time + job.run_time) / job.run_time avg_slowdown += slowdown / job_count queue.data.set_job_count(job_count) queue.data.cur_metric.slowdown = avg_slowdown else: children = self.tree.children(queue.tag) for child in children: self.cal_slowdown(child) job_count = 0 for child in children: job_count += child.data.get_job_count() queue.data.set_job_count(job_count) if job_count == 0: queue.data.cur_metric.slowdown = avg_slowdown return avg_slowdown for child in children: avg_slowdown += child.data.get_job_count( ) * child.data.get_slowdown() / job_count queue.data.cur_metric.slowdown = avg_slowdown return queue.data.get_slowdown() def cal_pending(self, queue=None): if queue is None: queue = self.get_root() if queue.is_leaf(): if len(queue.data.pendings) > 0: queue.data.cur_metric.pending = np.mean(queue.data.pendings) else: children = self.tree.children(queue.tag) for child in children: self.cal_pending(child) queue.data.cur_metric.pending += child.data.get_pending() return queue.data.get_pending() def cal_pending_division(self, queue=None): if queue is None: queue = self.get_root() division = 0.0 if self.is_leaf(queue.tag): return division else: children = self.tree.children(queue.tag) for child in children: self.cal_pending_division(child) count = len(children) avg_pending = queue.data.get_pending() * 1.0 / count square_sum = 0.0 for child in children: square_sum += np.square(child.data.get_pending() - avg_pending) division = np.sqrt(square_sum / count) queue.data.cur_metric.pending_div = division return division def cal_slowdown_division(self, queue=None): if queue is None: queue = self.get_root() division = 0.0 if self.is_leaf(queue.tag): return division else: children = self.tree.children(queue.tag) for child in children: self.cal_slowdown_division(child) square_sum = 0.0 count = len(children) for child in children: square_sum += np.square(child.data.get_slowdown() - queue.data.get_slowdown()) division = np.sqrt(square_sum / count) queue.data.cur_metric.slowdown_div = division return division def cal_memory_usage(self, queue=None): if queue is None: queue = self.get_root() if queue.is_leaf(): capacity = queue.data.get_abs_capacity() memory_usage = 0.0 if capacity != 0: memory_usage = 100.0 * queue.data.cal_queue_memory_usage( ) / capacity queue.data.set_mem_usage(memory_usage) else: children = self.tree.children(queue.tag) for child in children: self.cal_memory_usage(child) abs_memory_usage = 0 for child in children: abs_memory_usage += child.data.get_abs_memory_usage() queue.data.set_mem_usage(100.0 * abs_memory_usage / queue.data.get_abs_capacity()) return queue.data.get_mem_usage() def cal_mem_usage_division(self, queue=None): if queue is None: queue = self.get_root() std_division = 0.0 if self.is_leaf(queue.tag): queue.data.cur_metric.mem_usage_div = std_division return std_division else: children = self.tree.children(queue.tag) for child in children: self.cal_mem_usage_division(child) count = len(children) total_mem_usage = 0 for child in children: total_mem_usage += child.data.get_mem_usage() avg_mem_usage = total_mem_usage / count square_sum = 0 for child in children: square_sum += np.square(child.data.get_mem_usage() - avg_mem_usage) std_division = np.sqrt(square_sum / count) queue.data.cur_metric.mem_usage_div = std_division return std_division def cal_abs_capacity_bottom_up(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = 0.0 for child in children: self.cal_abs_capacity_bottom_up(child) abs_capacity += child.data.get_abs_capacity() queue.data.set_abs_capacity(abs_capacity) def cal_desired_abs_capacity_bottom_up(self, queue=None, delim=None): if queue is None: queue = self.get_root() delim = 1 if self.is_leaf(queue.tag): queue.data.wish.capacity = queue.data.wish.abs_capacity / delim else: children = self.tree.children(queue.tag) abs_capacity = 0.0 for child in children: self.cal_desired_abs_capacity_bottom_up( child, queue.data.config.abs_capacity * delim / 100) abs_capacity += child.data.wish.abs_capacity queue.data.wish.capacity = abs_capacity / delim def cal_abs_capacity_top_down(self, queue=None): if queue is None: queue = self.get_root() queue.data.set_abs_capacity(100.0) if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) for child in children: child.data.set_abs_capacity(queue.data.get_abs_capacity() * child.data.get_capacity() / 100.0) self.cal_abs_capacity_top_down(child) def cal_desired_capacity_top_down(self, queue=None): if queue is None: queue = self.get_root() queue.data.wish.capacity = 100.0 if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = queue.data.wish.abs_capacity for child in children: child.data.wish.capacity = child.data.config.capacity if abs_capacity == 0: child.data.wish.capacity = 0 else: child.data.wish.capacity = child.data.wish.abs_capacity / abs_capacity * 100.0 self.cal_desired_capacity_top_down(child) def cal_capacity_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = queue.data.get_abs_capacity() for child in children: if abs_capacity == 0: child.data.set_capacity(0) else: child.data.set_capacity(child.data.get_abs_capacity() / abs_capacity * 100) self.cal_capacity_top_down(child) def cal_abs_memory_top_down(self, queue=None): if queue is None: queue = self.get_root() queue.data.cal_totalMb_mean() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) for child in children: child.data.set_abs_memory(queue.data.get_abs_memory() * child.data.get_capacity() / 100) self.cal_abs_memory_top_down(child) def clear_mus_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): queue.data.clear_queue_memory_usage() else: children = self.tree.children(queue.tag) for child in children: self.clear_mus_top_down(child) def clear_jobs_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): queue.data.clear_jobs() else: children = self.tree.children(queue.tag) for child in children: self.clear_jobs_top_down(child) def clear_pendings_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): queue.data.clear_pendings() else: children = self.tree.children(queue.tag) for child in children: self.clear_pendings_top_down(child) def score(self): # self.cal_abs_capacity_bottom_up() # self.cal_capacity_top_down() # self.cal_abs_memory_top_down() # self.cal_slowdown() # self.cal_slowdown_division() self.cal_pending() self.cal_pending_division() self.cal_memory_usage() self.cal_mem_usage_division() # self.clear_jobs_top_down() # self.clear_pendings_top_down() # self.clear_mus_top_down() def predict(self): self.cal_desired_abs_capacity_bottom_up()
print(tree[child].tag) print(sep + "OOhh~ new members join Jill's family:") new_tree = Tree() new_tree.create_node("n1", 1) # root node new_tree.create_node("n2", 2, parent=1) new_tree.create_node("n3", 3, parent=1) tree.paste("jill", new_tree) tree.show() print(sep + "They leave after a while:") tree.remove_node(1) tree.show() print(sep + "Now Jill moves to live with Grand-x-father Harry:") tree.move_node("jill", "harry") tree.show() print(sep + "A big family for George to send message to the oldest Harry:") for node in tree.rsearch("george"): print(tree[node].tag) ########NEW FILE######## __FILENAME__ = folder_tree #!/usr/bin/env python # A file folder scanner contributed by @holger # # You can spicify the scanned folder and file pattern by changing rootPath # and pattern variables # __author__ = "holger"
sub_t = tree.subtree('diane') sub_t.show() print('\n') print("#"*4 + "Children of Diane") print tree.is_branch('diane') print('\n') print("#"*4 + "OOhh~ new members enter Jill's family") new_tree = Tree() new_tree.create_node("n1", 1) # root node new_tree.create_node("n2", 2, parent=1) new_tree.create_node("n3", 3, parent=1) tree.paste('jill', new_tree) tree.show() print('\n') print("#"*4 + "We are sorry they are gone accidently :(") tree.remove_node(1) tree.show() print('\n') print("#"*4 + "Now Jill moves to live with Grand-x-father Harry") tree.move_node('jill', 'harry') tree.show() print('\n') print("#"*4 + "A big family for George to talk to Grand-x-father Harry") for node in tree.rsearch('george', filter=lambda x: x != 'harry'): print node print('\n')
class Parser(): """ Parser class for parsing a NAL xml file """ # All of the roots are coming from the TM queries # Other items in NAL may be considered phenotypes or chemicals # But are not being labelled for our purposes __root2label = { 322:"Phenotype", 156:"Phenotype", 319:"Phenotype", 7812:"Chemical", 8:"Chemical", 264:"Chemical", 858:"Plant" } def __init__(self, namespace="usda_nal_thesaurus"): self.namespace = namespace self.tree = Tree() self.tree.create_node("Root","root") self.name2id = {} def parse(self, xmlFile): """ parsing function that parses an xml file """ nodes = {} tree = ET.parse(xmlFile) root = tree.getroot() # THESAURUS node for concept in root: # Parse xml with nodes as concepts iden = "NAL:"+concept.find("TNR").text # node source ID nodes[iden], name = self.parseNode(concept) # Parse the node from xml file self.tree.create_node(tag=name, identifier=iden, parent='root') # create the node in the tree self.name2id[name] = iden # For mapping edges # Iterate through twice because parent/child relationships are connected via name not id # in xml file and file is sorted alphabetically for nodeID, props in nodes.items(): # Add edges to node as a list of tuples nodes[nodeID]["edges"] = self.parseEdges(nodeID,props) for node,label in self.__root2label.items(): # Add specific labels to nodes sub = self.tree.subtree("NAL:{}".format(node)) for i in sub.all_nodes(): iden = i.identifier.split(".")[0] nodes[iden]["labels"].add(label) return nodes def parseNode(self,node): """ Parse each node from the xml file """ labels = set() name = node.find("DESCRIPTOR").text synonyms = extractElem(node, "UF") # Used for parents = extractElem(node, "BT") # Broader term children = extractElem(node, "NT") # Narrow term # associated = extractElem(node,"RT") # Related term # categories = extractElem(node, "SC") # Subject category # for cat in categories: # item = cat.split(" ")[0] # if item in self.labels.keys(): # labels.add(self.labels[item]) labels.add(self.namespace) return {"name": name, "synonyms": list(synonyms), "parents": parents, "children": children, "labels": labels}, name def parseEdges(self, nodeID, props): """ Add edges to nodes. Some nodes have multiple parents """ edges = [] multiParent = 0 for name in props["children"]: edges.append(("has_child",self.name2id[name])) for name in props["parents"]: parentID = self.name2id[name] if multiParent == 0: self.tree.move_node(nodeID,parentID) else: self.tree.create_node(tag=props["name"],identifier=nodeID+".{}".format(multiParent),parent=parentID) edges.append(("is_a",parentID)) multiParent += 1 return edges
def toDEP(self): ############################### # Etape 1 : construction du head_tree # parcours en largeur de tree afin de récupérer chaque id_node # pour chaque profondeur (init à 0) _! sans compter !_ les feuilles (EDUs) nodes_depth = [-1] * self.tree.size() for i in xrange(self.tree.size()): id_nodes = [0] depth = [999] * self.tree.size() while id_nodes: # False if empty id_node = id_nodes.pop(0) node = self.tree.get_node(id_node) if node.bpointer != None: node_parent = self.tree.get_node(node.bpointer) depth[node.identifier] = depth[node_parent.identifier] + 1 else: depth[node.identifier] = 0 if id_node == i: # print 'noeud ',i,' en profondeur', depth[node.identifier] if node.fpointer: nodes_depth[i] = depth[i] break if node.fpointer: id_nodes.append(node.fpointer[0]) id_nodes.append(node.fpointer[1]) # print nodes_depth id_nodes_depth = [] for d in xrange(self.tree.depth()): id_nodes_depth.append([]) for i in xrange(self.tree.size()): if nodes_depth[i] == d: id_nodes_depth[d].append(i) # print id_nodes_depth # # construction du head_tree head_tree = [-1] * self.treeNS.size() # pour chaque noeud (non EDU/feuille) en partant de la plus grande profondeur dans l'arbre for d in range(len(id_nodes_depth) - 1, -1, -1): for id_node in id_nodes_depth[d]: node = self.treeNS.get_node(id_node) node_left = self.treeNS.get_node(node.fpointer[0]) node_right = self.treeNS.get_node(node.fpointer[1]) if node_left.tag == "N": if head_tree[node_left.identifier] == -1: identifier = node_left.identifier else: identifier = head_tree[node_left.identifier] else: if head_tree[node_right.identifier] == -1: identifier = node_right.identifier else: identifier = head_tree[node_right.identifier] head_tree[id_node] = identifier # print head_tree ############################### # Etape 2 : construction du DEP # # construction du DEP # init # root est le premier noeud de head # pour chaque EDU son père est le root dans DEP dep_tree = Tree() id_root = head_tree[0] root = self.tree.get_node(id_root) # dep_tree.create_node(root.tag, root.identifier) dep_tree.create_node(root.tag, root.identifier) for id_EDU in xrange(len(head_tree)): if head_tree[id_EDU] == -1 and id_EDU != id_root: node = self.tree.get_node(id_EDU) # dep_tree.create_node(node.tag, node.identifier, parent=id_root) # dep_tree.create_node(str(id_EDU), node.identifier, parent=id_root) dep_tree.create_node(node.tag, node.identifier, parent=id_root) # print '//////////////////////' # print 'EDU', id_root # pour chaque EDU for id_EDU in xrange(len(head_tree)): if head_tree[id_EDU] == -1 and id_EDU != id_root: EDU_NS = self.treeNS.get_node(id_EDU) # print '.......................' # print 'EDU', id_EDU # print 'TAG', EDU_NS.tag if EDU_NS.tag == "N": # parcours en largeur jusqu'à trouver un S avec un head donc qui soit pas EDU id_nodes = [EDU_NS.identifier] visited = [False] * self.treeNS.size() while id_nodes: id_node = id_nodes.pop(0) EDU = self.tree.get_node(id_node) # print 'visited EDU', EDU.identifier visited[EDU.identifier] = True # cas d'arret head_EDU = head_tree[EDU.identifier] == -1 head_EDU = False node_tag = self.treeNS.get_node(EDU.identifier).tag # print ' head_EDU', head_EDU # print ' node_tag', node_tag if not head_EDU and node_tag == "S": break if EDU.bpointer: if not visited[EDU.bpointer]: id_nodes.append(EDU.bpointer) if EDU.fpointer: # sécurité if not visited[EDU.fpointer[0]]: id_nodes.append(EDU.fpointer[0]) if not visited[EDU.fpointer[1]]: id_nodes.append(EDU.fpointer[1]) # puis ajouter au DEP comme enfant du head du parent du noeud S id_head = head_tree[EDU.bpointer] # si parent S else: # parcours en largeur des ancêtre jusqu'à trouver un ancêtre avec un head parent = self.treeNS.get_node(EDU_NS.bpointer) id_head = head_tree[parent.identifier] # puis ajouter au DEP comme enfant de ce head if id_EDU != id_head: dep_tree.move_node(id_EDU, id_head) EDU = self.tree.get_node(id_EDU) # print '---- ajout de',EDU.identifier,' à',id_head # if id_EDU == id_head: # dep_tree.show() return dep_tree
class RMQueue(metaclass=Singleton): MAX_METRIC_COUNT = 12 CAL_INTERVAL_IN_SECOND = 2 * 60 * 60 # 2hours def __init__(self): self.tree = Tree() def set_stat_interval(self, interval): RMQueue.CAL_INTERVAL_IN_SECOND = interval def set_system_memory(self, size): root = self.get_root() root.data.set_abs_memory(float(size)) def create_queue(self, name=None, parent=None): data = QueueData() self.tree.create_node(name, name, parent, data) def display(self): self.tree.show() def display_score_old(self, queue=None, depth=0): if queue is None: queue = self.get_root() print('------------' + utils.get_str_time() + ' SCORE ----------') print(24 * ' ' + ' SLOWDOWN MEMORY USAGE ') print('QUEUE NAME' + 16 * ' ' + ' AVG DIV AVG DIV') if depth >= 0: print(queue.tag + (22 - len(queue.tag))*' ' + \ '%8.3f' % queue.data.get_slowdown(), \ ' %8.3f ' % queue.data.get_slowdown_div(), \ ' %8.3f' % queue.data.get_mem_usage(), \ ' %8.3f' % queue.data.get_mem_usage_div()) """ print(queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \ 'div: %.3f)' % queue.data.get_slowdown_div(), \ '(mem usage: %.3f' % queue.data.get_mem_usage(), \ 'div: %.3f)' % queue.data.get_mem_usage_div()) """ else: print('-'*depth + queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \ 'div: %.3f)' % queue.data.get_slowdown_div(), \ '(mem usage: %.3f' % queue.data.get_mem_usage(), \ 'div: %.3f)' % queue.data.get_mem_usage_div()) if self.is_leaf(queue.tag) == False: children = self.tree.children(queue.tag) for child in children: self.display_score(child, depth + 2) def display_score(self, queue=None, depth=0): if queue is None: queue = self.get_root() print('------------' + utils.get_str_time() + ' SCORE ----------') print(24 * ' ' + ' PENDING MEMORY USAGE ') print('QUEUE NAME' + 16 * ' ' + ' AVG DIV AVG DIV') if depth >= 0: print(queue.tag + (22 - len(queue.tag))*' ' + \ '%8.3f' % queue.data.get_pending(), \ ' %8.3f ' % queue.data.get_pending_div(), \ ' %8.3f' % queue.data.get_mem_usage(), \ ' %8.3f' % queue.data.get_mem_usage_div()) else: print('-'*depth + queue.tag, '(slowdown: %.3f' % queue.data.get_slowdown(), \ 'div: %.3f)' % queue.data.get_slowdown_div(), \ '(mem usage: %.3f' % queue.data.get_mem_usage(), \ 'div: %.3f)' % queue.data.get_mem_usage_div()) if self.is_leaf(queue.tag) == False: children = self.tree.children(queue.tag) for child in children: self.display_score(child, depth + 2) def display_prediction(self, queue=None, depth=0): if queue is None: queue = self.get_root() print('------------' + utils.get_str_time() + ' PREDICTION ----------') print('QUEUE NAME DESIRED CAPACITY') if depth >= 0: print(queue.tag + (22 - len(queue.tag)) * ' ', ' %8.3f' % queue.data.wish.capacity) # print(queue.tag, 'desired capacity: %.3f' % queue.data.wish.capacity) else: print('-' * depth + queue.tag, 'desired capacity: %.3f' % queue.data.wish.capacity) if self.is_leaf(queue.tag) == False: children = self.tree.children(queue.tag) for child in children: self.display_prediction(child, depth + 2) def write_score(self, path): with open(path, 'a') as f: self.write_score_top_down(output=f) def write_score_top_down_old(self, queue=None, depth=0, output=None): if queue is None: queue = self.get_root() output.writelines( ('\n---------', utils.get_str_time(), ' SCORE ---------\n')) output.writelines(24 * ' ' + ' SLOWDOWN MEMORY USAGE\n') output.writelines('QUEUE NAME' + 16 * ' ' + ' AVG DIV AVG DIV\n') if depth >= 0: output.writelines(queue.tag + (22 - len(queue.tag))*' ' + \ '%8.3f' % queue.data.get_slowdown() + \ ' %8.3f ' % queue.data.get_slowdown_div() + \ ' %8.3f' % queue.data.get_mem_usage() + \ ' %8.3f' % queue.data.get_mem_usage_div() + '\n') """ output.writelines( (queue.tag, ' (slowdown: %.3f' % queue.data.get_slowdown(), \ ' div: %.3f)' % queue.data.get_slowdown_div(), \ ' (mem usage: %.3f' % queue.data.get_mem_usage(), \ ' div: %.3f)' % queue.data.get_mem_usage_div(), '\n')) """ else: output.writelines(('-'*depth + queue.tag, ' (slowdown: %.3f' % queue.data.get_slowdown(), \ ' div: %.3f)' % queue.data.get_slowdown_div(), \ ' (mem usage: %.3f' % queue.data.get_mem_usage(), \ ' div: %.3f)' % queue.data.get_mem_usage_div(), '\n')) if self.is_leaf(queue.tag) == False: children = self.tree.children(queue.tag) for child in children: self.write_score_top_down(child, depth + 2, output) def write_score_top_down(self, queue=None, depth=0, output=None): if queue is None: queue = self.get_root() output.writelines( ('\n---------', utils.get_str_time(), ' SCORE ---------\n')) output.writelines(24 * ' ' + ' PENDING MEMORY USAGE\n') output.writelines('QUEUE NAME' + 16 * ' ' + ' AVG DIV AVG DIV\n') output.writelines(queue.tag + (22 - len(queue.tag))*' ' + \ '%8.3f' % queue.data.get_pending() + \ ' %8.3f ' % queue.data.get_pending_div() + \ ' %8.3f' % queue.data.get_mem_usage() + \ ' %8.3f' % queue.data.get_mem_usage_div() + '\n') if self.is_leaf(queue.tag) == False: children = self.tree.children(queue.tag) for child in children: self.write_score_top_down(child, depth + 2, output) def write_prediction(self, path): with open(path, 'a') as f: self.write_prediction_top_down(output=f) def write_prediction_top_down(self, queue=None, depth=0, output=None): if queue is None: queue = self.get_root() output.writelines(('\n---------', utils.get_str_time(), ' PREDICTION---------\n')) output.writelines('QUEUE NAME DESIRED CAPACITY\n') if depth >= 0: output.writelines(queue.tag + (22 - len(queue.tag)) * ' ' + ' %8.3f' % queue.data.wish.capacity + '\n') # output.writelines( (queue.tag, ' desired capacity: %.3f' % queue.data.wish.capacity, '\n')) else: output.writelines(('-'*depth + queue.tag, \ ' desired capacity: %.3f' % queue.data.wish.capacity, '\n')) if self.is_leaf(queue.tag) == False: children = self.tree.children(queue.tag) for child in children: self.write_prediction_top_down(child, depth + 2, output) def add_job(self, job, qname): queue = self.tree.get_node(qname) if queue.is_leaf(): queue.data.add_job(job) else: print("Canot add jobs to parent queue", queue.tag, queue.identifier) def add_metric(self, qname): queue = self.tree.get_node(qname) queue.data.add_metric(queue.cur_metric) if len(queue.data.metrics) > RMQueue.MAX_METRIC_COUNT: del queue.data.metrics[0] def remove_queue(self, qname): """ Remove a queue indicated by 'qname'; all the successors are removed as well. Return the number of removed nodes. """ self.tree.remove_node(qname) def move_queue(self, src, dest): """ Move a queue indicated by @src parameter to be a child of @dest. """ self.tree.move_node(src, dest) def get_queue(self, qname): return self.tree.get_node(qname) def get_root(self): return self.get_queue('root') def is_leaf(self, qname): queue = self.tree.get_node(qname) return queue.is_leaf() def cal_slowdown(self, queue=None): """ if current queue is a leaf queue: calculate the average slowdown in is jobs. else: calculate the average slowdown of its chilren; calculate the average slowdown of current queue through its chilren's average slowdown. """ if queue is None: queue = self.get_root() avg_slowdown = 0.0 if queue.is_leaf(): job_count = len(queue.data.jobs) for i in list(range(job_count)): job = queue.data.jobs[i] slowdown = (job.wait_time + job.run_time) / job.run_time avg_slowdown += slowdown / job_count queue.data.set_job_count(job_count) queue.data.cur_metric.slowdown = avg_slowdown else: # parent queue # First, get its all chilren queue, and call each child's cal_slowdown function children = self.tree.children(queue.tag) for child in children: self.cal_slowdown(child) # Second, get the job count job_count = 0 for child in children: job_count += child.data.get_job_count() queue.data.set_job_count(job_count) # Finally, calculate the average slowdown of the queue if job_count == 0: queue.data.cur_metric.slowdown = avg_slowdown return avg_slowdown for child in children: avg_slowdown += child.data.get_job_count( ) * child.data.get_slowdown() / job_count queue.data.cur_metric.slowdown = avg_slowdown return queue.data.get_slowdown() def cal_pending(self, queue=None): """ if current queue is a leaf queue: calculate the average pending count in is pendings. else: calculate the average pending of its chilren; calculate the pending of current queue through the sum all of its chilren's pending. """ if queue is None: queue = self.get_root() if queue.is_leaf(): queue.data.cal_leaf_pending() else: # parent queue # First, get its all chilren queue, and call each child's cal_pending function # Second, get the sum of all its children pending children = self.tree.children(queue.tag) for child in children: self.cal_pending(child) queue.data.cur_metric.pending += child.data.get_pending() return queue.data.get_pending() def cal_pending_division(self, queue=None): """ if current queue is a leaf queue: stdDivision is zero. else: calculate the standard division of its chilren; calculate the standard division of current queue through its chilren's average pending. """ if queue is None: queue = self.get_root() division = 0.0 if self.is_leaf(queue.tag): return division else: # parent queue children = self.tree.children(queue.tag) # First, get its all chilren queue, and call each child's calSlowDown function for child in children: self.cal_pending_division(child) # Second, calculate the square sum of division count = len(children) avg_pending = queue.data.get_pending() * 1.0 / count squareSum = 0.0 for child in children: squareSum += np.square(child.data.get_pending() - avg_pending) # Finally, calculate the standard division of the queue # if count == 0: # queue.data.cur_metric.slowdown_div = division # return division division = np.sqrt(squareSum / count) queue.data.cur_metric.pending_div = division return division def cal_slowdown_division(self, queue=None): """ if current queue is a leaf queue: stdDivision is zero. else: calculate the standard division of its chilren; calculate the standard division of current queue through its chilren's average slowdown. """ if queue is None: queue = self.get_root() division = 0.0 if self.is_leaf(queue.tag): return division else: # parent queue children = self.tree.children(queue.tag) # First, get its all chilren queue, and call each child's calSlowDown function for child in children: self.cal_slowdown_division(child) # Second, calculate the square sum of division squareSum = 0.0 count = len(children) for child in children: squareSum += np.square(child.data.get_slowdown() - queue.data.get_slowdown()) # Finally, calculate the standard division of the queue # if count == 0: # queue.data.cur_metric.slowdown_div = division # return division division = np.sqrt(squareSum / count) queue.data.cur_metric.slowdown_div = division return division def cal_memory_usage_old(self, queue=None): """ if current queue is a leaf queue: MemoryUsage is the (sum of job memorySeconds )/(self.absMemory * CAL_INTERVAL_IN_SECOND) Get absUsedMemory by self.memoryUsage * self.absMemory else: calculate the memory usage of its chilren; calculate the absolute used memory of the queue. MemoryUsage = absUsedMemory / absMemory """ if queue is None: queue = self.get_root() memory_usage = 0.0 if queue.is_leaf(): total_memory_seconds = queue.data.cal_leaf_mem_second() total_memory_capacity = queue.data.get_abs_memory( ) * RMQueue.CAL_INTERVAL_IN_SECOND memory_usage = 1.0 * total_memory_seconds / total_memory_capacity queue.data.set_mem_usage(memory_usage) queue.data.cal_abs_used_memory() else: # parent queue # First, get its all chilren queue, and call each child's calMemoryUsage function children = self.tree.children(queue.tag) for child in children: self.cal_memory_usage_old(child) # Second, calculate the absUsedMemory of current queue abs_used_memory = 0 for child in children: abs_used_memory += child.data.get_abs_used_memory() queue.data.set_abs_used_memory(abs_used_memory) # Finally, calculate the memory usage of the queue queue.data.set_mem_usage(1.0 * queue.data.get_abs_used_memory() / queue.data.get_abs_memory()) return queue.data.get_mem_usage() def cal_memory_usage(self, queue=None): """ if current queue is a leaf queue: MemoryUsage is the abs_memoryusage/self.absMemoryCapacity else: calculate the memory usage of its chilren; calculate the absolute memory of the queue. MemoryUsage = absUsedMemory / absMemory """ if queue is None: queue = self.get_root() memory_usage = 0.0 if queue.is_leaf(): abs_memory_usage = queue.data.cal_queue_memory_usage() abs_memory_capacity = queue.data.get_abs_capacity() memory_usage = 100.0 * abs_memory_usage / abs_memory_capacity queue.data.set_mem_usage(memory_usage) queue.data.set_abs_memory_usage(abs_memory_usage) else: # parent queue # First, get its all chilren queue, and call each child's calMemoryUsage function children = self.tree.children(queue.tag) for child in children: self.cal_memory_usage(child) # Second, calculate the absUsedMemoryUsage of current queue abs_memory_usage = 0 for child in children: abs_memory_usage += child.data.get_abs_memory_usage() queue.data.set_abs_memory_usage(abs_memory_usage) # Finally, calculate the memory usage of the queue queue.data.set_mem_usage(100.0 * queue.data.get_abs_memory_usage() / queue.data.get_abs_capacity()) return queue.data.get_mem_usage() def cal_mem_usage_division(self, queue=None): """ if current queue is a leaf queue: memUsageDivision is zero. else: calculate the standard division of its chilren; calculate the standard division of current queue through its chilren's average memoryUsage """ if queue is None: queue = self.get_root() std_division = 0.0 if self.is_leaf(queue.tag): queue.data.cur_metric.mem_usage_div = std_division return std_division else: # parent queue # First, get its all chilren queue, and call each child's calSlowDown function children = self.tree.children(queue.tag) for child in children: self.cal_mem_usage_division(child) # Second, calculate the average memory usage of all its children count = len(children) total_mem_usage = 0 for child in children: total_mem_usage += child.data.get_mem_usage() # print(child.data.get_mem_usage()) avg_mem_usage = total_mem_usage / count # Finally, calculate the standard division of the queue squareSum = 0 for child in children: squareSum += np.square(child.data.get_mem_usage() - avg_mem_usage) std_division = np.sqrt(squareSum / count) queue.data.cur_metric.mem_usage_div = std_division return std_division def cal_abs_capacity_bottom_up(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = 0 for child in children: # print("Queue name: %s, abs_capacity: %.2f" %(child.tag, child.data.get_abs_capacity())) self.cal_abs_capacity_bottom_up(child) abs_capacity += child.data.get_abs_capacity() queue.data.set_abs_capacity(abs_capacity) def cal_desired_abs_capacity_bottom_up(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = 0.0 fixed_capacity = 0.0 for child in children: self.cal_desired_abs_capacity_bottom_up(child) if child.data.config.fixed: # print("FIXED") # print(child.data.config.capacity) # print(child.data.config.abs_capacity) fixed_capacity += child.data.config.capacity else: abs_capacity += child.data.wish.abs_capacity for child in children: if child.data.config.fixed: child.data.wish.abs_capacity = abs_capacity / ( 100.0 - fixed_capacity) * child.data.config.capacity queue.data.wish.abs_capacity = abs_capacity * 100.0 / ( 100.0 - fixed_capacity) def clear_desired_abs_capacity(self, queue=None): if queue is None: queue = self.get_root() queue.data.wish.abs_capacity = 0 if self.is_leaf(queue.tag): return else: queue.data.cur_metric.pending = 0.0 children = self.tree.children(queue.tag) for child in children: self.clear_desired_abs_capacity(child) def cal_abs_capacity_top_down(self, queue=None): """ This function calculate the abs capacity of each queue by its capacity. This function should only be called once at the start time. """ if queue is None: queue = self.get_root() queue.data.set_abs_capacity(100.0) if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) for child in children: child.data.set_abs_capacity(queue.data.get_abs_capacity() * child.data.get_capacity() / 100.0) # print(child.data.get_abs_capacity()) self.cal_capacity_top_down(child) def cal_desired_capacity_top_down(self, queue=None): if queue is None: queue = self.get_root() queue.data.wish.capacity = 100.0 if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = queue.data.wish.abs_capacity remain_capaciy = 100.0 for child in children: if child.data.config.fixed: child.data.wish.capacity = child.data.config.capacity elif abs_capacity == 0: child.data.wish.capacity = 0 else: child.data.wish.capacity = child.data.wish.abs_capacity / abs_capacity * 100.0 self.cal_desired_capacity_top_down(child) def cal_capacity_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) abs_capacity = queue.data.get_abs_capacity() for child in children: if abs_capacity == 0: child.data.set_capacity(0) else: child.data.set_capacity(child.data.get_abs_capacity() / abs_capacity * 100) self.cal_capacity_top_down(child) def cal_abs_memory_top_down(self, queue=None): if queue is None: queue = self.get_root() queue.data.cal_totalMb_mean() queue.data.clear_totalMb() if self.is_leaf(queue.tag): return else: children = self.tree.children(queue.tag) for child in children: child.data.set_abs_memory(queue.data.get_abs_memory() * child.data.get_capacity() / 100) self.cal_abs_memory_top_down(child) def clear_mus_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): queue.data.clear_queue_memory_usage() else: children = self.tree.children(queue.tag) for child in children: self.clear_mus_top_down(child) def clear_jobs_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): queue.data.clear_jobs() else: children = self.tree.children(queue.tag) for child in children: self.clear_jobs_top_down(child) def clear_pendings_top_down(self, queue=None): if queue is None: queue = self.get_root() if self.is_leaf(queue.tag): queue.data.clear_pendings() else: children = self.tree.children(queue.tag) for child in children: self.clear_pendings_top_down(child) def before_scoring(self): self.cal_abs_capacity_bottom_up() self.cal_capacity_top_down() self.cal_abs_memory_top_down() def after_scoreing(self): self.clear_jobs_top_down() self.clear_pendings_top_down() self.clear_mus_top_down() def score(self): self.before_scoring() # self.cal_slowdown() # self.cal_slowdown_division() self.cal_pending() self.cal_pending_division() self.cal_memory_usage() self.cal_mem_usage_division() self.after_scoreing() def before_predict(self): self.cal_desired_abs_capacity_bottom_up() def after_predict(self): self.clear_desired_abs_capacity() def predict(self): self.before_predict() self.cal_desired_capacity_top_down() self.after_predict()
inner_bags = get_inner_bags(split_line[1]) existing_bags = find_existing_bag(baggage_claim, outer_bag) if (len(existing_bags) > 0): for bag in existing_bags: for i in inner_bags: baggage_claim.create_node(i, parent=bag.identifier) else: outer_bag_node = baggage_claim.create_node(outer_bag, parent='baggage claim') for i in inner_bags: inner_existing = find_existing_top_level_bag(baggage_claim, i) print(f'inner_existing {inner_existing}') if (len(inner_existing) > 0): #tree.move(node, new_parent) baggage_claim.move_node(inner_existing[0].identifier, outer_bag_node.identifier) # if inner already exists # move existing else: baggage_claim.create_node(i, parent=outer_bag_node.identifier) baggage_claim.show() shiny = list( baggage_claim.filter_nodes(lambda n: n.tag == 'shiny gold bag')) print(f'shiny gold bags: {len(shiny)}') for child in baggage_claim.children('baggage claim'): pass # print(f'{child.tag} {baggage_claim.children(child.identifier)[0]}')
if words[0] == 'extmodule': words = line.split() extmodule = words[1] # print(extmodule) tree.create_node(extmodule + '(ext)', extmodule, parent=circuit) cur_module = extmodule if words[ 0] == 'inst': # @inst, re-assign the parent of inst_module from top to cur_module words = line.split() inst_name = words[1] inst_module = words[3] # print(inst_module) # print(cur_module) tree.move_node(inst_module, cur_module) tree.show() total_parent_cnt = 0 total_children_cnt = 0 total_modules = 0 for node in tree.expand_tree(mode=Tree.DEPTH): total_modules += 1 if (tree[node].is_leaf()): continue total_parent_cnt += 1 total_children_cnt += len(tree.children(node)) tmp_list = []
class Mwrp: def __init__(self, world, number_of_agent, start_node): self.number_of_agent = number_of_agent self.open_list = np.array([start_node]) self.tree = Tree() self.tree.create_node(start_node.pos.__str__(), start_node.pos.__str__(), data=start_node) self.world = world self.node_expend_index = 1 self.need_to_see = np.sum(self.world.grid_map == 0) def insert_to_open_list(self, new_node): for index, data in enumerate(self.open_list): if data.f > new_node.f: self.open_list = np.insert(self.open_list, index, new_node) return self.open_list = np.append(self.open_list, new_node) def pop_open_list(self): return self.open_list[0].pos def move_from_open_to_close(self, index=0): self.open_list = np.delete(self.open_list, index) def heuristic(self, state): return 1 # #old # def fix_g_subtree(self,neighbor,state): # tmp_new_open = [] # #self.tree.move_node(neighbor.__str__(), state.__str__()) # # for node in mwrp.tree.subtree(state.data.pos.__str__()).expand_tree(mode=Tree.DEPTH): # parent = mwrp.tree.parent(node) # # new_g = parent.data.g + Utils.n_dim_distance( # self.tree.get_node(node).data.pos, parent.data.pos) # self.tree.get_node(node).data.f = self.tree.get_node(node).data.f - self.tree.get_node(node).data.g + new_g # # self.tree.get_node(node).data.g = new_g # tmp_new_open.append(self.tree.get_node(node)) # # for need_new_open in tmp_new_open: # for index, data in enumerate(self.open_list): # if (np.all(data.pos == need_new_open.data.pos)): # self.open_list = np.delete(self.open_list, index) # self.insert_to_open_list(self.tree.get_node(need_new_open.data.pos.__str__()).data) # break def get_all_seen(self, state): tmp_node = self.tree.get_node(state.identifier) tmp_seen = self.world.get_seen(tmp_node) while not self.tree.get_node(tmp_node.identifier).is_root(): tmp_node = self.tree.get_node( tmp_node.predecessor(self.tree.identifier)) tmp_seen = np.vstack((tmp_seen, self.world.get_seen(tmp_node))) all_seen = np.unique(tmp_seen, axis=0) return all_seen # def get_all_seen1(self, state): # # tmp_node = self.tree.get_node(state.identifier) # dictOfWords = {i.__str__(): 0 for i in self.world.get_seen(tmp_node)} # # while not self.tree.get_node(tmp_node.identifier).is_root(): # tmp_node = self.tree.get_node(tmp_node.predecessor(self.tree.identifier)) # for seen in self.world.get_seen(tmp_node): # if not seen.__str__() in dictOfWords: # dictOfWords[seen.__str__()]=0 # # return dictOfWords # def expend_all(self,state): # for neighbor in self.world.get_neighbors(state): # if self.world.in_bund(neighbor) and self.world.is_obstical(neighbor): # new_g = mwrp.tree.get_node(state.__str__()).data.g + Utils.n_dim_distance(state, neighbor) # if not self.tree.get_node(neighbor.__str__()): # h = self.heuristic(state) # new_node=Node(neighbor,new_g,new_g+h) # self.tree.create_node(neighbor.__str__(),neighbor.__str__(),parent=(state.__str__()),data=new_node) # self.insert_to_open_list(new_node) # else: # self.fix_g(neighbor, state) # # #self.tree.show() # self.move_from_open_to_close() def expend(self, state): move_index = np.zeros(self.number_of_agent).astype(int) for i in range(LOS**number_of_agent): for j in range(number_of_agent): i, index = divmod(i, LOS) move_index[j] = index neighbor = self.world.get_one_neighbor(state, move_index) if self.world.in_bund(neighbor) and self.world.is_obstical( neighbor): new_g = mwrp.tree.get_node( state.__str__()).data.g + Utils.n_dim_distance( state, neighbor) if not self.tree.contains(neighbor.__str__()): h = self.heuristic(state) new_node = Node(neighbor, new_g, new_g + h) self.tree.create_node(neighbor.__str__(), neighbor.__str__(), parent=(state.__str__()), data=new_node) self.insert_to_open_list(new_node) else: if new_g < self.tree.get_node(neighbor.__str__()).data.g: self.fix_g(neighbor, state) self.move_from_open_to_close() def fix_g(self, old_state, new_parent): state = self.tree.get_node(old_state.__str__()) old_parent = self.tree.get_node(state.predecessor( self.tree.identifier)) new_parent = self.tree.get_node(new_parent.__str__()) if self.seen_comparison(new_parent, old_parent): self.tree.move_node(state.identifier, new_parent.identifier) new_g = new_parent.data.g + \ Utils.n_dim_distance(self.tree.get_node(state.identifier).data.pos, new_parent.data.pos) self.tree.get_node(state.identifier).data.f = self.tree.get_node(state.identifier).data.f - \ self.tree.get_node(state.identifier).data.g + new_g self.tree.get_node(state.identifier).data.g = new_g for index, data in enumerate(self.open_list): if np.all(data.pos == state.data.pos): self.open_list = np.delete(self.open_list, index) self.insert_to_open_list( self.tree.get_node(state.identifier).data) break def goal_test(self, state): seen_number = self.world.get_seen(state).shape[0] if (seen_number == self.need_to_see): return True return False def seen_comparison(self, state_new, state_old): seen_new = self.get_all_seen(state_new).tolist() seen_old = self.get_all_seen(state_old).tolist() if seen_new.shape[0] < seen_old.shape[0]: return False for one_seen in seen_old: if one_seen not in seen_new: return False return True
from treelib import Tree, Node if __name__ == '__main__': # 树的创建,每个节点都有唯一的identifier作为标记,可以手动指定 tree = Tree() # 增加树的节点,tag是树输出时的显示,identifier是唯一标志,根节点可以不指定父 tree.create_node(tag='root', identifier='root', data=0) tree.create_node(tag='1_child', identifier='1_child', data=1, parent='root') tree.create_node(tag='2_child', identifier='2_child', data=2, parent='root') tree.create_node(tag='3_child', identifier='3_child', data=3, parent='1_child') # 树的粘贴,需要注意的是这个nid是tree的identifier,不是tree2的 tree2 = Tree() tree2.create_node(tag='tutu', identifier='tutu', data=0) tree.paste(nid='root', new_tree=tree2) # 删除树的节点 tree.remove_node('tutu') # 移动树的节点 tree.move_node('3_child', 'root') # 打印树的结构 tree.show()
def TreeSequenceToTreeClass(simulation, tree_event_sequence, is_AA_mutation_in_root_node=False): t1 = time.time() tree_size = len(tree_event_sequence.tree_sequence) tree_class_tree = Tree() root_id = 'Unknown' array_tree = simulation.GetTree() for i in range(tree_size): if array_tree[i] == -1: root_id = i tree_class_tree.create_node(root_id, root_id, data=None) # placeholder on root break # there can be only one root if root_id == 'Unknown': raise ValueError("There is no root in this tree") for i in range(tree_size): if i != root_id: tree_class_tree.create_node( i, i, parent=root_id, data=None) # placeholder on other places for i in range(tree_size): if i != root_id: tree_class_tree.move_node(i, array_tree[i]) for i in range(tree_size): noc = len(tree_class_tree.get_node(i).fpointer) # number of children ni = tree_event_sequence.tree_sequence[i].node_id iam = tree_event_sequence.tree_sequence[i].is_a_mutation on = tree_event_sequence.tree_sequence[i].old_nucleotyde nn = tree_event_sequence.tree_sequence[i].new_nucleotyde mc = tree_event_sequence.tree_sequence[i].mutation_cite tti = tree_event_sequence.tree_sequence[i].tree_time tty = tree_event_sequence.tree_sequence[i].tree_type if (i == root_id) and (is_AA_mutation_in_root_node == True): tree_class_tree.update_node(i, data=TreeEvent(is_a_mutation=True, number_of_children=noc, old_nucleotyde=0, new_nucleotyde=0, mutation_cite=0, tree_time=0, tree_type='coalescence', node_id=ni)) else: tree_event = TreeEvent(is_a_mutation=iam, number_of_children=noc, old_nucleotyde=on, new_nucleotyde=nn, mutation_cite=mc, tree_time=tti, tree_type=tty, node_id=ni) tree_class_tree.update_node(i, data=tree_event) t2 = time.time() print('Time spent on conversion to tree class = ', t2 - t1) return tree_class_tree
print tree.is_branch('diane') print('\n') print("#"*4 + "OOhh~ new members enter Jill's family") new_tree = Tree() new_tree.create_node("n1", 1) # root node new_tree.create_node("n2", 2, parent=1) new_tree.create_node("n3", 3, parent=1) tree.paste('jill', new_tree) tree.show() print('\n') print("#"*4 + "We are sorry they are gone accidently :(") tree.remove_node(1) tree.show() print('\n') print("#"*4 + "Now Jill moves to live with Grand-x-father Harry") tree.move_node('jill', 'harry') tree.show() print('\n') print("#"*4 + "A big family for George to talk to Grand-x-father Harry") for node in tree.rsearch('george', filter=lambda x: x.identifier != 'harry'): print node print('harry') print('\n')