class AcquisitionChain(object): def __init__(self): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() def add(self, master, slave): slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer: raise RuntimeError( "Cannot add acquisition device %s to multiple masters, current master is %s" % (slave, slave_node._bpointer) ) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave_node, master_node) def _execute(self, func_name): tasks = list() prev_level = None for dev in reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]): node = self._tree.get_node(dev) level = self._tree.depth(node) if prev_level != level: gevent.joinall(tasks) tasks = list() func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) gevent.joinall(tasks) def prepare(self, dm, scan_info): # self._devices_tree = self._get_devices_tree() for master in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] for dev in self._tree.get_node(master).fpointer: master.slaves.append(dev) dm_prepare_task = gevent.spawn(dm.prepare, scan_info, self._tree) self._execute("_prepare") dm_prepare_task.join() def start(self): self._execute("_start") for acq_dev in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionDevice)): acq_dev.wait_reading() dispatcher.send("end", acq_dev)
def is_valid_tree(self, parse_tree: Tree): is_violateing = [ self._is_violating_node(node, parse_tree) for node in parse_tree.expand_tree() ] if any(is_violateing): print('here') return not any(is_violateing)
def is_projective_tree(self, parse_tree: Tree): is_violateing = [ len(parse_tree.children(node)) > 2 for node in parse_tree.expand_tree() ] if any(is_violateing): print('here') return not any(is_violateing)
class AcquisitionChain(object): def __init__(self, parallel_prepare=False): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() self._presets_list = list() self._parallel_prepare = parallel_prepare self._device2one_shot_flag = weakref.WeakKeyDictionary() @property def nodes_list(self): nodes_gen = self._tree.expand_tree() nodes_gen.next() # first node is 'root' return list(nodes_gen) def add(self, master, slave): self._device2one_shot_flag.setdefault(slave, False) slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if (slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer): raise RuntimeError( "Cannot add acquisition device %s to multiple masters, current master is %s" % (slave, slave_node._bpointer)) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave, master) slave.parent = master def add_preset(self, preset): self._presets_list.append(preset) def set_stopper(self, device, stop_flag): """ By default any top master device will stop the scan. In case of several top master, you can define which one won't stop the scan """ self._device2one_shot_flag[device] = not stop_flag def __iter__(self): if len(self._tree) > 1: return AcquisitionChainIter( self, parallel_prepare=self._parallel_prepare) else: return iter(())
class AcquisitionChain(object): def __init__(self, parallel_prepare=False): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() self._presets_list = list() self._parallel_prepare = parallel_prepare self._device2one_shot_flag = weakref.WeakKeyDictionary() @property def nodes_list(self): nodes_gen = self._tree.expand_tree() nodes_gen.next() # first node is 'root' return list(nodes_gen) def add(self, master, slave): self._device2one_shot_flag.setdefault(slave, False) slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if(slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer): raise RuntimeError("Cannot add acquisition device %s to multiple masters, current master is %s" % ( slave, slave_node._bpointer)) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node( tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node( tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave, master) slave.parent = master def add_preset(self, preset): self._presets_list.append(preset) def set_stopper(self, device, stop_flag): """ By default any top master device will stop the scan. In case of several top master, you can define which one won't stop the scan """ self._device2one_shot_flag[device] = not stop_flag def __iter__(self): if len(self._tree) > 1: return AcquisitionChainIter(self, parallel_prepare=self._parallel_prepare) else: return iter(())
def _build_tree(self, scores: ndarray, bin_edges: ndarray) -> Tree: # Build tree with specified number of children at each level tree = Tree() tree.add_node(Node()) # root node nodes_prev = [tree.get_node(tree.root)] for level in range(self.depth): nodes_current = [] for node in nodes_prev: children = [] for _ in range(self.n_children[level]): child = Node() tree.add_node(child, parent=node) children.append(child) nodes_current.extend(children) nodes_prev = nodes_current assignments = np.digitize(scores, bin_edges) - 1 # Store instance ids in leaves leaves = tree.leaves() for k, node in enumerate(leaves): instance_ids = np.where(assignments == k)[0] if instance_ids.size == 0: tree.remove_node(node.identifier) else: node.data = instance_ids # Prune empty leaves check_for_empty_leaves = True while check_for_empty_leaves: check_for_empty_leaves = False leaves = tree.leaves() for node in leaves: if node.data is None and len(node.successors( tree.identifier)) == 0: # Node is empty and has no siblings tree.remove_node(node.identifier) check_for_empty_leaves = True # Simplify tree: remove nodes that only have one child for nid in tree.expand_tree(mode=tree.WIDTH): children = tree.children(nid) if len(children) == 1: tree.link_past_node(nid) return tree
def do_md_section(sysid, levelid, connection_str, output_format, output_file, compact, doc_handler, dotree): """ Given the MD ids, dump the content in the output file and produce the product tree diagrams :param sysid: MagicDraw subsystem id :param levelid: MagicDraw level id (package id containing the tree to extract) :param connection_str: MagicDraw encoded connection string :param output_format: OutputFormat :param output_file: File to dump :param compact: True if the portrait diagrams have to be in compact form :param doc_handler: the document id :param sub_name: name of the subsystem :return: none """ global template_path global productTree global tree_dict productTree = Tree() products = [] tree_dict = {} # get the information from MagicDraw mdr = build_md_tree(sysid, levelid, connection_str) print("\n Product tree depth:", productTree.depth()) nodes = productTree.expand_tree(key=lambda x: x.data.index, sorting=True) for n in nodes: products.append(productTree[n].data) tree_dict[productTree[n].data.id] = productTree[n].data print(f" Found {{np}} products (including container folders).".format( np=len(tree_dict))) envs = Environment(loader=ChoiceLoader([ FileSystemLoader(Config.TEMPLATE_DIRECTORY), PackageLoader('ptree', 'templates') ]), lstrip_blocks=True, trim_blocks=True, autoescape=None) # dump a csv file if dotree: do_csv(products, output_file) # create the diagrams tex files do_trees_diagrams(productTree, output_file, products[0].shortname, compact) # define the template template_path = f"section.{Config.TEMPLATE_LANGUAGE}.jinja2" else: # no diagrams are created # use a simplified template template_path = f"notree_section.{Config.TEMPLATE_LANGUAGE}.jinja2" # sort tree dictionary based mdp = productTree.to_dict(with_data=False) # get ordered dictionary template = envs.get_template(template_path) new_mdpt = dict() for k0 in mdp: new_mdpt[k0] = order_tree_level(mdp[k0]) # dump the tex section metadata = dict() metadata["template"] = template.filename text = template.render(metadata=metadata, mdrev=mdr, filename=output_file, doc_handler=doc_handler, mdt_dict=tree_dict, mdp=new_mdpt, mdps=products) tex_file_name = output_file + ".tex" file = open(tex_file_name, "w") print(_as_output_format(text, output_format), file=file) file.close() return tree_dict
tree.create_node("Harry", "harry") # root node tree.create_node("Jane", "jane", parent="harry") tree.create_node("Bill", "bill", parent="harry") tree.create_node("Diane", "diane", parent="jane") tree.create_node("George", "george", parent="diane") tree.create_node("Mary", "mary", parent="diane") tree.create_node("Jill", "jill", parent="george") tree.create_node("Mark", "mark", parent="jane") sep = "-"*20 + '\n' print(sep + "Tree of the whole family:") tree.show(key=lambda x: x.tag, reverse=True) print(sep + "All family members in DEPTH mode:") for node in tree.expand_tree(mode=Tree.ZIGZAG): print(tree[node].tag) print(sep + "All family members without Diane sub-family:") tree.show(idhidden=False, filter=lambda x: x.identifier != 'diane') # for node in tree.expand_tree(filter=lambda x: x.identifier != 'diane', mode=Tree.DEPTH): # print tree[node].tag print(sep + "Let me introduce Diane family only:") sub_t = tree.subtree('diane') sub_t.show() print(sep + "Children of Diane") for child in tree.is_branch('diane'): print(tree[child].tag)
class ClusterTree(object): """Alternat clustering implementation class. :param object: [description] :type object: [type] :return: [description] :rtype: [type] """ # the bounding boxes should fall within threshold * 100 of width or height OVERLAP_THRESHOLD = 0.95 def __init__(self): self.tree = Tree() self.root = "root" self.tree.create_node(self.root, self.root, data=[]) def add_node(self, parent_name: str, node_name: str, data): """Adds a node in the tree (uses treelib). :param parent_name: Tag of the parent in the tree. :type parent_name: str :param node_name: Tag of the node to be added in the tree. :type node_name: str :param data: Data to be stored in the node. :type data: [type] """ self.tree.create_node(node_name.upper(), node_name, data=data, parent=parent_name) def _find_next_overlapping_line_top(self, lines_sorted_top, current_line, line_with_max_bottom): """Find the next overlapping element based on top values of line. :param lines_sorted_top: Sorted list of lines based on their top values. :type lines_sorted_top: [type] :param current_line: Current line object. :type current_line: [type] :param line_with_max_bottom: Line with max bottom value. :type line_with_max_bottom: [type] :return: [description] :rtype: [type] """ starting_index = lines_sorted_top.index(current_line) remaining_lines = lines_sorted_top[starting_index + 1:] max_line_height = line_with_max_bottom["bottom"] - line_with_max_bottom["top"] threshold_height = max_line_height * self.OVERLAP_THRESHOLD overlapping_lines = list(filter(lambda line: (line["top"] >= current_line["top"]) and (line["top"] < (line_with_max_bottom["top"] + threshold_height)), remaining_lines)) if len(overlapping_lines) > 0: next_overlapping_line = overlapping_lines[0] new_line_with_max_bottom = line_with_max_bottom if next_overlapping_line["bottom"] > line_with_max_bottom["bottom"]: new_line_with_max_bottom = next_overlapping_line return [next_overlapping_line, new_line_with_max_bottom] else: return [None, line_with_max_bottom] def _find_next_overlapping_line_left(self, lines_sorted_left, current_line, line_with_max_right): """Find the next overlapping element based on left values of line. :param lines_sorted_left: Sorted list of lines based on their left values. :type lines_sorted_left: [type] :param current_line: Current line object. :type current_line: [type] :param line_with_max_right: Line with max right value. :type line_with_max_right: [type] :return: [description] :rtype: [type] """ starting_index = lines_sorted_left.index(current_line) remaining_lines = lines_sorted_left[starting_index + 1:] max_line_right = line_with_max_right["right"] - line_with_max_right["left"] threshold_width = max_line_right * self.OVERLAP_THRESHOLD overlapping_lines = list(filter(lambda line: (line["left"] >= current_line["left"]) and (line["left"] < (line_with_max_right["left"] + threshold_width)), remaining_lines)) if len(overlapping_lines) > 0: next_overlapping_line = overlapping_lines[0] new_line_with_max_right = line_with_max_right if next_overlapping_line["right"] > line_with_max_right["right"]: new_line_with_max_right = next_overlapping_line return [next_overlapping_line, new_line_with_max_right] else: return [None, line_with_max_right] def _create_row_clusters(self, data: list): """Create row clusters based on the transformed line data. :param data: Transformed line data with bounding box info in the form {top: value, right: value, bottom: value, left: value}. :type data: list :return: [description] :rtype: [type] """ clusters = [] cluster_number = 0 line_index = 0 sorted_top = sorted(data, key=lambda line: line["top"]) while line_index < len(data): cluster = [] cluster_number += 1 line_with_max_bottom = None next_line_within_height = sorted_top[line_index] line_with_max_bottom = sorted_top[line_index] while next_line_within_height is not None: cluster.append(next_line_within_height) current_line = next_line_within_height overlapping_line_data = self._find_next_overlapping_line_top(sorted_top, current_line, line_with_max_bottom) next_line_within_height = overlapping_line_data[0] line_with_max_bottom = overlapping_line_data[1] sorted_cluster = sorted(cluster, key=lambda line: line["left"]) clusters.append(sorted_cluster) line_index += len(cluster) return clusters def _create_column_clusters(self, data: list): """Creates column clusters in the transformed line data. :param data: Transformed line data with bounding box info in the form {top: value, right: value, bottom: value, left: value}. :type data: list :return: [description] :rtype: [type] """ clusters = [] cluster_number = 0 line_index = 0 sorted_left = sorted(data, key=lambda line: line["left"]) while line_index < len(data): cluster = [] cluster_number += 1 line_with_max_right = None next_line_within_width = sorted_left[line_index] line_with_max_right = sorted_left[line_index] while next_line_within_width is not None: cluster.append(next_line_within_width) current_line = next_line_within_width overlapping_line_data = self._find_next_overlapping_line_left(sorted_left, current_line, line_with_max_right) next_line_within_width = overlapping_line_data[0] line_with_max_right = overlapping_line_data[1] sorted_cluster = sorted(cluster, key=lambda line: line["top"]) clusters.append(sorted_cluster) line_index += len(cluster) return clusters def _combine_nodes(self, data: list): """Combine nodes which are part of the cluster because they are closed to each other :param data: Transformed line data with bounding box info in the form {top: value, right: value, bottom: value, left: value}. :type data: list :return: [description] :rtype: [type] """ node_text = "" sorted_top = sorted(data, key=lambda line: line["top"]) for line_data in sorted_top: node_text += line_data["text"] + " " return { "left": min([line_data["left"] for line_data in sorted_top]), "top": min([line_data["top"] for line_data in sorted_top]), "right": max([line_data["right"] for line_data in sorted_top]), "bottom": max([line_data["bottom"] for line_data in sorted_top]), "text": node_text } def _create_tree(self, cluster_type: str, lines_data: list, parent_name: str, parent_cluster_type: str = None): """Recursive method to cluster lines data into columns and rows clusters. The recursion return when the leaf node is encountered. :param cluster_type: The type cluster to make (ROW / COL). :type cluster_type: str :param lines_data: Transformed line data with bounding box info in the form {top: value, right: value, bottom: value, left: value}. :type lines_data: list :param parent_name: Name / Tag of the parent node. :type parent_name: str :param parent_cluster_type: The type of cluster for parent, defaults to None (ROW) :type parent_cluster_type: str, optional """ row_index = 0 column_index = 0 if cluster_type == "ROW": if len(lines_data) > 1: cluster_data = self._create_row_clusters(lines_data) if parent_cluster_type == "COL" and len(cluster_data) == 1: # add a leaf node with combined data node = self._combine_nodes(cluster_data[0]) node_name = parent_name + "L" self.add_node(parent_name, node_name, { "children": [], "val": node }) else: for data in cluster_data: row_index += 1 node_name = parent_name + "R" + str(row_index) self.add_node(parent_name, node_name, { "children": data }) # print("Row cluster : ", row_index, data, node_name) self._create_tree("COL", data, node_name, "ROW") else: if len(lines_data) == 1: node_name = parent_name + "L" self.add_node(parent_name, node_name, { "children": [], "val": lines_data[0] }) if cluster_type == "COL": if len(lines_data) > 1: cluster_data = self._create_column_clusters(lines_data) if parent_cluster_type == "ROW" and len(cluster_data) == 1: # add a leaf node with combined data node = self._combine_nodes(cluster_data[0]) node_name = parent_name + "L" self.add_node(parent_name, node_name, { "children": [], "val": node }) else: for data in cluster_data: column_index += 1 node_name = parent_name + "C" + str(column_index) self.add_node(parent_name, node_name, { "children": data }) # print("Column cluster : ", column_index, data, node_name) self._create_tree("ROW", data, node_name, "COL") else: if len(lines_data) == 1: node_name = parent_name + "L" self.add_node(parent_name, node_name, { "children": [], "val": lines_data[0] }) @staticmethod def _update_data(lines_data: list): """Transforms ocr bounding box data from rule engine of form [{x: left, y: top}, {x: right, y: top}, {x: right, y: bottom}, {x: left, y: bototm}] to [{top: value, right: value, left: value, bottom: value}] :param lines_data: OCR data after passing from rule engine. :type lines_data: list :return: [description] :rtype: [type] """ updated_lines_data = [] for line in lines_data: left = min([bb["x"] for bb in line["boundingBox"]]) top = min([bb["y"] for bb in line["boundingBox"]]) right = max([bb["x"] for bb in line["boundingBox"]]) bottom = max([bb["y"] for bb in line["boundingBox"]]) new_data = { "top": top, "left": left, "right": right, "bottom": bottom, "confidence": line["confidence"], "text": line["text"] } updated_lines_data.append(new_data) return updated_lines_data def _generate_ocr_text_from_tree(self): """Generates OCR text from the tree. Does the Depth-first traversal to find leaf nodes and appends the text to generate final OCR text. :return: [description] :rtype: [type] """ ocr_text = "" for node in self.tree.expand_tree(mode=Tree.DEPTH): node_id = self.tree[node].tag last_char_node_id = node_id[len(node_id) - 1] # only if the node is the leaf node if last_char_node_id == "L": # print("Running for node ", node_id, self.tree[node].data["val"]["text"]) ocr_text += self.tree[node].data["val"]["text"] + " " return ocr_text def generate_data(self, lines_data): """Main handler for alternat clustering implementation. The method internally creates clusters, save it on the tree and generates the final OCR text from it. :param lines_data: OCR data after passing from rule engine. :type lines_data: [type] :return: [description] :rtype: [type] """ modified_lines_data = self._update_data(lines_data) # # print(" line data : ", modified_lines_data) cluster_type = "ROW" parent_name = self.root # adding a general exception to catch infinite loop error try: self._create_tree(cluster_type, modified_lines_data, parent_name) except: OCRClusteringException() return "" ocr_data = self._generate_ocr_text_from_tree() return ocr_data
def tree_answer(input_json): root_post = input_json["root_post"]["root_post"] posts = input_json["posts"] ids = posts["ids"] text = posts["text"] parent_ids = posts["parent_ids"] scores = posts["scores"] categories = posts["categories"] posts = [] for i in range(len(ids)): posts.append( [ids[i], text[i], parent_ids[i], categories[i], scores[i]]) # root_post: id, text, category, score # other posts: id, text, parent id, category, score tree = Tree() root_post[1] = root_post[1].replace('\n', ' ') for post in posts: post[1] = post[1].replace('\n', ' ') tree.create_node("root post", root_post[0], data=ForumPost(root_post[1], "none", "none")) for post in posts: id = post[0] text = post[1] parent_id = post[2] category = post[3] score = post[4] tree.create_node(str(score) + ": " + category + ": " + text[:20], id, parent=parent_id, data=ForumPost(text, category, score)) """ Stuff now: Go through each child to the root. If solution: In each subtree, find number of helpfuls, add up. Put the child: [name, # of helpfuls] in a results array """ children_to_root = [tree[node].identifier for node in tree.expand_tree()] results = [] for identifier in children_to_root: if (tree[identifier].data.category == "solution"): sub_t = tree.subtree(identifier) sub_t_scores = [ tree[node].data.score for node in sub_t.expand_tree() ] sub_t_categories = [ tree[node].data.category for node in sub_t.expand_tree() ] total_score = 0 for i in range(len(sub_t_scores)): if sub_t_categories[i] != "other": total_score += sub_t_scores[i] results.append([total_score, tree[identifier].data.text]) results.sort(key=lambda x: x[0], reverse=True) score = [] post = [] for i in results: score.append(i[0]) post.append(i[1]) return (score, post)
tree.create_node("Child2", "Child2", data={"Name": "Brian"}, parent="Root") tree.create_node("Child3", "Child3", data={"Name": "Bridget"}, parent="Root") tree.create_node("Grand_Child2", "Grand_Child2", data={"Name": "Brima"}, parent="Child3") print(tree) # def func1(x): # return True if x.data["Name"] != "Bruce" else False # for node in tree.expand_tree(filter=func1): # print(tree.get_node(node).data) # for node in tree.expand_tree(filter=lambda x: x.identifier != "Child1"): # print(node) root_childrens_list = tree.children(tree.root) print(tree.depth()) for tag in tree.expand_tree(mode=Tree.DEPTH, reverse=False): print(tag) # for cn in root_childrens_list: # print("##################################################") # sub_tree = tree.subtree(cn.identifier) # depth = sub_tree.depth() # print(sub_tree) # print("{}".format(depth)) # for tag in sub_tree.expand_tree("Great_Grand_Child1", reverse=True): # print(tag)
def construct_celltree(nucleus_file, config): ''' Construct cell tree structure with cell names :param nucleus_file: the name list file to the tree initilization :param max_time: the maximum time point to be considered :return cell_tree: cell tree structure where each time corresponds to one cell (with specific name) ''' ## Construct cell # Add unregulized naming cell_tree = Tree() cell_tree.create_node('P0', 'P0') cell_tree.create_node('AB', 'AB', parent='P0') cell_tree.create_node('P1', 'P1', parent='P0') cell_tree.create_node('EMS', 'EMS', parent='P1') cell_tree.create_node('P2', 'P2', parent='P1') cell_tree.create_node('P3', 'P3', parent='P2') cell_tree.create_node('C', 'C', parent='P2') cell_tree.create_node('P4', 'P4', parent='P3') cell_tree.create_node('D', 'D', parent='P3') cell_tree.create_node('Z2', 'Z2', parent='P4') cell_tree.create_node('Z3', 'Z3', parent='P4') # EMS cell_tree.create_node('E', 'E', parent='EMS') cell_tree.create_node('MS', 'MS', parent='EMS') # Read the name excel and construct the tree with complete SegCell df_time = pd.read_csv(nucleus_file) # read and combine all names from different acetrees ## Get cell number try: with open('./ShapeUtil/number_dictionary.txt', 'rb') as f: number_dictionary = pickle.load(f) except: ace_files = glob.glob('./ShapeUtil/AceForLabel/*.csv') cell_list = [x for x in cell_tree.expand_tree()] for ace_file in ace_files: ace_pd = pd.read_csv(os.path.join(ace_file)) cell_list = list(ace_pd.cell.unique()) + cell_list cell_list = list(set(cell_list)) cell_list.sort() number_dictionary = dict(zip(cell_list, range(1, len(cell_list) + 1))) with open('./ShapeUtil/number_dictionary.txt', 'wb') as f: pickle.dump(number_dictionary, f) with open('./ShapeUtil/name_dictionary.txt', 'wb') as f: pickle.dump(dict(zip(range(1, len(cell_list) + 1), cell_list)), f) max_time = config.get('max_time', 100) df_time = df_time[df_time.time <= max_time] all_cell_names = list(df_time.cell.unique()) for cell_name in list(all_cell_names): if cell_name not in number_dictionary: continue times = list(df_time.time[df_time.cell == cell_name]) cell_info = cell_node() cell_info.set_number(number_dictionary[cell_name]) cell_info.set_time(times) if not cell_tree.contains(cell_name): if "Nuc" not in cell_name: parent_name = cell_name[:-1] cell_tree.create_node(cell_name, cell_name, parent=parent_name, data=cell_info) else: cell_tree.update_node(cell_name, data=cell_info) return cell_tree, max_time
def main(): total_cells_list = [ 100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000, 1000000 ] # total_cells_list = [100000] # total_cells_list = [600000] # total_cells_list = [1000, 2000, 3000, 3610, 4000, 5000, 6000, 6890, 7000, 8000] # total_cells_list = [100000] # total_cells_list = [100] avg_children = 4 avg_module_size = 300 # small BOOM avg sd = 1658 # small BOOM SD # avg_children = 4 # avg_module_size = 4 # # sd = 1658 # sd = 1 for total_cells in total_cells_list: theory_module_num = total_cells / avg_module_size depth = math.ceil(math.log(theory_module_num, avg_children)) # virtual tree construction and distribute the cells to modules tree = Tree() top_name = "" left_cells = total_cells module_get_assigned = 0 total_assigned_cells = 0 total_created_modules = 0 for d in range(depth + 1): if (d == 0): # get cell numbers assigned_cells = random.randint(0, avg_module_size * 2) # assigned_cells = 0 # while True : # # assigned_cells = int(random.gauss(avg_module_size, sd)) # assigned_cells = random.randint(0,avg_module_size*2) # if assigned_cells > 0 : # break module_get_assigned += 1 total_assigned_cells += assigned_cells left_cells -= assigned_cells # create_top top_name = "Snxn" + str(int(total_cells / 1000)) + "k" tree.create_node( str(assigned_cells) + " " + top_name, top_name) # encode cell number at the node name total_created_modules += 1 elif (d == 1): for i in range(avg_children): if left_cells < 0: break assigned_cells = random.randint(0, avg_module_size * 2) # assigned_cells = 0 # while True : # # assigned_cells = int(random.gauss(avg_module_size, sd)) # assigned_cells = random.randint(0,avg_module_size*2) # if assigned_cells > 0 : # break module_get_assigned += 1 total_assigned_cells += assigned_cells left_cells -= assigned_cells print("module%d: at level %d, %d cells" % (module_get_assigned, d, assigned_cells)) print("total_assigned_cells %d" % (total_assigned_cells)) sub_name = "SnxnLv" + str(d) + "Inst" + str(i) tree.create_node(str(assigned_cells) + " " + sub_name, sub_name, parent=top_name) total_created_modules += 1 else: for i in range(avg_children**(d)): # 0 ~ 15 if left_cells < 0: break for j in range(avg_children**(d - 1)): # 0 ~ 3 if (math.floor(i / avg_children) == j): assigned_cells = 0 while True: assigned_cells = random.randint( 0, avg_module_size * 2) if assigned_cells > 4: break module_get_assigned += 1 total_assigned_cells += assigned_cells left_cells -= assigned_cells print("module%d: at level %d, %d cells" % (module_get_assigned, d, assigned_cells)) print("total_assigned_cells %d" % (total_assigned_cells)) sub_name = "SnxnLv" + str(d) + "Inst" + str(i) parent_name = "SnxnLv" + str(d - 1) + "Inst" + str(j) tree.create_node(str(assigned_cells) + " " + sub_name, sub_name, parent=parent_name) total_created_modules += 1 tree.show() print("module_num = ", total_created_modules) print("depth = ", tree.depth() + 1) print("total_assigned_cells = %d" % (total_assigned_cells)) #todo: extend to Pyrope and Verilog synthetic #step-1: get some cells from the encode node tag, a tag example "327 SnxnLv2Inst5" #step-2: create module name based on the tree node tag #step-3: mimic previous script to construct the body #step-4: check the chidren module name, create children instantiations #step-5: create some interaction for the children output and the parent output f = open("%s.scala" % (tree[tree.root].tag.split()[1]), "w+") f.write("// total modules: %s\n" % (total_created_modules)) f.write("// design tree depth: %s\n" % (tree.depth() + 1)) f.write("// design size: %s\n" % (total_assigned_cells)) f.write("// avg module size: %s\n" % (avg_module_size)) f.write("package %s\n" % (tree[tree.root].tag).split()[1]) f.write("import chisel3._\n") f.write("import scala.language.reflectiveCalls\n\n") for node in tree.expand_tree(mode=Tree.WIDTH): assigned_cells = 0 module_name = "" for word in tree[node].tag.split(): if word.isdigit(): assigned_cells = math.ceil(int(word) / 4) else: module_name = word f.write("\nclass %s extends Module {\n" % (module_name)) f.write(" val io = IO(new Bundle {\n") f.write(" val a = Input(UInt(1.W))\n") f.write(" val b = Input(UInt(1.W))\n") f.write(" val z = Output(UInt(1.W))\n") f.write(" })\n") # construct the main body module2last_inv = {} for i in range(assigned_cells): if i == 0: f.write(" val t0 = io.a + io.b\n") f.write(" val inv0 = ~t0\n") f.write(" val x0 = t0 ^ inv0\n") f.write(" val invx0 = ~x0\n") else: f.write(" val t%d = x%d + invx%d\n" % (i, i - 1, i - 1)) f.write(" val inv%d = ~t%d\n" % (i, i)) f.write(" val x%d = t%d ^ inv%d\n" % (i, i, i)) f.write(" val invx%d = ~x%d\n" % (i, i)) if (i == assigned_cells - 1): module2last_inv[module_name] = "invx%d" % (i) # print("module %s last inv is : %s" %(module_name, module2last_inv[module_name])) # create children instances for child in tree.children(node): # print(child.tag) submodule_name = "" for word in child.tag.split(): if word.isdigit() == False: submodule_name = word f.write("\n val inst_%s = Module(new %s())\n" % (submodule_name, submodule_name)) f.write(" inst_%s.io.a := io.a\n" % (submodule_name)) f.write(" inst_%s.io.b := io.b\n" % (submodule_name)) # make children instances outputs interact with parent's output if (tree[node].is_leaf() == True): f.write("\n") f.write(" io.z := %s\n" % (module2last_inv[module_name])) f.write("}\n") continue # continue next iterator of expand_tree i = 0 for child in tree.children(node): submodule_name = "" for word in child.tag.split(): if word.isdigit() == False: submodule_name = word if i == 0: f.write("\n val sum = inst_%s.io.z" % (submodule_name)) else: f.write(" + inst_%s.io.z" % (submodule_name)) i = i + 1 f.write("\n") f.write(" io.z := sum ^ io.a\n") f.write("}\n")
tree = Tree() tree.create_node("Harry", "harry") # root node tree.create_node("Jane", "jane", parent="harry") tree.create_node("Bill", "bill", parent="harry") tree.create_node("Diane", "diane", parent="jane") tree.create_node("George", "george", parent="diane") tree.create_node("Mary", "mary", parent="diane") tree.create_node("Jill", "jill", parent="george") tree.create_node("Mark", "mark", parent="jane") print("#"*4 + "Breakdown of out family") tree.show() print('\n') print("#"*4 + "All family members in DEPTH mode") for node in tree.expand_tree(mode=Tree.DEPTH): print tree[node].tag print('\n') print("#"*4 + "All family members without Diane sub-family") for node in tree.expand_tree(filter=lambda x: x != 'diane', mode=Tree.DEPTH): print tree[node].tag print('\n') print("#"*4 + "Let me introduce Diane family only") sub_t = tree.subtree('diane') sub_t.show() print('\n') print("#"*4 + "Children of Diane") print tree.is_branch('diane')
class AcquisitionChainIter(object): def __init__(self, acquisition_chain, parallel_prepare=True): self.__sequence_index = -1 self._parallel_prepare = parallel_prepare self.__acquisition_chain_ref = weakref.ref(acquisition_chain) # set all slaves into master for master in (x for x in acquisition_chain._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] master.slaves.extend( acquisition_chain._tree.get_node(master).fpointer) # create iterators tree self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") device2iter = dict() for dev in acquisition_chain._tree.expand_tree(): if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)): continue dev_node = acquisition_chain._tree.get_node(dev) parent = device2iter.get(dev_node.bpointer, "root") try: it = iter(dev) except TypeError: one_shot = self.acquisition_chain._device2one_shot_flag.get( dev, True) dev_iter = DeviceIterator(dev, one_shot) else: dev_iter = DeviceIteratorWrapper(it) device2iter[dev] = dev_iter self._tree.create_node( tag=dev.name, identifier=dev_iter, parent=parent) @property def acquisition_chain(self): return self.__acquisition_chain_ref() def prepare(self, scan, scan_info): preset_tasks = list() if self.__sequence_index == 0: preset_tasks.extend([gevent.spawn(preset.prepare) for preset in self.acquisition_chain._presets_list]) scan.prepare(scan_info, self.acquisition_chain._tree) self._execute( "_prepare", wait_between_levels=not self._parallel_prepare) if self.__sequence_index == 0: gevent.joinall(preset_tasks, raise_error=True) def start(self): if self.__sequence_index == 0: preset_tasks = [gevent.spawn( preset.start) for preset in self.acquisition_chain._presets_list] gevent.joinall(preset_tasks, raise_error=True) self._execute("_start") def stop(self): self._execute("stop", master_to_slave=True, wait_all_tasks=True) preset_tasks = [gevent.spawn(preset.stop) for preset in self.acquisition_chain._presets_list] gevent.joinall(preset_tasks) # wait to call all stop on preset gevent.joinall(preset_tasks, raise_error=True) def next(self): self.__sequence_index += 1 gevent.joinall([gevent.spawn(dev_iter.wait_ready) for dev_iter in self._tree.expand_tree() if dev_iter is not 'root'], raise_error=True) try: if self.__sequence_index: for dev_iter in self._tree.expand_tree(): if dev_iter is 'root': continue dev_iter.next() except StopIteration: # should we stop all devices? for acq_dev_iter in (x for x in self._tree.expand_tree() if x is not 'root' and isinstance(x.device, (AcquisitionDevice, AcquisitionMaster))): if hasattr(acq_dev_iter, 'wait_reading'): acq_dev_iter.wait_reading() dispatcher.send("end", acq_dev_iter.device) raise return self def _execute(self, func_name, master_to_slave=False, wait_between_levels=True, wait_all_tasks=False): tasks = list() prev_level = None if master_to_slave: devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:] else: devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]) for dev in devs: node = self._tree.get_node(dev) level = self._tree.depth(node) if wait_between_levels and prev_level != level: gevent.joinall(tasks, raise_error=True) tasks = list() prev_level = level func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) # ensure that all tasks are executed # (i.e: don't raise the first exception on stop) if wait_all_tasks: gevent.joinall(tasks) gevent.joinall(tasks, raise_error=True)
tree.create_node("Harry", "harry") # root node tree.create_node("Jane", "jane", parent="harry") tree.create_node("Bill", "bill", parent="harry") tree.create_node("Diane", "diane", parent="jane") tree.create_node("George", "george", parent="diane") tree.create_node("Mary", "mary", parent="diane") tree.create_node("Jill", "jill", parent="george") tree.create_node("Mark", "mark", parent="jane") sep = "-" * 20 + '\n' print(sep + "Tree of the whole family:") tree.show(key=lambda x: x.tag, reverse=True) print(sep + "All family members in DEPTH mode:") for node in tree.expand_tree(mode=Tree.ZIGZAG): print(tree[node].tag) print(sep + "All family members without Diane sub-family:") tree.show(idhidden=False, filter=lambda x: x.identifier != 'diane') # for node in tree.expand_tree(filter=lambda x: x.identifier != 'diane', mode=Tree.DEPTH): # print tree[node].tag print(sep + "Let me introduce Diane family only:") sub_t = tree.subtree('diane') sub_t.show() print(sep + "Children of Diane") for child in tree.is_branch('diane'): print(tree[child].tag)
class DependencyReader: """DependencyReader object""" def __init__(self): self.tempDirectoryPath = mkdtemp(dir=".") self.tree = Tree() self.dependencies = {} self.graphRelationships = [] def getPom(self, pomPath): shutil.copy(pomPath, self.tempDirectoryPath) os.chdir(self.tempDirectoryPath) def getDependencies(self): mavenTreeOutput = subprocess.Popen('mvn org.apache.maven.plugins:maven-dependency-plugin:RELEASE:tree -DoutputType=tgf', stdout=subprocess.PIPE, shell=True) while True: line = mavenTreeOutput.stdout.readline().rstrip() if not line or re.search(r"BUILD SUCCESS", line): break match = re.match(r"\[INFO\]\s(\d*)\s*(.*):(.*):(\w+):([0-9\.]*)", line) if match: if not match.group(1) in self.dependencies.keys(): self.dependencies[match.group(1)] = DependencyNode(match.group(2), match.group(3), match.group(5), match.group(1)) if not self.tree.leaves(): self.tree.create_node(match.group(1), match.group(1), data=self.dependencies[match.group(1)]) self.dependencies[match.group(1)].get('jar', self.tempDirectoryPath) match = re.match(r"\[INFO\]\s(\d*)\s(\d*)", line) if match and match.group(2): self.graphRelationships.append((match.group(1), match.group(2))) def relateDependencies(self): while self.graphRelationships: for item in self.graphRelationships: node = self.tree.get_node(item[0]) if node is not None: parent = self.dependencies[item[0]] child = self.dependencies[item[1]] self.tree.create_node(child.referenceId, child.referenceId, parent=parent.referenceId, data=child) self.graphRelationships.remove(item) def scanDependencies(self): # Need to run on each package with oneshot to get identifiers # unless update dosocsv2 to create identifiers on scan # or fix up dosocsv2 to create identifiers on scan instead for node in self.tree.expand_tree(mode=Tree.DEPTH): treeNode = self.tree.get_node(node) subprocess.call('dosocs2 oneshot ' + treeNode.data.jarName, shell=True) def createRelationships(self): # Pass packages as relationships to new dosocsv2 command created self.recursiveRelationship(self.tree.root) def recursiveRelationship(self, parent): for node in self.tree.is_branch(parent): parentNode = self.tree.get_node(parent) childNode = self.tree.get_node(node) subprocess.call('dosocs2 packagerelate ' + parentNode.data.jarName + ' ' + childNode.data.jarName, shell=True) self.recursiveRelationship(node) def retrieve_dependencies(self, jarName): if jarName is None: root = self.tree.get_node(self.tree.root) root = root.data.jarName else: root = jarName tgfOutput = subprocess.Popen('dosocs2 dependencies ' + root, stdout=subprocess.PIPE, shell=True) count = 0 tree = Tree() dependencies = [] relationships = [] while True: line = tgfOutput.stdout.readline() if not line: break match = re.match(r"(\d+) - (.*)", line) if match: if count == 0: count = count + 1 tree.create_node(match.group(2), match.group(1)) else: dependencies.append((match.group(2), match.group(1))) match = re.match(r"(\d+) (\d+)", line) if match: relationships.append((match.group(1), match.group(2))) if not relationships: print("No child relationships for " + jarName) return None while relationships: for item in relationships: node = tree.get_node(item[0]) if node is not None: rel = [item for item in relationships if int(item[0]) == int(node.identifier)] if rel is not None: rel = rel[0] dep = [item for item in dependencies if int(item[1]) == int(rel[1])] if dep is not None: dep = dep[0] tree.create_node(dep[0], dep[1], parent=node.identifier) relationships.remove(rel) dependencies.remove(dep) tree.show() if jarName is None: os.chdir(os.pardir)
tree.create_node("Child3", "Child3", data=TreeNode(key_list, node_type, [3, 2]), parent="Child1") tree.create_node("Child4", "Child4", data=TreeNode(key_list, node_type, [3, 3]), parent="Child1") tree.create_node("Child5", "Child5", data=TreeNode(key_list, node_type, [4, 4]), parent="Child4") print(tree) l = tree.expand_tree(mode=Tree.WIDTH, filter=lambda x: x.data.get_change() == 0) print(next(l)) print(next(l)) print(next(l)) print(next(l)) # # for node in tree.all_nodes(): # print(node.tag) # tree_node = node.data # print(tree_node.node_type) # tree_node.values = ['1', '2'] # print("#########################################") # for node in tree.all_nodes(): # print(node.tag) # print(node.data.values)
tree.create_node("Harry", "harry") # root node tree.create_node("Jane", "jane", parent="harry") tree.create_node("Bill", "bill", parent="harry") tree.create_node("Diane", "diane", parent="jane") tree.create_node("George", "george", parent="diane") tree.create_node("Mary", "mary", parent="diane") tree.create_node("Jill", "jill", parent="george") tree.create_node("Mark", "mark", parent="jane") sep = "-" * 20 + "\n" print(sep + "Tree of the whole family:") tree.show(key=lambda x: x.tag, reverse=True) print(sep + "All family members in DEPTH mode:") for node in tree.expand_tree(mode=Tree.DEPTH): print(tree[node].tag) print(sep + "All family members without Diane sub-family:") tree.show(idhidden=False, filter=lambda x: x.identifier != "diane") # for node in tree.expand_tree(filter=lambda x: x.identifier != 'diane', mode=Tree.DEPTH): # print tree[node].tag print(sep + "Let me introduce Diane family only:") sub_t = tree.subtree("diane") sub_t.show() print(sep + "Children of Diane") for child in tree.is_branch("diane"): print(tree[child].tag)
tree.create_node("Diane", "diane", parent="jane") tree.create_node("George", "george", parent="diane") tree.create_node("Mary", "mary", parent="diane") tree.create_node("Jill", "jill", parent="george") tree.create_node("Mark", "mark", parent="jane") print("#"*4 + "Breakdown of out family") tree.show(cmp=lambda x,y: cmp(x.tag, y.tag), key=None, reverse=True) #tree.show(key=lambda x: x.tag, reverse=False) tree.save2file("/home/chenxm/Desktop/tree.txt", idhidden=False) print('\n') print("#"*4 + "All family members in DEPTH mode") for node in tree.expand_tree(mode=Tree.DEPTH): print tree[node].tag print('\n') print("#"*4 + "All family members without Diane sub-family") tree.show(idhidden=False, filter=lambda x: x.identifier != 'diane') # for node in tree.expand_tree(filter=lambda x: x.identifier != 'diane', mode=Tree.DEPTH): # print tree[node].tag print('\n') print("#"*4 + "Let me introduce Diane family only") sub_t = tree.subtree('diane') sub_t.show() print('\n')
class AcquisitionChainIter(object): def __init__(self, acquisition_chain, parallel_prepare=True): self.__sequence_index = -1 self._parallel_prepare = parallel_prepare self.__acquisition_chain_ref = weakref.ref(acquisition_chain) # set all slaves into master for master in (x for x in acquisition_chain._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] master.slaves.extend( acquisition_chain._tree.get_node(master).fpointer) # create iterators tree self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") device2iter = dict() for dev in acquisition_chain._tree.expand_tree(): if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)): continue dev_node = acquisition_chain._tree.get_node(dev) parent = device2iter.get(dev_node.bpointer, "root") try: it = iter(dev) except TypeError: one_shot = self.acquisition_chain._device2one_shot_flag.get( dev, True) dev_iter = DeviceIterator(dev, one_shot) else: dev_iter = DeviceIteratorWrapper(it) device2iter[dev] = dev_iter self._tree.create_node(tag=dev.name, identifier=dev_iter, parent=parent) @property def acquisition_chain(self): return self.__acquisition_chain_ref() def prepare(self, scan, scan_info): preset_tasks = list() if self.__sequence_index == 0: preset_tasks.extend([ gevent.spawn(preset.prepare) for preset in self.acquisition_chain._presets_list ]) scan.prepare(scan_info, self.acquisition_chain._tree) self._execute("_prepare", wait_between_levels=not self._parallel_prepare) if self.__sequence_index == 0: gevent.joinall(preset_tasks, raise_error=True) def start(self): if self.__sequence_index == 0: preset_tasks = [ gevent.spawn(preset.start) for preset in self.acquisition_chain._presets_list ] gevent.joinall(preset_tasks, raise_error=True) self._execute("_start") def stop(self): self._execute("stop", master_to_slave=True, wait_all_tasks=True) preset_tasks = [ gevent.spawn(preset.stop) for preset in self.acquisition_chain._presets_list ] gevent.joinall(preset_tasks) # wait to call all stop on preset gevent.joinall(preset_tasks, raise_error=True) def next(self): self.__sequence_index += 1 gevent.joinall([ gevent.spawn(dev_iter.wait_ready) for dev_iter in self._tree.expand_tree() if dev_iter is not 'root' ], raise_error=True) try: if self.__sequence_index: for dev_iter in self._tree.expand_tree(): if dev_iter is 'root': continue dev_iter.next() except StopIteration: # should we stop all devices? for acq_dev_iter in ( x for x in self._tree.expand_tree() if x is not 'root' and isinstance(x.device, ( AcquisitionDevice, AcquisitionMaster))): if hasattr(acq_dev_iter, 'wait_reading'): acq_dev_iter.wait_reading() dispatcher.send("end", acq_dev_iter.device) raise return self def _execute(self, func_name, master_to_slave=False, wait_between_levels=True, wait_all_tasks=False): tasks = list() prev_level = None if master_to_slave: devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:] else: devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]) for dev in devs: node = self._tree.get_node(dev) level = self._tree.depth(node) if wait_between_levels and prev_level != level: gevent.joinall(tasks, raise_error=True) tasks = list() prev_level = level func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) # ensure that all tasks are executed # (i.e: don't raise the first exception on stop) if wait_all_tasks: gevent.joinall(tasks) gevent.joinall(tasks, raise_error=True)
class Route(object): def __init__(self, universe): self.route = Tree() self.universe = universe self.max_hops = 4 def show(self): self.route.show() def asString(self): return (','.join([self.route[node].tag for node in self.route.expand_tree(mode=Tree.DEPTH)])) def getRoute(self): return self.route def byScore_key(self, s): return s.score def findRoute(self, start): parent = self.universe.findSystem(start) self.route.create_node(start, start, data=parent) systems = self.findNextSystems(start, start) self.buildRoute(systems, start) return self.route def buildRoute(self, systems, parent): for s in systems: n = s.name h = 0 if (self.route.contains(n) == False): self.route.create_node(n, n, parent=parent, data=s) hop = h + self.route.depth(n) if (hop < self.max_hops): sub_systems = self.findNextSystems(parent, n) self.buildRoute(sub_systems, n) else: n = parent + ' --> ' + n self.route.create_node(n, n, parent=parent, data=s) def getSystemId(self, name, i=0): if (self.route.contains(name) == False): return name else: i += 1 n = name + '(' + str(i) + ')' return self.getSystemId(n) def findNextSystems(self, parent, start): systems = [] optimal = self.universe.distances.findOptimalSystems(start) for s in sorted(set(optimal)): if (s != parent): i = self.universe.findSystem(s) if (i.permit == False): systems.append(i) s = sorted(systems, key = self.byScore_key) return s[:self.max_hops] # http://xiaming.me/treelib/examples.html # # class SystemTree(object): # def __init__(self): # self.tree = Tree() # # def addNode(self, id, o): # self.tree.create_node(o, id) # # def addChildNode(self, p, id, o): # self.tree.create_node(o, id, parent=p) # # def getNode(self, id): # return self.tree.subtree(id) # # def __repr__(self): # return self.tree.to_json(with_data=True) # # # t = SystemTree() # t.addNode('Aerial', 'Aerial') # t.addChildNode('Aerial', 'Jotun', 'Jotun') # t.addChildNode('Jotun', 'Rusani', 'Rusani') # n = t.getNode('Jotun') # print(n) # n = t.tree.contains('Invalid') # print(n) # t.tree.show()