def readmap(orbitmap): """Reads an orbit map""" COM = None ERRANDS = Node("--ERRAND--") for orbit in orbitmap: center, orbiter = orbit.strip().split(")") if center == "COM": COM = Node(center) centerNode = None if COM: centerNode = find_by_attr(COM, center) if centerNode is None: centerNode = find_by_attr(ERRANDS, center) if centerNode is None: centerNode = Node(center, parent=ERRANDS) orbiterNode = find_by_attr(ERRANDS, orbiter) if orbiterNode is not None: orbiterNode.parent = centerNode else: orbiterNode = Node(orbiter, parent=centerNode) return COM
def test_binding_at_template_assignment(): template0 = XMLTemplateParser(""" <template name="template0" size="4"> </template> """).parse() template1 = XMLTemplateParser(""" <template name="template0"> <layout name="layout0"> <area name="area0"> <field name="field1_size" size="4"></field> <field name="field1" size="{field1_size, byteorder=little}"></field> <field name="field2" size="{field1_size, byteorder=big}"></field> </area> </layout> </template>""").parse() template_provider0 = TemplateProvider(template0) template_provider1 = TemplateProvider(template1) data_provider = DataProvider(io.BytesIO(bytes(4 * [0x0]))) binalyzer = Binalyzer(template_provider0, data_provider) binalyzer.template = template_provider1.template binalyzer.data = io.BytesIO(bytes([0x04, 0x00, 0x00, 0x00])) field1_size = find_by_attr(template1, "field1_size") field1 = find_by_attr(template1, "field1") field2 = find_by_attr(template1, "field2") assert field1_size.size == 4 assert field1_size.value == bytes([0x04, 0x00, 0x00, 0x00]) assert field1.size == 0x4 assert field2.size == 0x4000000
def get_path_capacity(source, target): """ given an origin and a destination, get bandwidth capacity """ node_tree = get_node_tree() source_node = find_by_attr(node_tree, name='id', value=source) target_node = find_by_attr(node_tree, name='id', value=target) return min(source_node.capacity, target_node.capacity)
def generate_tree(filename): """ It generates a tree level structure of a MS-COCO things and stuff label with leaf node act as an actual label while the upper nodes are supercategories :param filename: A panoptic categories filename generated from `create_category_json()` :return: root node of things and stuff label """ categories = json.load(open(filename)) things_tree = categories['things']['tree'] stuff_tree = categories['stuff']['tree'] # CHILD:PARENT structure child_t, _ = things_tree[1].strip().split(':') root_things = Node(child_t) child_s, _ = stuff_tree[1].strip().split(':') root_stuff = Node(child_s) for line in things_tree[2:]: child, parent = line.strip().split(':') Node(child, parent=find_by_attr(root_things, parent)) for line in stuff_tree[2:]: child, parent = line.strip().split(':') Node(child, parent=find_by_attr(root_stuff, parent)) return root_things, root_stuff
def akm_distance(name_1: str, name_2: str, tree: Node): message = 'Unknown name "%s". The name must be associated to a node in the tree.' start = find_by_attr(tree, name_1) end = find_by_attr(tree, name_2) if start is None: raise ValueError(message % name_1) if end is None: raise ValueError(message % name_2) return sum([calculate_weight(edge) for edge in calculate_path(start, end)])
def findNumericalConcept(subject_concept, value_set): remove = "$,\"[]()? -!abcdefghijklmnopqstuvw" normalized_value_set = [] for txt in value_set[1:]: for r in remove: txt = txt.replace(r, '') if len(txt) > 0: normalized_value_set.append(float(txt)) # findign the domains correspond to concepts candidates_to_domain = findDomains( findNearestNeighbours(normalized_value_set, 20)) predicted_concept = None b = False for candidate in candidates_to_domain.keys(): if candidates_to_domain[candidate] != None and candidates_to_domain[ candidate][28:] == subject_concept: b = True predicted_concept = candidate[28:] break # loading the concept tree root = load_tree() if not b: min_dis = float('inf') min_dist_concept = None for candidate in candidates_to_domain.keys(): x = [ ancestor.name for ancestor in find_by_attr(root, subject_concept).ancestors ] x.append(subject_concept) x = set(x) y = None if candidates_to_domain[candidate] != None: try: y = [ ancestor.name for ancestor in find_by_attr( root, candidates_to_domain[candidate] [28:]).ancestors ] y.append(candidates_to_domain[candidate][28:]) y = set(y) curr_dis = len(list(x.union(y) - x.intersection(y))) if curr_dis < min_dis: min_dis = curr_dis min_dist_concept = candidate except: continue predicted_concept = min_dist_concept[28:] return "http://dbpedia.org/ontology/" + predicted_concept
def test_find_by_attr(): f = Node("f") b = Node("b", parent=f) Node("a", parent=b) d = Node("d", parent=b) c = Node("c", parent=d, foo=4) Node("e", parent=d) g = Node("g", parent=f) i = Node("i", parent=g) Node("h", parent=i) eq_(find_by_attr(f, "d"), d) eq_(find_by_attr(f, name="foo", value=4), c) eq_(find_by_attr(f, name="foo", value=8), None)
def bind_data_to_template(root_template, bindings): for binding in bindings: data_url = binding.get('data_url') template_name = binding.get('template_name') template = find_by_attr(root_template, template_name) if template: data = requests.get(data_url).content template.value = data
def beam_search(inputs_to_ids_fn, starting_state, starting_id, stop_condition_fn, beam_width=3, \ num_states_returned=3, top_k=10, top_p=1.0, num_dist_samples=-1, temperature=1.0): """ Searches a space using a beam constrrained tree to find the most likely outcomes. inputs_to_ids_fn: (state, id)->(new_state, [float]): the prediction function returning an array of probabilities corresponding to the next valid ids starting_cache: an object containing the state of the prediction fn before being passed a new id starting_id: int: the integer used to prime the first distribution stop_condition_fn: (state)->bool: a function determining if the sequence should be stopped, this could include maximum length reached. beam_width: int: number of active beams to search at any particular point. returns: [state]: most probable terminated states """ terminated_states = [] active_states = [(0.0, starting_state, starting_id)] # we use log probabilities, and thus sum future log probs. log(p==1) = 0 beam_tree = Node(f"Start", state=starting_state["input_ids"].cpu().tolist().copy()) while len(terminated_states) < num_states_returned and len(active_states) != 0: p, best_state, best_next_id = heappop(active_states) new_state, next_id_probs = inputs_to_ids_fn(best_state, best_next_id) print("searching for", best_state["input_ids"].cpu().tolist().copy()) parent = find_by_attr(beam_tree, name="state", value=best_state["input_ids"].cpu().tolist()) print("found parent ",parent) print("adding node with state ",new_state["input_ids"].cpu().tolist().copy()) added_node = Node(f"{best_next_id}", state=new_state["input_ids"].cpu().tolist().copy(), parent=parent) if stop_condition_fn(new_state): terminated_states.append((-p, new_state.copy())) continue # choose best IDs to add to tree next_id_probs = np.array(next_id_probs) scaled_logits = np.log(next_id_probs) / temperature exp_logits = np.exp(scaled_logits) next_id_probs = exp_logits / np.sum(exp_logits) # softmax sorted_ids = np.argsort(next_id_probs)[::-1] cumulative_probs = np.cumsum(next_id_probs[sorted_ids]) top_p_ids = [idx for idx, cumulative_p in zip(sorted_ids,cumulative_probs) if cumulative_p<=top_p] top_k_ids = top_p_ids[:top_k] if num_dist_samples<1: sampled_ids = top_k_ids[:beam_width] else: sampled_ids = np.random.choice(top_k_ids, min(num_dist_samples, len(top_k_ids)), p=normalize(next_id_probs[top_k_ids]), replace=False) for idx in sampled_ids: new_prob = p-np.log(next_id_probs[idx]) heappush(active_states, (new_prob, new_state.copy(), idx)) active_states = nsmallest(beam_width, active_states) for pre, fill, node in RenderTree(beam_tree): print("%s%s" % (pre, node.name)) return terminated_states
def load_tree(): root = None f = open('input.txt', 'r') lines = f.readlines()[1:] root = Node(lines[0].split(" ")[0]) for line in lines: line = line.split(" ") Node("".join(line[1:]).strip(), parent=find_by_attr(root, line[0])) return root
def get_common_path_alt(path_a, path_b): """ given two paths, return common path source and target """ node_tree = get_node_tree() source_a = find_by_attr(node_tree, name='id', value=path_a[0]) target_a = find_by_attr(node_tree, name='id', value=path_a[1]) source_b = find_by_attr(node_tree, name='id', value=path_b[0]) target_b = find_by_attr(node_tree, name='id', value=path_b[1]) source = source_a if source_a == source_b else None target = None for node_a, node_b in zip(target_a.path, target_b.path): if node_a == node_b: target = node_a else: break return source.id, target.id
def add_children(lines,node,tree): for x in lines: parent = x.split(')')[0] child = x.split(')')[1] if parent == node: Node(child, parent=find_by_attr(tree, node)) add_children(lines,child,tree) return tree
def add_entered_folder(self, parent, folder_name): root_folder = self.get_current_storage().root_folder parent_folder = anytree.find_by_attr(root_folder, name="id", value=parent.label_id) current_folder = ThemeFolders(folder_name, parent=parent_folder) self.populate_tree_view(self.get_current_storage().tree_view, parent, current_folder)
def is_descendent(parent, child): """ given two nodes, check if one is ancestor of the other """ root = get_node_tree() parent_node = find_by_attr(root, name='id', value=parent) decendency = False for node in parent_node.descendants: decendency = node.id == child or decendency return decendency
def is_ancestor(parent, child): """ given two nodes, check if one is ancestor of the other """ root = get_node_tree() child_node = find_by_attr(root, name='id', value=child) ancestry = False for node in child_node.ancestors: ancestry = node.id == parent or ancestry return ancestry
def get_common_path_list(path_a, path_b): """ given two paths, return common path source and target """ node_tree = get_node_tree() source_a = find_by_attr(node_tree, name='id', value=path_a[0]) target_a = find_by_attr(node_tree, name='id', value=path_a[1]) source_b = find_by_attr(node_tree, name='id', value=path_b[0]) target_b = find_by_attr(node_tree, name='id', value=path_b[1]) source = source_a if source_a == source_b else None common_path = [source] for node_a, node_b in zip(target_a.path, target_b.path): if node_a == node_b: common_path.append(node_a) else: break return common_path
def tree(self, selected_refs, synonyms, hyponyms): #Adding lists to class: self.selected_refs = selected_refs self.synonyms = synonyms self.hyponyms = hyponyms participants = Node('participants') ##hyponyms for hyp in self.hyponyms: if not findall_by_attr(participants, hyp): hypo = Node(hyp, parent=participants) for hyper in self.hyponyms[hyp]: hyper = Node(hyper, parent=hypo) else: for hyper in self.hyponyms[hyp]: hypo = find_by_attr(participants, hyp) hyper = Node(hyper, parent=hypo) ##synonyms for syn in self.synonyms: overlap = findall_by_attr(participants, syn) if overlap: #The synonym already exists in the tree for n in overlap: syno = find_by_attr(n, syn) for s in self.synonyms[syn]: s0 = Node(s, parent=syno) #Creating node s1 = Node(syno.name, parent=s0) #Creating additional node for doubling and reversing the edge. else: #If the synonym does not exist in the tree syno = Node(syn, parent=participants) #Creating node for s in self.synonyms[syn]: s0 = Node(s, parent=syno) #Creating synonym s1 = Node(syno.name, parent=s0) #Reverse node ##remaining participants for n in self.selected_refs: if not findall_by_attr(participants, n): node = Node(n, parent=participants) self.tree = participants return participants
def test_field_cross_reference(): data = io.BytesIO(bytes([0x04, 0x00, 0x00, 0x00])) template = XMLTemplateParser(""" <template name="template0"> <layout name="layout0"> <area name="area0"> <field name="field1-size" size="4"></field> <field name="field1" size="{field1-size, byteorder=little}"></field> <field name="field2" size="{field1-size, byteorder=big}"></field> </area> </layout> </template>""").parse() binalyzer = Binalyzer(template, data) field1_size = find_by_attr(template, "field1-size") field1 = find_by_attr(template, "field1") field2 = find_by_attr(template, "field2") assert field1_size.size == 4 assert field1_size.value == bytes([0x04, 0x00, 0x00, 0x00]) assert field1.size == 0x4 assert field2.size == 0x4000000
def test_binding_at_stream_assignment(): template = XMLTemplateParser(""" <template name="template0"> <layout name="layout0"> <area name="area0"> <field name="field1_size" size="4"></field> <field name="field1" size="{field1_size, byteorder=little}"></field> <field name="field2" size="{field1_size, byteorder=big}"></field> </area> </layout> </template>""").parse() binalyzer = Binalyzer(template, io.BytesIO(bytes(4 * [0x0]))) binalyzer.data = io.BytesIO(bytes([0x04, 0x00, 0x00, 0x00])) field1_size = find_by_attr(template, "field1_size") field1 = find_by_attr(template, "field1") field2 = find_by_attr(template, "field2") assert field1_size.size == 4 assert field1_size.value == bytes([0x04, 0x00, 0x00, 0x00]) assert field1.size == 0x4 assert field2.size == 0x4000000
def test_template_reference_with_provider(binalyzer): data = io.BytesIO(bytes([0xE5, 0x8E, 0x26])) template = XMLTemplateParser( """ <template name="template0"> <layout name="layout0"> <area name="area0"> <field name="field1_size" size="3"></field> <field name="field1" size="{field1_size, provider=mock.ref_custom}"></field> </area> </layout> </template>""", binalyzer=binalyzer, ).parse() binalyzer = Binalyzer(template, data) field1_size = find_by_attr(template, "field1_size") field1 = find_by_attr(template, "field1") assert field1_size.size == 3 assert field1_size.value == bytes([0xE5, 0x8E, 0x26]) assert field1.size == 2526949
def find_deepest_concept(L): maxDepth = -float('inf') deepestConceptInClassTree = "" root = load_tree() for c, v in L.items(): depth = find_by_attr(root, "Guitarist").depth L[c] = depth * v if L[c] > maxDepth: maxDepth = L[c] deepestConceptInClassTree = c return 'http://dbpedia.org/ontology/' + deepestConceptInClassTree
async def analytics_callback_tree_api(request, user): # look at the current callbacks and return their data in a more manageable tree format # http://anytree.readthedocs.io/en/latest/ query = await db_model.operation_query() operation = await db_objects.get(query, name=user['current_operation']) query = await db_model.callback_query() dbcallbacks = await db_objects.execute(query.where(Callback.operation == operation)) callbacks = [] # Default values here display_config = {} display_config['inactive'] = True # by default, include all callbacks display_config['strikethrough'] = False # The POST version of this API function will provide modifiers for what specific information to provide in the callback tree, but the main logic remains the same if request.method == 'POST': data = request.json if 'inactive' in data: display_config['inactive'] = data['inactive'] if 'strikethrough' in data: display_config['strikethrough'] = data['strikethrough'] for dbc in dbcallbacks: if display_config['inactive']: # include everything callbacks.append(dbc.to_json()) elif dbc.active: callbacks.append(dbc.to_json()) elif not dbc.active: json_val = dbc.to_json() json_val['description'] = "CALLBACK DEAD " + json_val['description'] callbacks.append(json_val) # every callback with a pcallback of null should be at the root (remove them from list as we place them) tree = [] while len(callbacks) != 0: # when we hit 0 we are done processing for c in callbacks: # this is the root of a 'tree' if c['pcallback'] == 'null': display = await analytics_callback_tree_api_function(c, display_config) tree.append(Node(str(c['id']), display=display)) callbacks.remove(c) # remove the one we just processed from our list else: for t in tree: # for each tree in our list, see if we can find the parent leaf = find_by_attr(t, str(c['pcallback'])) if leaf: display = await analytics_callback_tree_api_function(c, display_config) Node(str(c['id']), parent=leaf, display=display) callbacks.remove(c) break output = "" for t in tree: output += str(RenderTree(t, style=DoubleStyle).by_attr("display")) + "\n" return text(output)
def load_tree(): root = None f = open('dbp_tree_input.txt', 'r') lines = f.readlines()[1:] root = Node(lines[0].split(" ")[0]) for line in lines: line = line.split(" ") Node("".join(line[1:]).strip(), parent=find_by_attr(root, line[0])) ### print the tree # for pre, _, node in RenderTree(root): # print("%s%s" % (pre, node.name)) return root
def get_parent_gate(self, gate_id): """ Retrieve the gate ID for a parent gate of the given gate ID. :param gate_id: text string of a gate ID :return: text string of the parent gate ID, None if given gate reference has no parent gate. """ n = anytree.find_by_attr(self._gate_tree, gate_id) if n is None: raise ValueError("Gate ID %s was not found in gating strategy" % gate_id) return n.parent.gate
def make_data(self): train_path = os.path.join(self.raw_folder, 'ILSVRC2012_img_train') test_path = os.path.join(self.raw_folder, 'ILSVRC2012_img_val') meta_path = os.path.join(self.raw_folder, 'ILSVRC2012_devkit_t12') extract_file(os.path.join(self.raw_folder, 'ILSVRC2012_img_train.tar'), train_path) extract_file(os.path.join(self.raw_folder, 'ILSVRC2012_img_val.tar'), test_path) extract_file( os.path.join(self.raw_folder, 'ILSVRC2012_devkit_t12.tar.gz'), meta_path) for archive in [ os.path.join(train_path, archive) for archive in os.listdir(train_path) ]: extract_file(archive, os.path.splitext(archive)[0], delete=True) classes_to_labels, classes_size = make_meta(meta_path) with open( os.path.join(meta_path, 'data', 'ILSVRC2012_validation_ground_truth.txt'), 'r') as f: test_id = f.readlines() test_id = [int(i) for i in test_id] test_img = sorted( [os.path.join(test_path, file) for file in os.listdir(test_path)]) test_wnid = [] for test_id_i in test_id: test_node_i = anytree.find_by_attr(classes_to_labels['label'], name='id', value=test_id_i) test_wnid.append(test_node_i.name) for test_wnid_i in set(test_wnid): os.mkdir(os.path.join(test_path, test_wnid_i)) for test_wnid_i, test_img in zip(test_wnid, test_img): shutil.move( test_img, os.path.join(test_path, test_wnid_i, os.path.basename(test_img))) train_img, train_label = make_img( os.path.join(self.raw_folder, 'ILSVRC2012_img_train'), IMG_EXTENSIONS, classes_to_labels['label']) test_img, test_label = make_img( os.path.join(self.raw_folder, 'ILSVRC2012_img_val'), IMG_EXTENSIONS, classes_to_labels['label']) train_target = {'label': train_label} test_target = {'label': test_label} return (train_img, train_target), (test_img, test_target), (classes_to_labels, classes_size)
def make_tree(root, name, attribute=None): if len(name) == 0: return if attribute is None: attribute = {} this_name = name[0] next_name = name[1:] this_attribute = {k: attribute[k][0] for k in attribute} next_attribute = {k: attribute[k][1:] for k in attribute} this_node = anytree.find_by_attr(root, this_name) this_index = root.index + [len(root.children)] if this_node is None: this_node = anytree.Node(this_name, parent=root, index=this_index, **this_attribute) make_tree(this_node, next_name, next_attribute) return
def is_descendent(parent, child): """ given two nodes, check if one is ancestor of the other """ root = get_node_tree() parent_node = find_by_attr(root, name='id', value=get_ip_by_handle(parent)) decendency = False for node in parent_node.descendants: if "." in child: decendency = node.id == child or decendency else: decendency = next( (True for x in node.properties['handles'] if x == child), False) or decendency return decendency
def make_img(path, extensions, classes_to_labels): img, label = [], [] classes = [] leaf_nodes = classes_to_labels.leaves for node in leaf_nodes: classes.append(node.name) for c in sorted(classes): d = os.path.join(path, c) if not os.path.isdir(d): continue for root, _, filenames in sorted(os.walk(d)): for filename in sorted(filenames): if has_file_allowed_extension(filename, extensions): cur_path = os.path.join(root, filename) img.append(cur_path) label.append(anytree.find_by_attr(classes_to_labels, c).flat_index) return img, label
def printTree(edges_list): # create root root = Node("Root") for (node1, node2) in edges_list: Node(node2, parent=find_by_attr(root, node1)) for pre, _, node in RenderTree(root): print("%s%s" % (pre, node.name)) # Print to file --- GRAPVIZ MUST BE INSTALLED!!! installed_packages = [ "%s" % i.key for i in pip.get_installed_distributions() ] if 'graphviz' in installed_packages: RenderTreeGraph(root).to_picture("../Outputs/Part3/tree.png") print('Tree generated correctly')
def organizeParentPanels(panels, m): print('todo: organizing parent panels') root = Node('root') print('created root node') for key in panels: n = Node(key) print(n) parent = panels[key]['parent'] print(f'node:{key}, parent:={parent}') print(f'{type(parent)}, {parent}') if not parent: print('not parent check') n.parent = root else: pn = find_by_attr(root, parent) n.parent = pn m['nodes'] = root pass