def readmap(orbitmap):
"""Reads an orbit map"""
COM = None
ERRANDS = Node("--ERRAND--")
for orbit in orbitmap:
center, orbiter = orbit.strip().split(")")
if center == "COM":
COM = Node(center)
centerNode = None
if COM:
centerNode = find_by_attr(COM, center)
if centerNode is None:
centerNode = find_by_attr(ERRANDS, center)
if centerNode is None:
centerNode = Node(center, parent=ERRANDS)
orbiterNode = find_by_attr(ERRANDS, orbiter)
if orbiterNode is not None:
orbiterNode.parent = centerNode
else:
orbiterNode = Node(orbiter, parent=centerNode)
return COM
def test_binding_at_template_assignment():
template0 = XMLTemplateParser("""
<template name="template0" size="4">
</template>
""").parse()
template1 = XMLTemplateParser("""
<template name="template0">
<layout name="layout0">
<area name="area0">
<field name="field1_size" size="4"></field>
<field name="field1" size="{field1_size, byteorder=little}"></field>
<field name="field2" size="{field1_size, byteorder=big}"></field>
</area>
</layout>
</template>""").parse()
template_provider0 = TemplateProvider(template0)
template_provider1 = TemplateProvider(template1)
data_provider = DataProvider(io.BytesIO(bytes(4 * [0x0])))
binalyzer = Binalyzer(template_provider0, data_provider)
binalyzer.template = template_provider1.template
binalyzer.data = io.BytesIO(bytes([0x04, 0x00, 0x00, 0x00]))
field1_size = find_by_attr(template1, "field1_size")
field1 = find_by_attr(template1, "field1")
field2 = find_by_attr(template1, "field2")
assert field1_size.size == 4
assert field1_size.value == bytes([0x04, 0x00, 0x00, 0x00])
assert field1.size == 0x4
assert field2.size == 0x4000000
def get_path_capacity(source, target):
""" given an origin and a destination, get bandwidth capacity """
node_tree = get_node_tree()
source_node = find_by_attr(node_tree, name='id', value=source)
target_node = find_by_attr(node_tree, name='id', value=target)
return min(source_node.capacity, target_node.capacity)
def generate_tree(filename):
"""
It generates a tree level structure of a MS-COCO things
and stuff label with leaf node act as an actual label
while the upper nodes are supercategories
:param filename: A panoptic categories filename generated
from `create_category_json()`
:return: root node of things and stuff label
"""
categories = json.load(open(filename))
things_tree = categories['things']['tree']
stuff_tree = categories['stuff']['tree']
# CHILD:PARENT structure
child_t, _ = things_tree[1].strip().split(':')
root_things = Node(child_t)
child_s, _ = stuff_tree[1].strip().split(':')
root_stuff = Node(child_s)
for line in things_tree[2:]:
child, parent = line.strip().split(':')
Node(child, parent=find_by_attr(root_things, parent))
for line in stuff_tree[2:]:
child, parent = line.strip().split(':')
Node(child, parent=find_by_attr(root_stuff, parent))
return root_things, root_stuff
def akm_distance(name_1: str, name_2: str, tree: Node):
message = 'Unknown name "%s". The name must be associated to a node in the tree.'
start = find_by_attr(tree, name_1)
end = find_by_attr(tree, name_2)
if start is None:
raise ValueError(message % name_1)
if end is None:
raise ValueError(message % name_2)
return sum([calculate_weight(edge) for edge in calculate_path(start, end)])
def findNumericalConcept(subject_concept, value_set):
remove = "$,\"[]()? -!abcdefghijklmnopqstuvw"
normalized_value_set = []
for txt in value_set[1:]:
for r in remove:
txt = txt.replace(r, '')
if len(txt) > 0:
normalized_value_set.append(float(txt))
# findign the domains correspond to concepts
candidates_to_domain = findDomains(
findNearestNeighbours(normalized_value_set, 20))
predicted_concept = None
b = False
for candidate in candidates_to_domain.keys():
if candidates_to_domain[candidate] != None and candidates_to_domain[
candidate][28:] == subject_concept:
b = True
predicted_concept = candidate[28:]
break
# loading the concept tree
root = load_tree()
if not b:
min_dis = float('inf')
min_dist_concept = None
for candidate in candidates_to_domain.keys():
x = [
ancestor.name
for ancestor in find_by_attr(root, subject_concept).ancestors
]
x.append(subject_concept)
x = set(x)
y = None
if candidates_to_domain[candidate] != None:
try:
y = [
ancestor.name for ancestor in find_by_attr(
root, candidates_to_domain[candidate]
[28:]).ancestors
]
y.append(candidates_to_domain[candidate][28:])
y = set(y)
curr_dis = len(list(x.union(y) - x.intersection(y)))
if curr_dis < min_dis:
min_dis = curr_dis
min_dist_concept = candidate
except:
continue
predicted_concept = min_dist_concept[28:]
return "http://dbpedia.org/ontology/" + predicted_concept
def test_find_by_attr():
f = Node("f")
b = Node("b", parent=f)
Node("a", parent=b)
d = Node("d", parent=b)
c = Node("c", parent=d, foo=4)
Node("e", parent=d)
g = Node("g", parent=f)
i = Node("i", parent=g)
Node("h", parent=i)
eq_(find_by_attr(f, "d"), d)
eq_(find_by_attr(f, name="foo", value=4), c)
eq_(find_by_attr(f, name="foo", value=8), None)
def bind_data_to_template(root_template, bindings):
for binding in bindings:
data_url = binding.get('data_url')
template_name = binding.get('template_name')
template = find_by_attr(root_template, template_name)
if template:
data = requests.get(data_url).content
template.value = data
def beam_search(inputs_to_ids_fn, starting_state, starting_id, stop_condition_fn, beam_width=3, \
num_states_returned=3, top_k=10, top_p=1.0, num_dist_samples=-1, temperature=1.0):
"""
Searches a space using a beam constrrained tree to find the most likely outcomes.
inputs_to_ids_fn: (state, id)->(new_state, [float]): the prediction function returning an
array of probabilities corresponding to the next valid ids
starting_cache: an object containing the state of the prediction fn before being passed a new id
starting_id: int: the integer used to prime the first distribution
stop_condition_fn: (state)->bool: a function determining if the sequence should be stopped,
this could include maximum length reached.
beam_width: int: number of active beams to search at any particular point.
returns: [state]: most probable terminated states
"""
terminated_states = []
active_states = [(0.0, starting_state, starting_id)] # we use log probabilities, and thus sum future log probs. log(p==1) = 0
beam_tree = Node(f"Start", state=starting_state["input_ids"].cpu().tolist().copy())
while len(terminated_states) < num_states_returned and len(active_states) != 0:
p, best_state, best_next_id = heappop(active_states)
new_state, next_id_probs = inputs_to_ids_fn(best_state, best_next_id)
print("searching for", best_state["input_ids"].cpu().tolist().copy())
parent = find_by_attr(beam_tree, name="state", value=best_state["input_ids"].cpu().tolist())
print("found parent ",parent)
print("adding node with state ",new_state["input_ids"].cpu().tolist().copy())
added_node = Node(f"{best_next_id}", state=new_state["input_ids"].cpu().tolist().copy(), parent=parent)
if stop_condition_fn(new_state):
terminated_states.append((-p, new_state.copy()))
continue
# choose best IDs to add to tree
next_id_probs = np.array(next_id_probs)
scaled_logits = np.log(next_id_probs) / temperature
exp_logits = np.exp(scaled_logits)
next_id_probs = exp_logits / np.sum(exp_logits) # softmax
sorted_ids = np.argsort(next_id_probs)[::-1]
cumulative_probs = np.cumsum(next_id_probs[sorted_ids])
top_p_ids = [idx for idx, cumulative_p in zip(sorted_ids,cumulative_probs) if cumulative_p<=top_p]
top_k_ids = top_p_ids[:top_k]
if num_dist_samples<1:
sampled_ids = top_k_ids[:beam_width]
else:
sampled_ids = np.random.choice(top_k_ids, min(num_dist_samples, len(top_k_ids)), p=normalize(next_id_probs[top_k_ids]), replace=False)
for idx in sampled_ids:
new_prob = p-np.log(next_id_probs[idx])
heappush(active_states, (new_prob, new_state.copy(), idx))
active_states = nsmallest(beam_width, active_states)
for pre, fill, node in RenderTree(beam_tree):
print("%s%s" % (pre, node.name))
return terminated_states
def load_tree():
root = None
f = open('input.txt', 'r')
lines = f.readlines()[1:]
root = Node(lines[0].split(" ")[0])
for line in lines:
line = line.split(" ")
Node("".join(line[1:]).strip(), parent=find_by_attr(root, line[0]))
return root
def get_common_path_alt(path_a, path_b):
""" given two paths, return common path source and target """
node_tree = get_node_tree()
source_a = find_by_attr(node_tree, name='id', value=path_a[0])
target_a = find_by_attr(node_tree, name='id', value=path_a[1])
source_b = find_by_attr(node_tree, name='id', value=path_b[0])
target_b = find_by_attr(node_tree, name='id', value=path_b[1])
source = source_a if source_a == source_b else None
target = None
for node_a, node_b in zip(target_a.path, target_b.path):
if node_a == node_b:
target = node_a
else:
break
return source.id, target.id
def add_children(lines,node,tree):
for x in lines:
parent = x.split(')')[0]
child = x.split(')')[1]
if parent == node:
Node(child, parent=find_by_attr(tree, node))
add_children(lines,child,tree)
return tree
def add_entered_folder(self, parent, folder_name):
root_folder = self.get_current_storage().root_folder
parent_folder = anytree.find_by_attr(root_folder,
name="id",
value=parent.label_id)
current_folder = ThemeFolders(folder_name, parent=parent_folder)
self.populate_tree_view(self.get_current_storage().tree_view, parent,
current_folder)
def is_descendent(parent, child):
""" given two nodes, check if one is ancestor of the other """
root = get_node_tree()
parent_node = find_by_attr(root, name='id', value=parent)
decendency = False
for node in parent_node.descendants:
decendency = node.id == child or decendency
return decendency
def is_ancestor(parent, child):
""" given two nodes, check if one is ancestor of the other """
root = get_node_tree()
child_node = find_by_attr(root, name='id', value=child)
ancestry = False
for node in child_node.ancestors:
ancestry = node.id == parent or ancestry
return ancestry
def get_common_path_list(path_a, path_b):
""" given two paths, return common path source and target """
node_tree = get_node_tree()
source_a = find_by_attr(node_tree, name='id', value=path_a[0])
target_a = find_by_attr(node_tree, name='id', value=path_a[1])
source_b = find_by_attr(node_tree, name='id', value=path_b[0])
target_b = find_by_attr(node_tree, name='id', value=path_b[1])
source = source_a if source_a == source_b else None
common_path = [source]
for node_a, node_b in zip(target_a.path, target_b.path):
if node_a == node_b:
common_path.append(node_a)
else:
break
return common_path
def tree(self, selected_refs, synonyms, hyponyms):
#Adding lists to class:
self.selected_refs = selected_refs
self.synonyms = synonyms
self.hyponyms = hyponyms
participants = Node('participants')
##hyponyms
for hyp in self.hyponyms:
if not findall_by_attr(participants, hyp):
hypo = Node(hyp, parent=participants)
for hyper in self.hyponyms[hyp]:
hyper = Node(hyper, parent=hypo)
else:
for hyper in self.hyponyms[hyp]:
hypo = find_by_attr(participants, hyp)
hyper = Node(hyper, parent=hypo)
##synonyms
for syn in self.synonyms:
overlap = findall_by_attr(participants, syn)
if overlap: #The synonym already exists in the tree
for n in overlap:
syno = find_by_attr(n, syn)
for s in self.synonyms[syn]:
s0 = Node(s, parent=syno) #Creating node
s1 = Node(syno.name, parent=s0) #Creating additional node for doubling and reversing the edge.
else: #If the synonym does not exist in the tree
syno = Node(syn, parent=participants) #Creating node
for s in self.synonyms[syn]:
s0 = Node(s, parent=syno) #Creating synonym
s1 = Node(syno.name, parent=s0) #Reverse node
##remaining participants
for n in self.selected_refs:
if not findall_by_attr(participants, n):
node = Node(n, parent=participants)
self.tree = participants
return participants
def test_field_cross_reference():
data = io.BytesIO(bytes([0x04, 0x00, 0x00, 0x00]))
template = XMLTemplateParser("""
<template name="template0">
<layout name="layout0">
<area name="area0">
<field name="field1-size" size="4"></field>
<field name="field1" size="{field1-size, byteorder=little}"></field>
<field name="field2" size="{field1-size, byteorder=big}"></field>
</area>
</layout>
</template>""").parse()
binalyzer = Binalyzer(template, data)
field1_size = find_by_attr(template, "field1-size")
field1 = find_by_attr(template, "field1")
field2 = find_by_attr(template, "field2")
assert field1_size.size == 4
assert field1_size.value == bytes([0x04, 0x00, 0x00, 0x00])
assert field1.size == 0x4
assert field2.size == 0x4000000
def test_binding_at_stream_assignment():
template = XMLTemplateParser("""
<template name="template0">
<layout name="layout0">
<area name="area0">
<field name="field1_size" size="4"></field>
<field name="field1" size="{field1_size, byteorder=little}"></field>
<field name="field2" size="{field1_size, byteorder=big}"></field>
</area>
</layout>
</template>""").parse()
binalyzer = Binalyzer(template, io.BytesIO(bytes(4 * [0x0])))
binalyzer.data = io.BytesIO(bytes([0x04, 0x00, 0x00, 0x00]))
field1_size = find_by_attr(template, "field1_size")
field1 = find_by_attr(template, "field1")
field2 = find_by_attr(template, "field2")
assert field1_size.size == 4
assert field1_size.value == bytes([0x04, 0x00, 0x00, 0x00])
assert field1.size == 0x4
assert field2.size == 0x4000000
def test_template_reference_with_provider(binalyzer):
data = io.BytesIO(bytes([0xE5, 0x8E, 0x26]))
template = XMLTemplateParser(
"""
<template name="template0">
<layout name="layout0">
<area name="area0">
<field name="field1_size" size="3"></field>
<field name="field1" size="{field1_size, provider=mock.ref_custom}"></field>
</area>
</layout>
</template>""",
binalyzer=binalyzer,
).parse()
binalyzer = Binalyzer(template, data)
field1_size = find_by_attr(template, "field1_size")
field1 = find_by_attr(template, "field1")
assert field1_size.size == 3
assert field1_size.value == bytes([0xE5, 0x8E, 0x26])
assert field1.size == 2526949
def find_deepest_concept(L):
maxDepth = -float('inf')
deepestConceptInClassTree = ""
root = load_tree()
for c, v in L.items():
depth = find_by_attr(root, "Guitarist").depth
L[c] = depth * v
if L[c] > maxDepth:
maxDepth = L[c]
deepestConceptInClassTree = c
return 'http://dbpedia.org/ontology/' + deepestConceptInClassTree
async def analytics_callback_tree_api(request, user):
# look at the current callbacks and return their data in a more manageable tree format
# http://anytree.readthedocs.io/en/latest/
query = await db_model.operation_query()
operation = await db_objects.get(query, name=user['current_operation'])
query = await db_model.callback_query()
dbcallbacks = await db_objects.execute(query.where(Callback.operation == operation))
callbacks = []
# Default values here
display_config = {}
display_config['inactive'] = True # by default, include all callbacks
display_config['strikethrough'] = False
# The POST version of this API function will provide modifiers for what specific information to provide in the callback tree, but the main logic remains the same
if request.method == 'POST':
data = request.json
if 'inactive' in data:
display_config['inactive'] = data['inactive']
if 'strikethrough' in data:
display_config['strikethrough'] = data['strikethrough']
for dbc in dbcallbacks:
if display_config['inactive']: # include everything
callbacks.append(dbc.to_json())
elif dbc.active:
callbacks.append(dbc.to_json())
elif not dbc.active:
json_val = dbc.to_json()
json_val['description'] = "CALLBACK DEAD " + json_val['description']
callbacks.append(json_val)
# every callback with a pcallback of null should be at the root (remove them from list as we place them)
tree = []
while len(callbacks) != 0: # when we hit 0 we are done processing
for c in callbacks:
# this is the root of a 'tree'
if c['pcallback'] == 'null':
display = await analytics_callback_tree_api_function(c, display_config)
tree.append(Node(str(c['id']), display=display))
callbacks.remove(c) # remove the one we just processed from our list
else:
for t in tree:
# for each tree in our list, see if we can find the parent
leaf = find_by_attr(t, str(c['pcallback']))
if leaf:
display = await analytics_callback_tree_api_function(c, display_config)
Node(str(c['id']), parent=leaf, display=display)
callbacks.remove(c)
break
output = ""
for t in tree:
output += str(RenderTree(t, style=DoubleStyle).by_attr("display")) + "\n"
return text(output)
def load_tree():
root = None
f = open('dbp_tree_input.txt', 'r')
lines = f.readlines()[1:]
root = Node(lines[0].split(" ")[0])
for line in lines:
line = line.split(" ")
Node("".join(line[1:]).strip(), parent=find_by_attr(root, line[0]))
### print the tree
# for pre, _, node in RenderTree(root):
# print("%s%s" % (pre, node.name))
return root
def get_parent_gate(self, gate_id):
"""
Retrieve the gate ID for a parent gate of the given gate ID.
:param gate_id: text string of a gate ID
:return: text string of the parent gate ID, None if given gate reference has no parent gate.
"""
n = anytree.find_by_attr(self._gate_tree, gate_id)
if n is None:
raise ValueError("Gate ID %s was not found in gating strategy" %
gate_id)
return n.parent.gate
def make_data(self):
train_path = os.path.join(self.raw_folder, 'ILSVRC2012_img_train')
test_path = os.path.join(self.raw_folder, 'ILSVRC2012_img_val')
meta_path = os.path.join(self.raw_folder, 'ILSVRC2012_devkit_t12')
extract_file(os.path.join(self.raw_folder, 'ILSVRC2012_img_train.tar'),
train_path)
extract_file(os.path.join(self.raw_folder, 'ILSVRC2012_img_val.tar'),
test_path)
extract_file(
os.path.join(self.raw_folder, 'ILSVRC2012_devkit_t12.tar.gz'),
meta_path)
for archive in [
os.path.join(train_path, archive)
for archive in os.listdir(train_path)
]:
extract_file(archive, os.path.splitext(archive)[0], delete=True)
classes_to_labels, classes_size = make_meta(meta_path)
with open(
os.path.join(meta_path, 'data',
'ILSVRC2012_validation_ground_truth.txt'),
'r') as f:
test_id = f.readlines()
test_id = [int(i) for i in test_id]
test_img = sorted(
[os.path.join(test_path, file) for file in os.listdir(test_path)])
test_wnid = []
for test_id_i in test_id:
test_node_i = anytree.find_by_attr(classes_to_labels['label'],
name='id',
value=test_id_i)
test_wnid.append(test_node_i.name)
for test_wnid_i in set(test_wnid):
os.mkdir(os.path.join(test_path, test_wnid_i))
for test_wnid_i, test_img in zip(test_wnid, test_img):
shutil.move(
test_img,
os.path.join(test_path, test_wnid_i,
os.path.basename(test_img)))
train_img, train_label = make_img(
os.path.join(self.raw_folder, 'ILSVRC2012_img_train'),
IMG_EXTENSIONS, classes_to_labels['label'])
test_img, test_label = make_img(
os.path.join(self.raw_folder, 'ILSVRC2012_img_val'),
IMG_EXTENSIONS, classes_to_labels['label'])
train_target = {'label': train_label}
test_target = {'label': test_label}
return (train_img, train_target), (test_img,
test_target), (classes_to_labels,
classes_size)
def make_tree(root, name, attribute=None):
if len(name) == 0:
return
if attribute is None:
attribute = {}
this_name = name[0]
next_name = name[1:]
this_attribute = {k: attribute[k][0] for k in attribute}
next_attribute = {k: attribute[k][1:] for k in attribute}
this_node = anytree.find_by_attr(root, this_name)
this_index = root.index + [len(root.children)]
if this_node is None:
this_node = anytree.Node(this_name, parent=root, index=this_index, **this_attribute)
make_tree(this_node, next_name, next_attribute)
return
def is_descendent(parent, child):
""" given two nodes, check if one is ancestor of the other """
root = get_node_tree()
parent_node = find_by_attr(root, name='id', value=get_ip_by_handle(parent))
decendency = False
for node in parent_node.descendants:
if "." in child:
decendency = node.id == child or decendency
else:
decendency = next(
(True for x in node.properties['handles'] if x == child),
False) or decendency
return decendency
def make_img(path, extensions, classes_to_labels):
img, label = [], []
classes = []
leaf_nodes = classes_to_labels.leaves
for node in leaf_nodes:
classes.append(node.name)
for c in sorted(classes):
d = os.path.join(path, c)
if not os.path.isdir(d):
continue
for root, _, filenames in sorted(os.walk(d)):
for filename in sorted(filenames):
if has_file_allowed_extension(filename, extensions):
cur_path = os.path.join(root, filename)
img.append(cur_path)
label.append(anytree.find_by_attr(classes_to_labels, c).flat_index)
return img, label
def printTree(edges_list):
# create root
root = Node("Root")
for (node1, node2) in edges_list:
Node(node2, parent=find_by_attr(root, node1))
for pre, _, node in RenderTree(root):
print("%s%s" % (pre, node.name))
# Print to file --- GRAPVIZ MUST BE INSTALLED!!!
installed_packages = [
"%s" % i.key for i in pip.get_installed_distributions()
]
if 'graphviz' in installed_packages:
RenderTreeGraph(root).to_picture("../Outputs/Part3/tree.png")
print('Tree generated correctly')
def organizeParentPanels(panels, m):
print('todo: organizing parent panels')
root = Node('root')
print('created root node')
for key in panels:
n = Node(key)
print(n)
parent = panels[key]['parent']
print(f'node:{key}, parent:={parent}')
print(f'{type(parent)}, {parent}')
if not parent:
print('not parent check')
n.parent = root
else:
pn = find_by_attr(root, parent)
n.parent = pn
m['nodes'] = root
pass