def create_tree(arr):
tree = Tree()
print("creating your tree ..")
print(tree.get_node("compsci"))
tree.create_node("compsci", "papa")
for course in arr:
num = comparator(course)
if (num[0] == '4'):
print(course)
tree.create_node(course, course, parent="papa")
pre_reqs = get_prerequisites(course)
time.sleep(1)
for pre in pre_reqs:
if (tree.get_node(pre) == None):
tree.create_node(pre, course + "" + pre, parent=course)
pre2 = get_prerequisites(pre)
time.sleep(1)
for pr in pre2:
if (tree.get_node(pr) == None):
tree.create_node(pr,
course + "" + pre + "" + pr,
parent=course + "" + pre)
tests = get_prerequisites(pr)
time.sleep(1)
for tst in tests:
if (tree.get_node(tst) == None):
tree.create_node(
tst,
course + "" + pre + "" + pr + "" + tst,
parent=course + "" + pre + "" + pr)
return tree
class Xml2Obj:
'''transform XML to Object'''
def __init__(self):
self.root = None
self.nodeStack = []
self.numberCount = 0
self.tree = Tree()
def StartElement(self, name, attributes):
'Expat start element event handler'
# put the element into stack and make it become child_element
if self.nodeStack:
parent = self.nodeStack[-1]
# make instance of class
element = Element(name, attributes, self.tree,
self.numberCount, parent.number)
parent.addChild(element)
self.tree = parent.getTree()
else:
element = self.root = Element(
name, attributes, self.tree, self.numberCount, None)
self.tree.create_node(self.root.name, self.root.number)
self.numberCount += 1
self.nodeStack.append(element)
def EndElement(self, name):
'Expat end element event handler'
if self.nodeStack[-1].cdata:
self.tree.get_node(
self.nodeStack[-1].number).tag += self.nodeStack[-1].cdata
self.nodeStack.pop()
def CharacterData(self, data):
'''Expat character data event handler'''
if data.strip():
element = self.nodeStack[-1]
element.cdata += data
def showTree(self):
self.tree.show(key=lambda node:node.identifier)
# In https://github.com/caesar0301/treelib/pull/180 :
# self.tree.show(sorting=False)
def toDot(self, file):
self.tree.to_graphviz(file)
# In https://github.com/caesar0301/treelib/pull/179 :
# self.tree.to_graphviz(file, sorting=False)
def Parse(self, filename):
# create Expat analyzer
Parser = expat.ParserCreate()
# Set the Expat event handlers to our methods
Parser.StartElementHandler = self.StartElement
Parser.EndElementHandler = self.EndElement
Parser.CharacterDataHandler = self.CharacterData
# analyz XML file
Parser.Parse(open(filename).read(), 1)
return self.root
class AcquisitionChain(object):
def __init__(self, parallel_prepare=False):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
self._presets_list = list()
self._parallel_prepare = parallel_prepare
self._device2one_shot_flag = weakref.WeakKeyDictionary()
@property
def nodes_list(self):
nodes_gen = self._tree.expand_tree()
nodes_gen.next() # first node is 'root'
return list(nodes_gen)
def add(self, master, slave):
self._device2one_shot_flag.setdefault(slave, False)
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if (slave_node.bpointer is not self._root_node
and master_node is not slave_node.bpointer):
raise RuntimeError(
"Cannot add acquisition device %s to multiple masters, current master is %s"
% (slave, slave_node._bpointer))
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(tag=master.name,
identifier=master,
parent="root")
if slave_node is None:
slave_node = self._tree.create_node(tag=slave.name,
identifier=slave,
parent=master)
else:
self._tree.move_node(slave, master)
slave.parent = master
def add_preset(self, preset):
self._presets_list.append(preset)
def set_stopper(self, device, stop_flag):
"""
By default any top master device will stop the scan.
In case of several top master, you can define which one won't
stop the scan
"""
self._device2one_shot_flag[device] = not stop_flag
def __iter__(self):
if len(self._tree) > 1:
return AcquisitionChainIter(
self, parallel_prepare=self._parallel_prepare)
else:
return iter(())
class AcquisitionChain(object):
def __init__(self):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
def add(self, master, slave):
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer:
raise RuntimeError(
"Cannot add acquisition device %s to multiple masters, current master is %s"
% (slave, slave_node._bpointer)
)
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root")
if slave_node is None:
slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master)
else:
self._tree.move_node(slave_node, master_node)
def _execute(self, func_name):
tasks = list()
prev_level = None
for dev in reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]):
node = self._tree.get_node(dev)
level = self._tree.depth(node)
if prev_level != level:
gevent.joinall(tasks)
tasks = list()
func = getattr(dev, func_name)
tasks.append(gevent.spawn(func))
gevent.joinall(tasks)
def prepare(self, dm, scan_info):
# self._devices_tree = self._get_devices_tree()
for master in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionMaster)):
del master.slaves[:]
for dev in self._tree.get_node(master).fpointer:
master.slaves.append(dev)
dm_prepare_task = gevent.spawn(dm.prepare, scan_info, self._tree)
self._execute("_prepare")
dm_prepare_task.join()
def start(self):
self._execute("_start")
for acq_dev in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionDevice)):
acq_dev.wait_reading()
dispatcher.send("end", acq_dev)
def generatetree(lista):
tree = Tree() # se genera el arbol
tree.create_node("calificador/dashboard/templates/dashboard", "raiz") # se agrega la raiz como el dashboard
for i in lista:
if tree.get_node(i[0])==None:# se revisa si el archivo esta en el arbol y se arregla
tree.create_node(i[0],i[0],parent="raiz") # se agrega el archivo al arbol
for i in lista:
if tree.get_node(i[1]) == None: # se revisa si la iimagen se ha agregado al arbol
tree.create_node(i[1],i[1],parent=i[0]) # se agrega la imagen al arbol
tree.save2file("imagenes.txt")
class AcquisitionChain(object):
def __init__(self, parallel_prepare=False):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
self._presets_list = list()
self._parallel_prepare = parallel_prepare
self._device2one_shot_flag = weakref.WeakKeyDictionary()
@property
def nodes_list(self):
nodes_gen = self._tree.expand_tree()
nodes_gen.next() # first node is 'root'
return list(nodes_gen)
def add(self, master, slave):
self._device2one_shot_flag.setdefault(slave, False)
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if(slave_node.bpointer is not self._root_node and
master_node is not slave_node.bpointer):
raise RuntimeError("Cannot add acquisition device %s to multiple masters, current master is %s" % (
slave, slave_node._bpointer))
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(
tag=master.name, identifier=master, parent="root")
if slave_node is None:
slave_node = self._tree.create_node(
tag=slave.name, identifier=slave, parent=master)
else:
self._tree.move_node(slave, master)
slave.parent = master
def add_preset(self, preset):
self._presets_list.append(preset)
def set_stopper(self, device, stop_flag):
"""
By default any top master device will stop the scan.
In case of several top master, you can define which one won't
stop the scan
"""
self._device2one_shot_flag[device] = not stop_flag
def __iter__(self):
if len(self._tree) > 1:
return AcquisitionChainIter(self, parallel_prepare=self._parallel_prepare)
else:
return iter(())
def test_modify_node_identifier_directly_failed(self):
tree = Tree()
tree.create_node("Harry", "harry")
tree.create_node("Jane", "jane", parent="harry")
n = tree.get_node("jane")
self.assertTrue(n.identifier == 'jane')
# Failed to modify
n.identifier = "xyz"
self.assertTrue(tree.get_node("xyz") is None)
self.assertTrue(tree.get_node("jane").identifier == 'xyz')
def test_modify_node_identifier_recursively(self):
tree = Tree()
tree.create_node("Harry", "harry")
tree.create_node("Jane", "jane", parent="harry")
n = tree.get_node("jane")
self.assertTrue(n.identifier == 'jane')
# Success to modify
tree.update_node(n.identifier, identifier='xyz')
self.assertTrue(tree.get_node("jane") is None)
self.assertTrue(tree.get_node("xyz").identifier == 'xyz')
def treeify_full_path(q: Queryable) -> None:
tree = Tree()
events = q.to_list()
for event in events:
path = list(reversed(event.data.full_path))
for idx, node in enumerate(path):
if tree.get_node(node):
continue
parent = path[idx - 1] if idx > 0 else None
parent_node = tree.get_node(parent)
tree.create_node(node, node, parent_node)
tree.show(line_type="ascii-em")
def treeify_category_path(q: Queryable) -> None:
tree = Tree()
events = q.to_list()
for event in events:
path = list(reversed(event.data.category_path))
for idx, node_name in enumerate(path):
if tree.get_node(node_name):
continue
parent = path[idx - 1] if idx > 0 else None
parent_node = tree.get_node(parent)
tree.create_node(
f"{node_name} time={query(events).where(lambda x: node_name in x.data.category_path).sum(lambda y: y.duration) // 1}",
node_name, parent_node)
tree.show(line_type="ascii-em")
def __init__(self, mol, atom0, atom1, depth=5):
self.mol = mol
tree = Tree()
bond_order = mol.GetBondBetweenAtoms(
atom0.GetIdx(), atom1.GetIdx()).GetBondTypeAsDouble()
tree.create_node(tag=[atom0.GetAtomicNum(), bond_order],
identifier=atom0.GetIdx(),
data=atom0)
tree.create_node(tag=[atom1.GetAtomicNum(), bond_order],
identifier=atom1.GetIdx(),
data=atom1,
parent=atom0.GetIdx())
for _ in range(depth):
for node in tree.all_nodes():
if node.is_leaf():
for atom in node.data.GetNeighbors():
tree_id = tree._identifier
if atom.GetIdx() != node.predecessor(tree_id=tree_id):
order = mol.GetBondBetweenAtoms(
atom.GetIdx(),
node.data.GetIdx()).GetBondTypeAsDouble()
identifier = atom.GetIdx()
while tree.get_node(identifier) is not None:
identifier += len(mol.GetAtoms())
tree.create_node(tag=[atom.GetAtomicNum(), order],
identifier=identifier,
data=atom,
parent=node.identifier)
self.tree = tree
def create_dataset(min_points: int, classes_per_layer: List[int], class_tree: Tree, is_perfectly_solvable=False) -> (
np.array, np.array):
class_value_ranges = []
for classes_in_current_level in classes_per_layer:
class_value_ranges.append(list(range(classes_in_current_level)))
# Local classes are unique within one tree layer, but not across layers. They are used to choose the next branch
# while traversing the class_tree, to find the global classes. Global classes are those given in the class_tree.
# They are unique among all nodes of the class_tree.
local_class_vectors = list(itertools.product(*class_value_ranges))
n_repetitions = math.ceil(min_points / len(local_class_vectors))
X = []
Y = []
for rep in range(n_repetitions):
for class_vector in local_class_vectors:
global_class_vector = []
current_node_id = class_tree.root
for local_class in class_vector:
possible_next_branches = class_tree.get_node(current_node_id).successors(0)
next_node_id = possible_next_branches[local_class]
global_class_vector.append(next_node_id)
current_node_id = next_node_id
datapoint = create_datapoint(global_class_vector, is_perfectly_solvable)
X.append(datapoint)
Y.append(global_class_vector)
return np.array(X), np.array(Y)
def read_tree_structure(
db_dir
): # tree.data [node category, accessibility, covered_num, total_num, abundance]
GCF = {}
f = open(db_dir + "/tree_structure.txt", "r")
lines = f.readlines()
if (len(lines) == 1):
tree = pkl.load(open(db_dir + '/tree.pkl', 'rb'))
return tree, GCF
tree = Tree()
sequences = []
if (lines[-1].split("\t")[1] != "N"):
for i in range(0, len(lines)):
if (lines[i].split("\t")[1] == 'N'):
break
for j in range(i, len(lines)):
sequences.append(lines[j])
for j in range(0, i):
sequences.append(lines[j])
else:
sequences = reversed(lines)
for i in sequences:
temp = i.rstrip().split("\t")
if (temp[1] == "N"):
tree.create_node(identifier=int(temp[0]))
else:
tree.create_node(identifier=int(temp[0]), parent=int(temp[1]))
if (len(temp) == 4):
GCF[tree.get_node(int(temp[0]))] = temp[3] # cluster with size 1
return tree, GCF
def generateTree(informacion):
#print(informacion)
tree = Tree() # se genera el arbol
tree.create_node("calificador/dashboard/models", "raiz") # se genera la raiz
for i in informacion.keys(): # se recorren las llaves del diccionario
if tree.get_node(i) == None:
tree.create_node(i, i, parent="raiz") # se agregan las llaves como hizos del nodo raiz
for i in informacion.keys():
lis = informacion[i] # se extrae la lista de menciones del cada llave
for j in lis:# se recorren la lista
if tree.get_node(j[0]) == None: # se revisa si la el archivo (valor 0 en la tupla) ya existe en el arbol si no se agrega el nodo hijo
tree.create_node(j[0], j[0], parent=i)
for k in lis: # se vuelve a recorrer la lista
if tree.get_node(k[1]) == None: # Se revisa que si la mencion (valor 1 en la tupla) ya existe en el nodo y si no se agrega al arbol con el archivo como padre
tree.create_node(k[1], k[1], parent=k[0])
tree.save2file("Modulos.txt") #se guarde el arbol en un archivo
def make_account_tree_from_trans(trans):
""" extract all accounts from a list of Gnucash account paths
Each account name is a full path. Parent accounts with no
transactions will be missing from the data, so reconstruct the
complete tree implied by the transaction data.
As long as the accounts are sorted hierarchically, the algorithm
should never encounter a missing parent except the first node.
If there are multiple heads in the data, they will all belong to
root, so the tree will still be a DAG
"""
tree = Tree()
tree.create_node(tag=ROOT_TAG, identifier=ROOT_ID)
accounts = trans['full account name'].unique()
for account in accounts:
branches = account.split(':') # example: Foo:Bar:Baz
for i, branch in enumerate(branches):
name = branch
if i == 0:
parent = ROOT_ID
else:
parent = branches[i - 1]
if not tree.get_node(name):
tree.create_node(tag=name, identifier=name, parent=parent)
tree = trim_excess_root(tree)
return tree
def print_prob_val(self, fname="OutTree.txt"):
n_tree = Tree(tree=self.tree)
for node in n_tree.nodes:
node = n_tree.get_node(node)
node.tag = "{tag} - {val} - {prob}".format(tag=node.tag, val=node.data[0], prob=node.data[1])
n_tree.save2file(fname)
self.tree = None
def get_best_meld_combins_from_arr(cls, arr: list) -> list:
sorted_arr = arr[:]
sorted_arr.sort()
tree = Tree()
identifier = 'root'
tree.create_node([], identifier)
MjMath.get_melds_tree_from_sorted_arr(sorted_arr, tree, identifier)
paths = tree.paths_to_leaves()
_combinations = []
for path in paths:
_combin = []
for identifier in path:
meld = tree.get_node(identifier).tag
_combin.append(meld)
_combin.sort()
if _combin not in _combinations:
_combinations.append(_combin)
len_arr = [len(x) for x in _combinations]
max_len = max(len_arr)
best_combinations = []
for _combin in _combinations:
if len(_combin) == max_len:
best_combinations.append(_combin)
return best_combinations
def pathTree(data):
tree = Tree()
for d in data:
path = d[0].split('-')
if len(path) > 2:
parent = path[0]
if tree.get_node(parent) is None:
tree.create_node(parent, parent)
for i in range(1, len(path) - 1):
n = parent + '-' + path[i]
if tree.get_node(n) is None:
tree.create_node(path[i], n, parent=parent)
parent = n
n = parent + '-' + path[-1]
tree.create_node(path[-1], n, parent=parent, data=d[1])
return tree
def convert_to_tree(line):
center, orbital = line.strip().split(')')
new_tree = Tree()
new_tree.create_node(identifier=center)
new_tree.create_node(identifier=orbital, parent=new_tree.get_node(center))
return new_tree
def create_tree(arr, depth='4'):
tree = Tree()
print("creating your tree ..")
# print(tree.get_node("compsci"))
tree.create_node("compsci", "papa")
for course in arr:
num = comparator(course)
if (num[0] == str(depth)):
# print(course)
tree.create_node(course, course, parent="papa")
pre_reqs = get_prerequisites(course)
time.sleep(1)
for pre in pre_reqs:
if (tree.get_node(course + "" + pre) == None):
tree.create_node(pre, course + "" + pre, parent=course)
pre2 = get_prerequisites(pre)
time.sleep(1)
for pr in pre2:
if (tree.get_node(course + "" + pre + "" +
pr) == None):
tree.create_node(pr,
course + "" + pre + "" + pr,
parent=course + "" + pre)
tests = get_prerequisites(pr)
time.sleep(1)
for tst in tests:
if (tree.get_node(course + "" + pre + "" + pr +
"" + tst) == None):
tree.create_node(
tst,
course + "" + pre + "" + pr + "" + tst,
parent=course + "" + pre + "" + pr)
print("Still loading...")
for course in arr:
if (not tree.contains(course)):
tree.create_node(course, course, parent="papa")
pre_reqs = get_prerequisites(course)
time.sleep(1)
for pre in pre_reqs:
if (tree.get_node(course + "" + pre) == None):
tree.create_node(pre, course + "" + pre, parent=course)
pre2 = get_prerequisites(pre)
time.sleep(1)
return tree, arr
def convert_buckets_to_classes(bucket_vector: List[int],
class_tree: Tree) -> List[int]:
current_node = class_tree.root
targets = []
for bucket_id in bucket_vector:
current_node = class_tree.get_node(current_node).successors(
0)[bucket_id]
targets.append(current_node)
return targets
def get_invoke_tree(self, method: MethodId, search_depth=3):
tree = Tree(deep=search_depth, identifier=method.address)
# Parent method with invoke address list
tree.create_node(identifier=method, data=[])
for _ in range(search_depth):
for leaf in tree.leaves():
uppers = self.apkinfo.find_upper_methods(leaf.identifier)
for offset, upper in uppers:
bytecode = self.apkinfo.find_bytecode_by_addr(
upper.dexindex, offset)
if not tree.contains(upper):
tree.create_node(identifier=upper,
data=[bytecode],
parent=leaf)
else:
tree.get_node(upper).data.append(bytecode)
return tree
def min_value(
s: ndarray,
moves: Tuple[Tuple[int, int, int, int, int], ...],
alpha: int,
beta: int,
depth: int,
start: float,
t: Tree,
parent: str,
):
if has_timed_out(start):
raise TimeoutException(moves)
if is_cutoff_state(s, depth, d):
return evaluate(s, species_played, heuristic_played), s, moves
v = infinity
next_state = s
next_moves = moves
successor_states, successor_move_options = get_successors(
s, TYPE_TO_OPPONENT_POSITION_INDEX[species_played])
for k in range(len(successor_move_options)):
successor_state = successor_states[k]
successor_moves = successor_move_options[k]
try:
# For debugging purposes, add tree node with the computed score
uid = str(uuid4())
t.create_node(uid, uid, parent=parent)
next_max = max_value(
successor_state,
successor_moves,
alpha,
beta,
depth + 1,
start,
t,
parent=uid,
)[0]
t.get_node(uid).tag = "{} - Us: {} vs Them: {}".format(
next_max,
get_our_size(successor_state, species_played),
get_opponent_size(successor_state, species_played),
)
except TimeoutException:
raise TimeoutException(moves)
if next_max < v:
v = next_max
next_state = successor_state
next_moves = successor_moves
if v <= alpha:
return v, next_state, next_moves
beta = min(beta, v)
return v, next_state, next_moves
class BKT(object):
def __init__(self, wordlist): #初始化，输入字典
self.wordlist = wordlist
self.BKtree = Tree()
self.BKtree.create_node(self.wordlist[0], self.wordlist[0])
self.result = [] #储存候选正确词
self.result_dist = [] #储存距离
def buildBK(self): #建树
for i in range(1, len(self.wordlist)):
currentFather = self.BKtree.get_node(
self.wordlist[0]) #以根节点作为初始父节点
self.insert(self.wordlist[i], currentFather)
def insert(self, word, currentFather): #插入新词
#print(currentFather.identifier)
dist = Levenshtein.distance(word, currentFather.identifier)
mark = 0
for ID in currentFather.fpointer:
if self.BKtree.get_node(ID).data == dist:
father = self.BKtree.get_node(ID)
mark = 1
self.insert(word, father)
if mark == 0: #mark为0，说明没有子树与当前父节点的距离为插入词到父节点的距离，要创造新节点
try: #如果插入词已经存在，这步会报错，所以这里用了try结构
self.BKtree.create_node(word,
word,
parent=currentFather,
data=dist)
except:
pass
def search(self, n, word): #搜索
self.result = [] #先将候选集清空
self.result_dist = []
currentFather = self.BKtree.get_node(self.wordlist[0])
self.find(n, word, currentFather)
if (len(self.result)) > 0:
d = min(self.result_dist)
index = self.result_dist.index(d)
word = self.result[index]
return d, word
else:
return len(word), word
def find(self, n, word, currentFather):
dist = Levenshtein.distance(word, currentFather.identifier)
if dist <= n: #判断父节点是不是与错词的编辑距离小于n，是的话就加入候选集
self.result.append(currentFather.identifier)
self.result_dist.append(dist)
for ID in currentFather.fpointer:
if (self.BKtree.get_node(ID).data <=
dist + n) and (self.BKtree.get_node(ID).data >= dist - n):
father = self.BKtree.get_node(ID)
self.find(n, word, father) #一步步往下走，直到叶节点停止
def build_primary_tree():
raw_tree = build_raw_tree()
def inherit(node, field, parent_node):
if None in (node, parent_node, parent_node.data):
return
if field is None:
for k,v in parent_node.data.items():
if k not in node.data:
node.data[k] = v
if parent_node.bpointer is not None:
inherit(node, field, raw_tree.get_node(parent_node.bpointer))
return
if field in parent_node.data:
node.data[field] = parent_node.data[field]
elif parent_node.bpointer is not None:
inherit(node, field, raw_tree.get_node(parent_node.bpointer))
tree = Tree()
dynamic_properties = ['inherit']
for node in raw_tree.all_nodes():
raw_data = {k:v for (k,v) in node.data.items() if k not in dynamic_properties} if node.data else {}
dynamic_map = {k:v for (k,v) in node.data.items() if k in dynamic_properties} if node.data else {}
gen_node = tree.create_node(tag=node.tag, identifier=node.identifier, parent=node.bpointer, data=raw_data)
if raw_data is None:
continue
if 'inherit' in dynamic_map:
inherit_behavior = dynamic_map['inherit']
if type(inherit_behavior) is list:
for inherit_field in inherit_behavior:
inherit(gen_node, inherit_field, tree.get_node(gen_node.bpointer))
elif type(inherit_behavior) is str and inherit_behavior == 'all':
inherit(gen_node, None, tree.get_node(gen_node.bpointer))
return tree
def map_tree_to_program(self, tree: Tree) -> str:
self._node_to_subprog = {}
frontier = [] # Tree nodes that are left to be explored
for leaf in tree.leaves():
span = leaf.data.span
self._node_to_subprog[span] = self._node_to_type(leaf)
parent = tree.parent(leaf.identifier)
if parent and parent not in frontier:
frontier.append(tree.parent(leaf.identifier))
while frontier:
node = frontier.pop()
children = tree.children(node.identifier)
assert len(children) == 2
# check if children were already discovered
if not all([
child.data.span in self._node_to_subprog
for child in children
]):
frontier.insert(0, node)
continue
child_1 = self._node_to_subprog[children[0].data.span]
child_2 = self._node_to_subprog[children[1].data.span]
try:
if child_1 and not child_2: # child_2=='NO_LABEL'
self._node_to_subprog[node.data.span] = child_1
elif not child_1 and child_2: # child_1=='NO_LABEL'
self._node_to_subprog[node.data.span] = child_2
elif not child_1 and not child_2: # Both children are assigned with 'NO_LABEL'
self._node_to_subprog[node.data.span] = self._node_to_type(
node) # ignore children and propagate parent
else:
assert child_2.is_full(
) # make sure child_2 value can be formed
self._node_to_subprog[node.data.span] = child_1.apply(
child_2)
except Exception as e:
try:
self._node_to_subprog[node.data.span] = child_2.apply(
child_1)
except Exception as e:
raise Exception('final apply_exception: {}'.format(e))
parent = tree.parent(node.identifier)
if parent and parent not in frontier:
frontier.insert(0, parent)
inner_program = self._node_to_subprog[tree.get_node(
tree.root).data.span].get_value() # return the root's value
return inner_program
def get_ascii_tree(list_tab_dir, project):
# Return ascii tree
tree = Tree()
tree.create_node(project, project)
for i in range(len(list_tab_dir)):
list_split = list_tab_dir[i][0].split("/")
for n in range(len(list_split)):
if n == 0 and tree.get_node(list_split[n]) == None:
tree.create_node(list_split[n] + ' ( ' +
str(list_tab_dir[i][1]) + ' lines )',
list_split[n],
parent=project)
else:
if tree.get_node(list_split[n]) == None:
tree.create_node(list_split[n] + ' ( ' +
str(list_tab_dir[i][1]) + ' lines )',
list_split[n],
parent=list_split[n - 1])
return tree
def build_header_tree(self):
header_tree = Tree()
root_field = ReportDefinitionField()
root_field.alias = 'root'
root_field.header = 'Root'
root_field.sub_fields = self.report_definition.fields
self.__build_header_tree(header_tree, None, root_field)
self.__prepare_header_data(header_tree, header_tree.get_node(header_tree.root))
return header_tree
def retrieve_dependencies(self, jarName):
if jarName is None:
root = self.tree.get_node(self.tree.root)
root = root.data.jarName
else:
root = jarName
tgfOutput = subprocess.Popen('dosocs2 dependencies ' + root, stdout=subprocess.PIPE, shell=True)
count = 0
tree = Tree()
dependencies = []
relationships = []
while True:
line = tgfOutput.stdout.readline()
if not line:
break
match = re.match(r"(\d+) - (.*)", line)
if match:
if count == 0:
count = count + 1
tree.create_node(match.group(2), match.group(1))
else:
dependencies.append((match.group(2), match.group(1)))
match = re.match(r"(\d+) (\d+)", line)
if match:
relationships.append((match.group(1), match.group(2)))
if not relationships:
print("No child relationships for " + jarName)
return None
while relationships:
for item in relationships:
node = tree.get_node(item[0])
if node is not None:
rel = [item for item in relationships if int(item[0]) == int(node.identifier)]
if rel is not None:
rel = rel[0]
dep = [item for item in dependencies if int(item[1]) == int(rel[1])]
if dep is not None:
dep = dep[0]
tree.create_node(dep[0], dep[1], parent=node.identifier)
relationships.remove(rel)
dependencies.remove(dep)
tree.show()
if jarName is None:
os.chdir(os.pardir)
def debug_serialize(tree: Tree):
def recursive(node) -> str:
result = ""
for child in tree.children(node.identifier):
result += "_" * tree.depth(node) + child.data + "\n"
if child.tag == "function":
result += recursive(child)
return result
return "Serialized tree view:\n" + recursive(tree.get_node(0))
def serialize(tree: Tree):
def recursive(node) -> str:
result = ""
for child in tree.children(node.identifier):
if child.tag == "operator" or child.tag == "expression":
result += child.data
elif child.tag == "function":
result += "{}({})".format(child.data, recursive(child))
return result
return recursive(tree.get_node(0))
def store_with_activity(
self, record: Union[ProvGeneration, ProvUsage], subtree: Tree
) -> Activity:
rfrom = record.formal_attributes[0][1]
rto = record.formal_attributes[1][1]
nfrom = subtree.get_node(self.entity_to_id(rfrom, subtree))
nto = subtree.get_node(self.entity_to_id(rto, subtree))
type = record.get_type().localpart
# nfrom.data = self.get(nfrom.data.id, downloadFile=False)
activity = self.get_activity(nfrom.data)
nto.data = self.store(nto.data, forceVersion=False)
if type == "Usage":
activity.used(nto.data)
activity["name"] = activity.get("name", "used")
elif type == "Generation":
activity.used(nto.data, wasExecuted=True)
activity["name"] = activity.get("name", "generated")
else:
activity["name"] = activity.get("name", "notype")
nfrom.data = self.store(nfrom.data, activity=activity, forceVersion=False)
return activity
def run(self, args):
tree = self._login_info_manager.tree()
if not args[0] in tree:
return
tmp_tree = Tree()
tmp_tree.create_node(args[0], '')
has_leef = False
for tree_node in tree.children(args[0]):
tmp_tree.paste('', tree.subtree(tree_node.identifier))
has_leef = True
if not has_leef:
tmp_tree.get_node('').tag = tree.get_node(args[0]).tag
tmp_tree.show()
def combine(T, alpha, beta):
"""
combine is a function that combines two nodes of tree T, changing nodes alpha and beta to be children of a new node gama, where alpha and beta are sisters.
param T: the given tree
param alpha: the target node of T
param beta: the target node of T
return: the new tree after combining operation.
"""
TT = Tree(T.subtree(T.root), deep=True)
new_id = max(ids_of_all_nodes(T)) + 1
parent_id = TT.get_node(alpha).bpointer
TT.create_node(tag=new_id, identifier=new_id, parent=parent_id, data=None)
TT.move_node(alpha, new_id)
TT.move_node(beta, new_id)
return TT
class StateMachine(object):
"""A class to track information about a state machine"""
def __init__(self, name):
self.name = name
self.events = {}
self.effects = {}
self.state_tree = Tree()
self.current_state = None
# Add the Root state automatically
self.add_state('Root')
def add_state(self, name):
assert isinstance(name, str)
state_node = Node(identifier=name, data=State(name))
if self.current_state is None:
self.state_tree.add_node(state_node)
self.current_state = state_node.data
else:
self.state_tree.add_node(state_node, self.current_state.name)
def add_event(self, ev):
assert isinstance(ev, Event)
self.events[ev.name] = ev
def add_effect(self, eff):
assert isinstance(eff, Effect)
self.effects[eff.name] = eff
def enter_state(self, state):
self.current_state = state
def exit_state(self, state):
self.current_state = self.state_tree.parent(state.name).data
def get_state_by_name(self, state_name):
return self.state_tree.get_node(state_name).data
def iiif_recurse(uri, tr=None, parent_nid=None,
separator='/', parent_type=None):
'''
Treelib nodes have a tag (human readable), and a nid (node id).
Sanitised uris are used in the nids, throughout, concatenated
with the parent id, with a separator, so the path segments can
be recreated.
Comments: Nid and denid adds an extra level of list handling, as it joins
and the splits. Might be better to create the path (no processing)
and then create the nid.
'''
try:
print 'URI: %s' % uri
obj = IIIF_Object(uri)
if not tr:
tr = Tree()
if parent_nid:
root_nid = separator.join([parent_nid, sanitise_uri(uri)])
if parent_type:
obj.data['parent_type'] = parent_type
if not tr.get_node(root_nid):
tr.create_node(uri, root_nid, parent=parent_nid, data=obj.data)
else:
root_nid = sanitise_uri(uri)
tr.create_node(uri, root_nid, data=obj.data)
recursion_lists = [
'members', 'collections', 'manifests', 'sequences', 'canvases']
dereferenceable = ['sc:Collection', 'sc:Manifest']
if obj.source_dict:
dict_parse(obj.source_dict, root_nid, tr, separator,
recursion_lists, dereferenceable)
except:
raise
return tr
class Maze:
def __init__(self, puzzleFile):
self.wrigglers = [] # Sorted list by ID
self.parsePuzzleFile(puzzleFile)
self.tree = Tree()
def parsePuzzleFile(self, filename):
f = open(filename, 'r')
columns, rows, numWrigglers = f.readline().split()
static.columns = int(columns)
static.rows = int(rows)
static.board = [['-1' for y in range(static.rows)] for x in range(static.columns)]
for y in range(static.rows):
line = f.readline().split()
for x in range(len(line)):
static.board[x][y] = line[x]
assert len(static.board) == static.columns
assert len(static.board[0]) == static.rows
static.goal = Coord(static.columns - 1, static.rows - 1)
for _ in range(int(numWrigglers)):
self.wrigglers.append(self.makeWriggler())
self.wrigglers.sort()
print("Empty board")
util.printBoard()
def clearMaze(self):
for y in range(static.rows):
for x in range(static.columns):
if(static.board[x][y] != static.WALL):
static.board[x][y] = static.EMPTY
def makeWriggler(self):
wriggler = Wriggler()
bodyCoords = []
tailCoord = None
breakOut = False
# Find head
for y in range(static.rows):
for x in range(static.columns):
if(wriggler.isHead(x, y)):
headCoord = Coord(x, y)
breakOut = True
break
if(breakOut):
break
nextCoord = Coord(newCoord=headCoord)
bFirst = True
while(True):
oldCoord = Coord(newCoord=nextCoord)
if(bFirst):
nextCoord, _ = wriggler.followBody(nextCoord, True)
static.board[oldCoord.x][oldCoord.y] = static.EMPTY
bFirst = False
else:
nextCoord, isTail = wriggler.followBody(nextCoord)
if(isTail):
tailCoord = Coord(newCoord=nextCoord)
wrigglerID = int(static.board[tailCoord.x][tailCoord.y])
static.board[tailCoord.x][tailCoord.y] = static.EMPTY
break
static.board[oldCoord.x][oldCoord.y] = static.EMPTY
bodyCoords.append(Coord(newCoord=oldCoord))
wrigglerCoords = [headCoord, bodyCoords, tailCoord]
wriggler.location = wrigglerCoords
wriggler.id = wrigglerID
return wriggler
def testGoal(self, state):
inGoal = False
wriggler = state.wrigglers[0] # assuming sorted
inGoal = self.checkGoal(wriggler.location[0]) or self.checkGoal(wriggler.location[2])
if(inGoal):
return True
return inGoal
# Returns true if coords are goal coords
def checkGoal(self, coord):
return coord.x == static.goal.x and coord.y == static.goal.y
def getWrigglerLocations(self):
wrigglerLocs = []
for wriggler in self.wrigglers:
wrigglerLocs.append(wriggler.location)
return wrigglerLocs
# Run normal heuristic
def a_star_normal(self):
return self.a_star(self.heuristic)
def a_star(self, heuristic):
node = self.tree.create_node(state=State(self.wrigglers), pathCost=0)
node.heuristic = heuristic(node)
frontier = PQDict()
stateFrontier = {}
explored = {}
# Sacrifice memory to have a huge speed up being able to instantly check for state in frontier
stateFrontier[str(node.state)] = node.heuristic
frontier.additem(node._identifier, node.heuristic)
while(True):
if(len(frontier) == 0):
return None
nodeID = frontier.popitem()[0]
node = self.tree.get_node(nodeID)
nodeStateStr = str(node.state)
del stateFrontier[nodeStateStr]
if self.testGoal(node.state):
return node
explored[nodeStateStr] = -1 # we don't care what the hash matches
actions = self.getActions(node.state)
for action in actions:
child = self.childNode(node, action)
child.heuristic = heuristic(child)
childStr = str(child.state)
inExplored = False
inFrontier = False
if childStr in explored:
inExplored = True
bGreater = False
if childStr in stateFrontier:
if(stateFrontier[childStr] < child.heuristic + child.pathCost):
bGreater = True
inFrontier = True
if(not inExplored and not inFrontier):
stateFrontier[childStr] = child.heuristic
frontier.additem(child._identifier, child.heuristic + child.pathCost)
elif(bGreater):
bHappened = False
for key in frontier:
if(str(self.tree.get_node(key).state) == childStr):
bHappened = True
frontier.pop(key)
frontier.additem(child._identifier, child.heuristic + child.pathCost)
break
assert bHappened
# Optimistic manhatten distance, but still terrible
def heuristic(self, node):
wriggler = node.state.wrigglers[0]
headDiffX = static.goal.x - wriggler.location[0].x
headDiffY = static.goal.y - wriggler.location[0].y
headDiff = headDiffX + headDiffY
tailDiffX = static.goal.x - wriggler.location[2].x
tailDiffY = static.goal.y - wriggler.location[2].y
tailDiff = tailDiffX + tailDiffY
return min(headDiff, tailDiff)
def euclidean_distance(self, coord0, coord1):
return math.sqrt((coord0.x - coord1.x) ** 2 + (coord0.y - coord1.y) ** 2)
def generateSolution(self, goalNode):
parentID = goalNode._bpointer
actions = []
text = ''
actions.append(goalNode.action)
while(parentID is not None):
parentNode = self.tree.get_node(parentID)
actions.append(parentNode.action)
parentID = parentNode._bpointer
actLength = 0
for action in reversed(actions):
if(action is not None):
if(action.headMoved):
headVal = '0'
else:
headVal = '1'
actLength += 1
text += str(action.wrigglerID) + " " + headVal + " " + str(action.movedCoord.x) + " " + str(action.movedCoord.y) + "\n"
return text, actLength
def getActions(self, state):
actions = []
wrigglers = state.wrigglers
self.clearMaze()
self.updateAllMaze(state.wrigglers)
for i in range(len(wrigglers)):
moves = wrigglers[i].getPossibleMoves()
for move in moves:
actions.append(Action(wrigglerID=i, movedCoord=move[0], headMoved=move[1]))
return actions
def updateAllMaze(self, wrigglers, bNot0=False):
if(bNot0):
for i in range(1, len(wrigglers)):
wrigglers[i].updateMaze()
else:
for wriggler in wrigglers:
wriggler.updateMaze()
def childNode(self, parent, action):
newState = State(parent.state.wrigglers, True)
wriggler = newState.wrigglers[action.wrigglerID]
if(action.headMoved):
symbol = wriggler.getSymbol(wriggler.location[0], action.movedCoord)
else:
symbol = wriggler.getSymbol(wriggler.location[2], action.movedCoord)
symbol = wriggler.convertToHeadSymbol(symbol)
wriggler.move(symbol, action.headMoved)
return self.tree.create_node(pathCost=parent.pathCost + 1, parent=parent._identifier, state=newState, action=action)
def getDepth(self, node):
parentID = node._bpointer
depth = 0
if(parentID is None):
return depth
while(parentID is not None):
depth += 1
parentNode = self.tree.get_node(parentID)
parentID = parentNode._bpointer
return depth
class WarcFileSystem( LoggingMixIn, Operations ):
"""Filesystem built on a WARC's URI paths."""
def __init__( self, warc ):
self.warc = warc
logger.debug( "Mounting %s" % self.warc )
self.fh = WarcRecord.open_archive( warc, gzip="auto", mode="rb" )
self.tree = Tree()
self._get_records()
def _get_records( self ):
"""Parses a WARC, building a hierarchical tree."""
statinfo = os.stat( self.warc )
self.gid = statinfo.st_gid
self.uid = statinfo.st_uid
self.tree.create_node( self.warc, "/" )
self.records = {}
bar = progressbar.ProgressBar( maxval=statinfo.st_size, widgets=[ progressbar.Bar( "=", "[", "]"), " ", progressbar.Percentage() ] )
bar.start()
for( offset, record, errors ) in self.fh.read_records( limit=None ):
if record is not None and record.type != WarcRecord.WARCINFO:
parent = "/"
segments = [ record.type ] + re.split( "/+", record.url )
for e in segments:
identifier = "/".join( [ parent, e ] )
if not self.tree.contains( identifier ):
node = WarcRecordNode( record, offset, tag=e, identifier=identifier )
self.tree.add_node( node, parent=parent )
parent = identifier
self.records[ record.url ] = ( offset, record )
bar.update( offset )
bar.finish()
logger.debug( self.tree.show() )
# def access( self, path, amode ):
# logger.debug( path )
# raise FuseOSError( EPERM )
def chmod( self, path, mode ):
raise FuseOSError( EPERM )
def chown( self, path, uid, gid ):
raise FuseOSError( EPERM )
def create( self, path, mode ):
raise FuseOSError( EPERM )
def destroy( self, path ):
self.fh.close()
# def flush( self, path, fh ):
# raise FuseOSError( EPERM )
def fsync( self, path, datasync, fh ):
raise FuseOSError( EPERM )
def fsyncdir( self, path, datasync, fh ):
raise FuseOSError( EPERM )
def getattr( self, path, fh=None ):
"""Returns stat info for a path in the tree."""
logger.debug( path )
if path == "/":
stat = os.stat( self.warc )
return dict( [
( "st_mode", ( S_IFDIR | 0444 ) ),
( "st_ino", stat.st_ino ),
( "st_dev", stat.st_dev ),
( "st_nlink", stat.st_nlink ),
( "st_uid", stat.st_uid ),
( "st_gid", stat.st_gid ),
( "st_size", stat.st_size ),
( "st_ctime", stat.st_ctime ),
( "st_mtime", stat.st_mtime ),
( "st_atime", stat.st_atime )
] )
else:
return self.name_to_attrs( "/%s" % path )
def getxattr( self, path, name, position=0 ):
"""Returns the value for an extended attribute."""
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
try:
return node.xattrs[ name ]
except KeyError:
raise FuseOSError( ENODATA )
def init( self, path ):
pass
def link( self, target, source ):
raise FuseOSError( EPERM )
def listxattr( self, path ):
"""Returns a list of extended attribute names."""
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
return node.xattrs.keys()
def mkdir( self, path, mode ):
raise FuseOSError( EPERM )
def mknod( self, path, mode, dev ):
raise FuseOSError( EPERM )
def open( self, path, flags ):
"""Should return numeric filehandle; returns file offset for convenience."""
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
return node.offset
# def opendir( self, path ):
# raise FuseOSError( EPERM )
def read( self, path, size, offset, fh ):
"""Reads 'size' data from 'path', starting at 'offset'."""
logger.debug( "read %s from %s at %s " % ( size, path, offset ) )
if path != "/":
path = "/%s" % path
node = self.tree.get_node( path )
if node is None:
raise FuseOSError( ENOENT )
offset += node.payload_offset
mime, data = node.record.content
end = offset + size
return data[ offset:end ]
def name_to_attrs( self, name ):
"""Retrieves attrs for a path name."""
logger.debug( name )
node = self.tree.get_node( name )
if node is None:
raise FuseOSError( ENOENT )
if node.is_leaf():
st_mode = ( S_IFREG | 0444 )
size = node.record.content_length
try:
timestamp = time.mktime( parse( node.record.date ).timetuple() )
except ValueError as v:
logger.warning( "Error parsing time: %s [%s]" % ( node.record.date, str( v ) ) )
timestamp = time.mktime( datetime.fromtimestamp( 0 ).timetuple() )
else:
st_mode = ( S_IFDIR | 0555 )
size = 0
timestamp = time.time()
return dict( [
( "st_mode", st_mode ),
( "st_ino", 0 ),
( "st_dev", 0 ),
( "st_nlink", 0 ),
( "st_uid", self.uid ),
( "st_gid", self.gid ),
( "st_size", size ),
( "st_ctime", timestamp ),
( "st_mtime", timestamp ),
( "st_atime", timestamp )
] )
def readdir( self, path, fh ):
"""Returns a tuple of all files in path."""
logger.debug( path )
if path != "/":
path = "/%s" % path
if self.tree.contains( path ):
names = []
for c in self.tree.get_node( path ).fpointer:
child = self.tree.get_node( c )
names.append( ( child.tag, self.name_to_attrs( child.identifier ), 0 ) )
return names
else:
raise FuseOSError( ENOENT )
def readlink( self, path ):
raise FuseOSError( EPERM )
# def release( self, path, fh ):
# raise FuseOSError( EPERM )
# def releasedir( self, path, fh ):
# raise FuseOSError( EPERM )
def removexattr( self, path, name ):
raise FuseOSError( EPERM )
def rename( self, old, new ):
raise FuseOSError( EPERM )
def rmdir( self, path ):
raise FuseOSError( EPERM )
def setxattr( self, path, name, value, options, position=0 ):
raise FuseOSError( EPERM )
def statfs( self, path ):
raise FuseOSError( EPERM )
def symlink( self, target, source ):
raise FuseOSError( EPERM )
def truncate( self, path, length, fh=None ):
raise FuseOSError( EPERM )
def unlink( self, path ):
raise FuseOSError( EPERM )
def utimens( self, path, times=None ):
raise FuseOSError( EPERM )
def write( self, path, data, offset, fh ):
raise FuseOSError( EPERM )
print nt_start
tree = Tree()
tree.create_node("S", nt_start)
tree.create_node("NP", nt_start+1, parent=nt_start)
tree.create_node("VP", nt_start+2, parent=nt_start)
tree.create_node("Jeg", s.index("Jeg"), parent=4)
tree.create_node("er", s.index("er"), parent=5)
tree.create_node("glad", s.index("glad"), parent=5)
tree.show()
searchword = "glad"
wid = s.index(searchword)
node = tree.get_node(wid)
print
entity = "er"
opinionw = "glad"
# Find lowest common ancestor
def get_LCA(root, n1, n2):
if root is None: return None
if root == n1 or root == n2: return root
class SentenceTree(object):
def __init__(self):
self.tree = Tree()
self.sentence = []
self.matrix = []
# Builds a tree by backtracking the sentence matrix
def build_tree(self, sentence_matrix):
self.matrix = sentence_matrix
sentence_length = len(sentence_matrix)-1
# Saves the ST's sentence as a list of strings
for i in range(1,sentence_length+1):
self.sentence.append(self.matrix[0][i][0])
# Finds the most probable sentence option
options = sentence_matrix[sentence_length][1]
max_option = [options[x] for x in options if options[x][1]==max([y[1] for y in options.values()])][0]
# Builds the tree
self._nid = sentence_length+1
root = max_option
self.tree.create_node(root[0], self._nid)
self._create_children(root, self._nid) # Call recursive function
# Ensures unique node id in _create_children()
def _nnid(self):
self._nid +=1
return self._nid
# Recursive function which builds the children nodes of a given parse_option and then builds their children
def _create_children(self, parse_option, pid):
if parse_option is None:
return None
else:
# If parse_option has children, extract those
if parse_option[2] is not None:
left_coord = parse_option[2]
right_coord = parse_option[3]
left_child = self.matrix[left_coord[0]][left_coord[1]][left_coord[2]]
right_child = self.matrix[right_coord[0]][right_coord[1]][right_coord[2]]
# Create left child as node (plus extra word node if leaf)
cid = self._nnid()
self.tree.create_node(left_child[0], cid, parent=pid)
if left_child[2] is None: #If left_child is a leaf node, append a word node
nid = left_coord[1]-1
word = self.matrix[left_coord[0]-1][left_coord[1]][0]
word = word.decode('utf-8', "ignore")
self.tree.create_node(word, nid, parent=cid)
else:
self._create_children(left_child, cid) # Create children of left_child
# Create right child as node (plus extra word node if leaf)
cid = self._nnid()
self.tree.create_node(right_child[0], cid, parent=pid)
if right_child[2] is None: #If right_child is a leaf node, append a word node
nid = right_coord[1]-1
word = self.matrix[right_coord[0]-1][right_coord[1]][0]
word = word.decode('utf-8', "ignore")
self.tree.create_node(word, nid, parent=cid)
else:
self._create_children(right_child, cid) # Create children of right_child
# Returns the sentence's sentiment score
def get_sentiment_score(self, sentimentDict, term):
total_score = 0
# negation dictionary
negationList = ["ikke", "ej"]
# Check the term against every sentiment word
n1 = self.sentence.index(term)
for word in sentimentDict:
if term==word: continue # If topic term is an opinion word, ignore.
n2 = self._in_sentence(word)
if n2 is not False:
d = self._get_distance(n1, n2)
if d == 0: score = float(sentimentDict[word])
score = float(sentimentDict[word]) / float(d)
# If SentWord is negated, flip the score derived from it
if self._is_negated(word, negationList):
score = score * -1
print "Term: %s | SentWord: %s | Distance: %s | Score: %s" % (term, word, d,score)
total_score += score
print "Total score:", total_score
return total_score
# Checks whether a word is within a specified threshold distance of a negation word
def _is_negated(self, w, negationList):
negationThreshold = 3
n1 = self._in_sentence(w)
if n1 is None: return False
for nw in negationList:
n2 = self._in_sentence(nw)
if n2 is not None:
if (self._get_distance(n1, n2)) < negationThreshold:
print "negating word", w
return True
return False
# Checks whether word w exists in the ST's sentence
def _in_sentence(self, w):
if w in self.sentence:
return self.sentence.index(w)
return False
# Returns distance between two nodes n1 and n2
def _get_distance(self, n1, n2):
LCA = self._get_LCA(self.tree.root, n1, n2)
distance = self.tree.depth(self.tree.get_node(n1)) + self.tree.depth(self.tree.get_node(n2))
distance = distance - 2 * self.tree.depth(self.tree.get_node(LCA))
return abs(distance)
# Returns lowest common ancestor of two nodes n1 and n2
# Supporting method of _get_distance()
def _get_LCA(self, current_node, n1, n2):
if current_node is None: return None
if current_node == n1 or current_node == n2: return current_node
if len(self.tree.get_node(current_node).fpointer) == 0: return None #if leaf, return None
if len(self.tree.get_node(current_node).fpointer) == 1: #if terminal node, check its single leaf node
return self._get_LCA(self.tree.get_node(current_node).fpointer[0], n1, n2)
if len(self.tree.get_node(current_node).fpointer) == 2:
left = self._get_LCA(self.tree.get_node(current_node).fpointer[0],n1,n2)
right = self._get_LCA(self.tree.get_node(current_node).fpointer[1],n1,n2)
if left is not None and right is not None: return current_node
if left is not None: return left
if right is not None:return right
return None
class DependencyReader:
"""DependencyReader object"""
def __init__(self):
self.tempDirectoryPath = mkdtemp(dir=".")
self.tree = Tree()
self.dependencies = {}
self.graphRelationships = []
def getPom(self, pomPath):
shutil.copy(pomPath, self.tempDirectoryPath)
os.chdir(self.tempDirectoryPath)
def getDependencies(self):
mavenTreeOutput = subprocess.Popen('mvn org.apache.maven.plugins:maven-dependency-plugin:RELEASE:tree -DoutputType=tgf', stdout=subprocess.PIPE, shell=True)
while True:
line = mavenTreeOutput.stdout.readline().rstrip()
if not line or re.search(r"BUILD SUCCESS", line):
break
match = re.match(r"\[INFO\]\s(\d*)\s*(.*):(.*):(\w+):([0-9\.]*)", line)
if match:
if not match.group(1) in self.dependencies.keys():
self.dependencies[match.group(1)] = DependencyNode(match.group(2), match.group(3), match.group(5), match.group(1))
if not self.tree.leaves():
self.tree.create_node(match.group(1), match.group(1), data=self.dependencies[match.group(1)])
self.dependencies[match.group(1)].get('jar', self.tempDirectoryPath)
match = re.match(r"\[INFO\]\s(\d*)\s(\d*)", line)
if match and match.group(2):
self.graphRelationships.append((match.group(1), match.group(2)))
def relateDependencies(self):
while self.graphRelationships:
for item in self.graphRelationships:
node = self.tree.get_node(item[0])
if node is not None:
parent = self.dependencies[item[0]]
child = self.dependencies[item[1]]
self.tree.create_node(child.referenceId, child.referenceId, parent=parent.referenceId, data=child)
self.graphRelationships.remove(item)
def scanDependencies(self):
# Need to run on each package with oneshot to get identifiers
# unless update dosocsv2 to create identifiers on scan
# or fix up dosocsv2 to create identifiers on scan instead
for node in self.tree.expand_tree(mode=Tree.DEPTH):
treeNode = self.tree.get_node(node)
subprocess.call('dosocs2 oneshot ' + treeNode.data.jarName, shell=True)
def createRelationships(self):
# Pass packages as relationships to new dosocsv2 command created
self.recursiveRelationship(self.tree.root)
def recursiveRelationship(self, parent):
for node in self.tree.is_branch(parent):
parentNode = self.tree.get_node(parent)
childNode = self.tree.get_node(node)
subprocess.call('dosocs2 packagerelate ' + parentNode.data.jarName + ' ' + childNode.data.jarName, shell=True)
self.recursiveRelationship(node)
def retrieve_dependencies(self, jarName):
if jarName is None:
root = self.tree.get_node(self.tree.root)
root = root.data.jarName
else:
root = jarName
tgfOutput = subprocess.Popen('dosocs2 dependencies ' + root, stdout=subprocess.PIPE, shell=True)
count = 0
tree = Tree()
dependencies = []
relationships = []
while True:
line = tgfOutput.stdout.readline()
if not line:
break
match = re.match(r"(\d+) - (.*)", line)
if match:
if count == 0:
count = count + 1
tree.create_node(match.group(2), match.group(1))
else:
dependencies.append((match.group(2), match.group(1)))
match = re.match(r"(\d+) (\d+)", line)
if match:
relationships.append((match.group(1), match.group(2)))
if not relationships:
print("No child relationships for " + jarName)
return None
while relationships:
for item in relationships:
node = tree.get_node(item[0])
if node is not None:
rel = [item for item in relationships if int(item[0]) == int(node.identifier)]
if rel is not None:
rel = rel[0]
dep = [item for item in dependencies if int(item[1]) == int(rel[1])]
if dep is not None:
dep = dep[0]
tree.create_node(dep[0], dep[1], parent=node.identifier)
relationships.remove(rel)
dependencies.remove(dep)
tree.show()
if jarName is None:
os.chdir(os.pardir)
def print_prob_val(tree):
n_tree = Tree(tree=tree)
for node in n_tree.nodes:
node = n_tree.get_node(node)
node.tag = "{tag} - {val} - {prob}".format(tag=node.tag, val=node.data[0], prob=node.data[1])
print n_tree
class SentenceTree(object): def __init__(self): self.tree = Tree() self.tree_logger = Logger() self.sentence = [] # Builds a tree by backtracking the sentence matrix def build_tree(self, sentence_matrix): self.tree_logger.start_timer() self.matrix = sentence_matrix sentence_length = len(sentence_matrix)-1 # Saves the ST's sentence as a list of strings for i in range(1,sentence_length+1): self.sentence.append(self.matrix[0][i][0]) # Check if the sentence resolves to a tree if len(sentence_matrix[sentence_length][1])==0: return None # Find the most probable sentence option maximum = 0 index = None for option in sentence_matrix[sentence_length][1]: if option.probability > maximum: maximum = option.probability index = sentence_matrix[sentence_length][1].index(option) if index is None: return None # Build the tree self._nid = sentence_length+2 root = sentence_matrix[sentence_length][1][index] self.tree.create_node(root.constituent, self._nid) self._create_children(root, self._nid) # Call recursive function self.tree_logger.stop_timer() # Ensures unique node id in _create_children() def _nnid(self): self._nid +=1 return self._nid # Recursive function which builds the children nodes of a given parse_option # and then builds their children def _create_children(self, parse_option, pid): if parse_option is None: return None else: # If parse_option has children, extract those if parse_option.left_coord is not None: left_child = self.matrix[parse_option.left_coord[0]][parse_option.left_coord[1]][parse_option.left_coord[2]] right_child = self.matrix[parse_option.right_coord[0]][parse_option.right_coord[1]][parse_option.right_coord[2]] # Create left child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(left_child.constituent, cid, parent=pid) if left_child.left_coord is None: #If left_child is a leaf node, append a word node nid = parse_option.left_coord[1]-1 word = self.matrix[parse_option.left_coord[0]-1][parse_option.left_coord[1]][0] self.tree.create_node(word, nid, parent=cid) else: self._create_children(left_child, cid) # Create children of left_child # Create right child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(right_child.constituent, cid, parent=pid) if right_child.right_coord is None: #If left_child is a leaf node, append a word node nid = parse_option.right_coord[1]-1 word = self.matrix[parse_option.right_coord[0]-1][parse_option.right_coord[1]][0] self.tree.create_node(word, nid, parent=cid) else: self._create_children(right_child, cid) # Create children of right_child # Returns the sentence's sentiment score def get_sentiment_score(self, sentimentDict, term): total_score = 0 # placeholder dictionaries -TESTING PURPOSES negationList = ["ikke", "liden"] # Check the term against every sentiment word n1 = self.sentence.index(term) for key in sentimentDict: n2 = self._in_sentence(key) if n2 is not False: d = self._get_distance(n1, n2) score = float(sentimentDict[key]) / float(d) # If SentWord is negated, flip the score derived from it if self._is_negated(key, negationList): score = score * -1 print ">>SENTIMENTSCORE: Term: %s | SentWord: %s | Distance: %s | Score: %s" % (term, key, d,score) total_score += score return total_score # Checks whether a word is within a specified threshold distance of a negation word def _is_negated(self, w, negationList): negationThreshold = 3 n1 = self._in_sentence(w) if n1 is None: return False for nw in negationList: n2 = self._in_sentence(nw) if n2 is not None: if (self._get_distance(n1, n2)) < negationThreshold: print "negating word", w return True return False # Checks whether word w exists in the ST's sentence def _in_sentence(self, w): if w in self.sentence: return self.sentence.index(w) return False # Returns distance between two nodes n1 and n2 def _get_distance(self, n1, n2): LCA = self._get_LCA(self.tree.root, n1, n2) distance = self.tree.depth(self.tree.get_node(n1)) + self.tree.depth(self.tree.get_node(n2)) distance = distance - 2 * self.tree.depth(self.tree.get_node(LCA)) return distance-2 # Returns lowest common ancestor of two nodes n1 and n2 # Supporting method of _get_distance() def _get_LCA(self, root, n1, n2): if root is None: return None if root == n1 or root == n2: return root if len(self.tree.get_node(root).fpointer) == 0: return None #if leaf, return None if len(self.tree.get_node(root).fpointer) == 1: #if terminal node, check its single leaf node return self._get_LCA(self.tree.get_node(root).fpointer[0], n1, n2) if len(self.tree.get_node(root).fpointer) == 2: left = self._get_LCA(self.tree.get_node(root).fpointer[0],n1,n2) right = self._get_LCA(self.tree.get_node(root).fpointer[1],n1,n2) if left is not None and right is not None: return root if left is not None: return left if right is not None:return right return None
class AcquisitionChainIter(object):
def __init__(self, acquisition_chain, parallel_prepare=True):
self.__sequence_index = -1
self._parallel_prepare = parallel_prepare
self.__acquisition_chain_ref = weakref.ref(acquisition_chain)
# set all slaves into master
for master in (x for x in acquisition_chain._tree.expand_tree() if isinstance(x, AcquisitionMaster)):
del master.slaves[:]
master.slaves.extend(
acquisition_chain._tree.get_node(master).fpointer)
# create iterators tree
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
device2iter = dict()
for dev in acquisition_chain._tree.expand_tree():
if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)):
continue
dev_node = acquisition_chain._tree.get_node(dev)
parent = device2iter.get(dev_node.bpointer, "root")
try:
it = iter(dev)
except TypeError:
one_shot = self.acquisition_chain._device2one_shot_flag.get(
dev, True)
dev_iter = DeviceIterator(dev, one_shot)
else:
dev_iter = DeviceIteratorWrapper(it)
device2iter[dev] = dev_iter
self._tree.create_node(
tag=dev.name, identifier=dev_iter, parent=parent)
@property
def acquisition_chain(self):
return self.__acquisition_chain_ref()
def prepare(self, scan, scan_info):
preset_tasks = list()
if self.__sequence_index == 0:
preset_tasks.extend([gevent.spawn(preset.prepare)
for preset in self.acquisition_chain._presets_list])
scan.prepare(scan_info, self.acquisition_chain._tree)
self._execute(
"_prepare", wait_between_levels=not self._parallel_prepare)
if self.__sequence_index == 0:
gevent.joinall(preset_tasks, raise_error=True)
def start(self):
if self.__sequence_index == 0:
preset_tasks = [gevent.spawn(
preset.start) for preset in self.acquisition_chain._presets_list]
gevent.joinall(preset_tasks, raise_error=True)
self._execute("_start")
def stop(self):
self._execute("stop", master_to_slave=True, wait_all_tasks=True)
preset_tasks = [gevent.spawn(preset.stop)
for preset in self.acquisition_chain._presets_list]
gevent.joinall(preset_tasks) # wait to call all stop on preset
gevent.joinall(preset_tasks, raise_error=True)
def next(self):
self.__sequence_index += 1
gevent.joinall([gevent.spawn(dev_iter.wait_ready) for dev_iter in self._tree.expand_tree()
if dev_iter is not 'root'],
raise_error=True)
try:
if self.__sequence_index:
for dev_iter in self._tree.expand_tree():
if dev_iter is 'root':
continue
dev_iter.next()
except StopIteration: # should we stop all devices?
for acq_dev_iter in (x for x in self._tree.expand_tree() if x is not 'root' and
isinstance(x.device, (AcquisitionDevice, AcquisitionMaster))):
if hasattr(acq_dev_iter, 'wait_reading'):
acq_dev_iter.wait_reading()
dispatcher.send("end", acq_dev_iter.device)
raise
return self
def _execute(self, func_name,
master_to_slave=False, wait_between_levels=True,
wait_all_tasks=False):
tasks = list()
prev_level = None
if master_to_slave:
devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]
else:
devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:])
for dev in devs:
node = self._tree.get_node(dev)
level = self._tree.depth(node)
if wait_between_levels and prev_level != level:
gevent.joinall(tasks, raise_error=True)
tasks = list()
prev_level = level
func = getattr(dev, func_name)
tasks.append(gevent.spawn(func))
# ensure that all tasks are executed
# (i.e: don't raise the first exception on stop)
if wait_all_tasks:
gevent.joinall(tasks)
gevent.joinall(tasks, raise_error=True)
class BrancherWidget(QWidget):
def __init__(self):
super(BrancherWidget, self).__init__()
self.initUI()
self.tree = None
def initUI(self):
self.l_w = QWidget(self)
self.r_u = QWidget(self)
self.r_l = QWidget(self)
self.r_s = QSplitter(Qt.Vertical, self)
self.l_grid = QGridLayout()
self.m_grid = QHBoxLayout()
self.splitter = QSplitter(self)
self.tree_view = QTreeWidget()
self.root = None
self.connect(self.tree_view, SIGNAL('itemSelectionChanged()'), self.item_changed)
self.add_btn = QPushButton('Add')
self.fin_btn = QPushButton('Finish')
self.clr_btn = QPushButton('Clear')
self.add_btn.clicked.connect(self.add_card)
self.fin_btn.clicked.connect(self.fin_card)
self.clr_btn.clicked.connect(self.clr_card)
self.l_grid.addWidget(self.tree_view, 1, 0)
self.l_grid.addWidget(self.add_btn, 2, 0)
self.l_grid.addWidget(self.fin_btn, 3, 0)
self.l_grid.addWidget(self.clr_btn, 3, 1)
self.l_w.setLayout(self.l_grid)
self.r_grid_u = QGridLayout()
w_label = QLabel(u'言葉:')
i_label = QLabel(u'意味:')
s_label = QLabel(u'例文:')
self.w_field = QLabel('')
self.i_field = QLabel('')
self.i_field.setTextInteractionFlags(Qt.TextSelectableByMouse)
self.s_field = QLabel('')
self.s_field.setTextInteractionFlags(Qt.TextSelectableByMouse)
self.r_grid_u.setSpacing(10)
self.r_grid_u.addWidget(w_label, 1, 0)
self.r_grid_u.addWidget(self.w_field, 1, 1)
self.r_grid_u.addWidget(i_label, 2, 0)
self.r_grid_u.addWidget(self.i_field, 2, 1)
self.r_grid_u.addWidget(s_label, 3, 0)
self.r_grid_u.addWidget(self.s_field, 3, 1)
self.r_u.setLayout(self.r_grid_u)
self.r_grid_l = QGridLayout()
w_label = QLabel(u'言葉:')
i_label = QLabel(u'意味:')
s_label = QLabel(u'例文:')
max_size = s_label.sizeHint().height() * 3
self.w_entry = QLineEdit('')
self.i_entry = QTextEdit('')
self.s_entry = QTextEdit('')
self.i_entry.setMaximumHeight(max_size)
self.s_entry.setMaximumHeight(max_size)
self.connect(self.w_entry, SIGNAL('textChanged()'), self.update_fields)
self.connect(self.i_entry, SIGNAL('textChanged()'), self.update_fields)
self.connect(self.s_entry, SIGNAL('textChanged()'), self.update_fields)
self.r_grid_l.setSpacing(10)
self.r_grid_l.addWidget(w_label, 1, 0)
self.r_grid_l.addWidget(self.w_entry, 1, 1)
self.r_grid_l.addWidget(i_label, 2, 0)
self.r_grid_l.addWidget(self.i_entry, 2, 1)
self.r_grid_l.addWidget(s_label, 3, 0)
self.r_grid_l.addWidget(self.s_entry, 3, 1)
self.r_l.setLayout(self.r_grid_l)
self.r_s.addWidget(self.r_u)
self.r_s.addWidget(self.r_l)
self.splitter.addWidget(self.l_w)
self.splitter.addWidget(self.r_s)
self.m_grid.addWidget(self.splitter)
self.setLayout(self.m_grid)
self.setGeometry(300, 300, 800, 500)
self.setWindowTitle('Review')
def fin_card(self, arg):
words = [i for i in self.tree.expand_tree(mode=Tree.DEPTH)][::-1]
for i in words:
data = self.tree[i].data
res = {u'Keyword':self.tree[i].tag, u'Expression':data[1], u'Definition':data[0]}
self.anki.addNote('Brancher', 'BrancherJapanese', res)
self.clr_card(arg)
def clr_card(self, arg):
if self.root is not None:
self.tree_view.invisibleRootItem().removeChild(self.root)
self.root = None
self.tree = None
def add_card(self, arg):
text, ok = QInputDialog.getText(self, 'Input Dialog',
'Unknown Word:')
if ok:
self.add_word(text)
def add_word(self, text):
search_res = Goo.search(text)
ls = ListSelection(search_res[1], "Select the correct word entry", self)
if ls.exec_() == QDialog.Accepted:
w_idx = ls.get_value()[0]
text_res = search_res[1][int(w_idx)].split(u'【')
if len(text_res) == 2:
text = text_res[1][:-1]
definitions = Goo.get_definition(text, link=search_res[0][int(w_idx)])
ls = ListSelection(definitions, "Select the desired definition", self, True)
if ls.exec_() == QDialog.Accepted:
#d_idx = ls.get_value()[0]
res = ls.get_value()
print 'Res', len(res)
for d_idx in res:
definition = definitions[int(d_idx)]
print d_idx
print definition
sentence = u''
#sen_res = Tatoeba.search(text)
#ls = ListSelection(sen_res, "Select a sample sentence", self)
#if ls.exec_() == QDialog.Accepted:
# s_idx = ls.get_value()[0]
# sentence = sen_res[int(s_idx)]
self.add_word_with_definition(text, definition, sentence)
def add_word_with_definition(self, text, definition, sentence):
if self.tree is None:
self.next_id = 0
text = str(self.next_id) + '.' + text
self.next_id += 1
node = QTreeWidgetItem(self.tree_view)
node.setText(0, text)
self.tree = Tree()
text = node.text(0)
self.tree.create_node(text, text, data=(definition, sentence))
self.root = node
node.setSelected(True)
else:
sel = self.tree_view.selectedItems()
if len(sel) == 1:
text = str(self.next_id) + '.' + text
self.next_id += 1
sel = sel[0]
node = QTreeWidgetItem(sel)
node.setText(0, text)
text = node.text(0)
self.tree.create_node(text, text, parent=sel.text(0),
data=(definition,sentence))
sel.setExpanded(True)
def item_changed(self):
print self.tree_view.selectedItems()
for i in self.tree_view.selectedItems():
self.w_field.setText('.'.join(i.text(0).split('.')[1:]))
self.w_entry.setText('.'.join(i.text(0).split('.')[1:]))
imi, reibun = self.tree.get_node(i.text(0)).data
self.i_field.setText(imi)
self.i_entry.setText(imi)
self.s_field.setText(reibun)
self.s_entry.setText(reibun)
def update_fields(self):
if len(self.tree_view.selectedItems()) != 1:
return
sel = self.tree_view.selectedItems()[0]
node = self.tree.get_node(sel.text(0))
node.data = (self.i_entry.toPlainText(), self.s_entry.toPlainText())
self.i_field.setText(node.data[0])
self.s_field.setText(node.data[1])
def keyPressEvent(self, event):
key = event.key()
if key == Qt.Key_B:
if self.i_field.hasSelectedText():
self.add_word(self.i_field.selectedText())
if self.s_field.hasSelectedText():
self.add_word(self.s_field.selectedText())
class DependencyGraph:
def __init__(self, module, db='openerp', server='http://localhost:8069', user='******', password='******'):
"""
Get a ERPPeek client, check the module we want to generate the graph for is in the database and generate the
hierarchy ready for formatting
:param module: Name of the module we want to generate the hierarchy for
:param db: Database to generate the hierarchy from
:param server: Odoo server address
:param user: Username - defaults to admin so we can access info on modules
:param password: Password for user we're connecting as
:return: An hierarchy object for the module specified
"""
self.client = get_erppeek_client(server=server, db=db, user=user, password=password)
self.mod_reg = 'ir.module.module'
self.dep_reg = 'ir.module.module.dependency'
# check that the module in question exists
module_present = self.module_search(module)
if not module_present:
raise RuntimeError("{m} module not found in database".format(m=module))
else:
self.hierarchy = Tree()
self.hierarchy.create_node(module, module)
self.get_hierarchy_for_module(module)
def get_hierarchy_for_module(self, module, parent=None):
# Check that module isn't already in hierarchy
installed = self.module_search(module)
mod_id = module
if installed:
if parent:
if self.hierarchy.get_node(module):
unique_id = uuid.uuid1()
mod_id = '{0}_{1}'.format(module, unique_id)
self.hierarchy.create_node(module, mod_id, parent=parent)
deps = self.get_dependencies_for_module(module)
for mod in deps:
self.get_hierarchy_for_module(mod, parent=mod_id)
def get_dependencies_for_module(self, module):
"""
Recursively get the hierarchy for a module
:param module: The module to get the hierarchy for
:return: Runs itself again with the new module or returns True
"""
# Search for modules that depend on the module
dependent_mod_ids = self.dependency_search(module)
if not dependent_mod_ids:
return []
dependent_mod_names = self.dependency_read(dependent_mod_ids)
return dependent_mod_names
def dependency_read(self, ids):
"""
Read ir.module.module.dependency for the names of the modules with the supplied IDs
:param ids: List of IDs for modules supplied
:return: List of names
"""
deps = self.client.read(self.dep_reg, ids, ['module_id'])
mod_ids = [dep['module_id'][0] for dep in deps]
mods = self.client.read(self.mod_reg, mod_ids, ['name'])
return [mod['name'] for mod in mods]
def dependency_search(self, module):
"""
Search ir.module.module.dependency for the IDs of modules that depend on the module
:param module: Name of the module to look for depending modules
:return: List of IDs
"""
return self.client.search(self.dep_reg, [['state', '=', 'installed'], ['name', '=', module]])
def module_search(self, module):
"""
Search ir.module.module for the IDs of module
:param module: Name of the module to look for
:return: List of IDs
"""
return self.client.search(self.mod_reg, [['state', '=', 'installed'], ['name', '=', module]])
class IIIF_Collection():
'''
Class for working with IIIF Collections.
iiif_recurse works through a collection and grabs all of the manifests
recursing down through any collections within collections.
'''
def __init__(self, uri):
self.uri = uri
self.master_manifest_list = []
self.tree = Tree()
self.iiif_recurse(self.uri)
def iiif_recurse(self, resource_uri, parent_id=None):
try:
source_data = json.loads(IIIF_Item(resource_uri).source_data)
# print source_data
if parent_id:
escaped_id = '/'.join([parent_id,
urllib.quote_plus(resource_uri)])
if not self.tree.get_node(escaped_id):
self.tree.create_node(
resource_uri,
escaped_id,
parent=parent_id)
else:
escaped_id = urllib.quote_plus(resource_uri)
if not self.tree.get_node(escaped_id):
self.tree.create_node(
resource_uri,
escaped_id)
if 'collections' in source_data:
collection_lists = get_recursively(json.loads(
IIIF_Item(resource_uri).source_data), 'collections')
for collection_list in collection_lists:
for item in collection_list:
# print item
item_id = item['@id']
escaped_item_id = '/'.join([escaped_id,
urllib.quote_plus(item_id)])
# print escaped_item_id
if not self.tree.get_node(escaped_item_id):
# print 'Creating a collection node'
self.tree.create_node(
item_id, escaped_item_id,
parent=escaped_id)
if 'manifests' in item:
for manifest in item['manifests']:
escaped_manifest_id = '/'.join(
[escaped_item_id, urllib.quote_plus(manifest['@id'])])
# print escaped_manifest_id
if not self.tree.get_node(escaped_manifest_id):
self.tree.create_node(
manifest[
'@id'], escaped_manifest_id,
parent=escaped_item_id)
# print self.tree.get_node(escaped_manifest_id).identifier
else:
# print 'getting the collection: %s' % item_id
self.iiif_recurse(item_id, parent_id=escaped_id)
if 'manifests' in source_data:
# print 'This bit it should be getting manifest'
for manifest in source_data['manifests']:
# print manifest['@id']
escaped_manifest_id = '/'.join(
[escaped_id, urllib.quote_plus(manifest['@id'])])
if not self.tree.get_node(escaped_manifest_id):
self.tree.create_node(
manifest['@id'],
escaped_manifest_id,
parent=escaped_id)
except ValueError:
print 'Could not load that collection as\
class verifier(object):
'''
This verifier object is intended to receive an Enfragmo instance file,
and then to determine the original formula corresponding to that file.
Each subformula which is an atom will be given as a lower-case letter,
and nesting of formulas will be indicated by appropriate bracketing.
The symbols used to represent operators are as follows:
And &
Or v
Not ~
Implication ->
Box box
Diamond dia
With unary operators applied directly to atoms being dropped.
The format of an instance file is assumed to be as follows:
TYPE Subformula [ 1.. n]
TYPE World [1..m]
PREDICATE Atom
...
PREDICATE And
...
PREDICATE Or
...
PREDICATE Not
...
PREDICATE Implication
...
PREDICATE Box
...
PREDICATE Diamond
...
PREDICATE SameAtom
Where "..." indicates that either singletons, pairs, or triples will occupy
the lines below the current PREDICATE delimiter and the next, signifying
the appropriate relationship between the subformulas. For example,
PREDICATE And
(1, 2, 3)
Indicates that the main connective of subformula 1 is conjunction, and that
subformula 2 and 3 are the two operands for that operator.
'''
def __init__(self, filename):
'''
Receives the name of the instance file to be verified, then uses this
to initialize the corresponding tree structure
'''
self.SameAtomList = unionfind.UnionFind()
self.filename = filename
def readProblemInstanceFile(self):
'''
This method assumes that the instance file exists and is correctly
formatted.
Subformulas are labeled in pre-order DFS traversal fashion, so as to
allow the numbering to reflect the operator/operand relationship.
'''
self.instanceFileLines = [line.strip() for line in open(self.filename) if line != '\n']
def parseProblemInstanceFile(self):
self.countNumTreeNodes()
#self.countNumTreeLeaves()
#self.countNumAtoms()
self.setUpSameAtomList()
self.buildTree()
def numWorlds(self):
return int(self.instanceFileLines[1][-2])
def countNumTreeNodes(self):
'''
The number of tree nodes is inherent in the number of subformulas,
which is given on the first line of a well-formed problem instance
file. In this way, the given formula is considered to be the first
subformula, where its main connective labels the root node, and its
'''
self.numTreeNodes = int(self.instanceFileLines[0].split(" ")[-1].split("]")[0])
def countNumTreeLeaves(self):
'''
The number of tree leaves is simply the number of singletons that
satisfy Atom, including duplicates.
'''
self.numTreeLeaves = \
int((self.instanceFileLines.index("PREDICATE SameAtom") - 1)- \
self.instanceFileLines.index("PREDICATE Atom"))
def countNumAtoms(self):
'''
Since multiple subformulas can refer to the same atom, need to subtract
the count of pairs satisfying SameAtom from the total count of
subformulas satisfying Atom. This can be done by counting the number of
entries between each of the appropriate PREDICATE identifiers (given a
well-formed instance file).
'''
self.numAtoms = int((self.instanceFileLines.index("PREDICATE And")- \
(self.instanceFileLines.index("PREDICATE Atom")+1) - \
(len(self.instanceFileLines) - \
(self.instanceFileLines.index("PREDICATE SameAtom")+1))))
return self.numAtoms
def assignSymbol(self, label):
if label == "And":
return "&"
elif label == "Or":
return "v"
elif label == "Not":
return "~"
elif label == "Implication":
return "->"
elif label == "Biconditional":
return "<->"
elif label == "Box":
return "box"
elif label == "Diamond":
return "dia"
def assignAtom(self, i):
for atomEquivClass in self.SameAtomList.get_sets():
if str(i) in atomEquivClass:
return self.SameAtomList.get_leader(str(i))
self.SameAtomList.insert(str(i))
return self.SameAtomList.get_leader(str(i))
def setUpSameAtomList(self):
'''
Using the Union Find datastructure, I will keep track of the equivalence
classes of SameAtoms and then supply a label based on the index of the
subset in which a subformula corresponding with an atom is contained.
'''
startIndexSameAtoms = findInFile(self.instanceFileLines, lambda x: "PREDICATE SameAtom" in x) + 1
sameAtomPairs = self.instanceFileLines[startIndexSameAtoms:]
for pair in sameAtomPairs:
tmp = pair.split(",")
label1 = tmp[0].split("(")[1]
label2 = tmp[1].split(")")[0]
self.SameAtomList.insert(label1, label2)
def determineConnective(self, i):
'''
Each subformula label is guaranteed to be the first argument of some
tuple; either it will correspond with an atom, the only argument,
or it will correspond with the main connective of a unary
(i.e. negation, box, or diamond) or binary (i.e. conjunction or
disjunction) subformula.
'''
SiThing=findInFile(self.instanceFileLines, lambda x: "("+str(i)+")" in x)
if SiThing:
for k in range(SiThing, 0, -1): # go back in the file until you can find out what predicate we're dealing with
if self.instanceFileLines[k].split(" ")[0] == "PREDICATE":
if self.instanceFileLines[k].split(" ")[1] == "Falsum":
return "false"
return self.assignAtom(i)
if findInFile(self.instanceFileLines, lambda x: "("+str(i)+"," in x):
SiAsMainConnective = findInFile(self.instanceFileLines, lambda x: "("+str(i)+"," in x)
for j in range(SiAsMainConnective, 0, -1): # go back in the file until you can find out what predicate we're dealing with
if self.instanceFileLines[j].split(" ")[0] == "PREDICATE":
if self.instanceFileLines[j].split(" ")[1] == "SameAtom": # if there are no tuples under SameAtom, then this isn't reached due to SiAsMainConnective
return self.assignAtom(i)
else:
return self.assignSymbol(self.instanceFileLines[j].split(" ")[1]) # if the predicate refers to an operator, then we need to find out which one!
def nodeCreation(self, predicate, SiConnective, i):
SiAsOperand = findInFile(self.instanceFileLines, predicate)
ParentOfSi = str(self.instanceFileLines[SiAsOperand].split(",")[0].split("(")[1])
self.syntaxTree.create_node(SiConnective, str(i), parent=ParentOfSi)
def makeSyntaxTreeNode(self, SiConnective, i):
if findInFile(self.instanceFileLines, lambda x: ","+str(i)+"," in x): #find where subformula i appears as second operand, and
self.nodeCreation(lambda x: ","+str(i)+"," in x, SiConnective, i)
elif findInFile(self.instanceFileLines, lambda x: ","+str(i)+")" in x):
self.nodeCreation(lambda x: ","+str(i)+")" in x, SiConnective, i)
else:
self.syntaxTree.create_node(SiConnective,str(i))
def buildTree(self):
'''
To build the syntax tree for the formula as laid out in the instance
file, we need to delve into the formula by means of stripping off the
main connective of each subformula (starting with the main connective
of the formula itself) and labeling a tree node with the symbol
corresponding with that connective. Note that each subformula appears
exactly once as the first argument of a tuple, and can appear at most
once as a second (or third, for binary operators) argument in a tuple.
'''
self.syntaxTree = Tree()
for i in range(1, self.numTreeNodes+1):
SiConnective = self.determineConnective(i)
self.makeSyntaxTreeNode(SiConnective, i)
def myShowTree(self, tree, root):
'''
In-order depth-first traversal of syntax tree using deep recursion; first
layer of recursion receives root of tree, where each sub-layer receives
respectively the left child then the right child as roots of those subtrees
with visitation of the root node occurring in the middle.
'''
rootNID = root.identifier
x=self.syntaxTree.children(rootNID)
if len(self.syntaxTree.children(rootNID)) == 2:
print("(", end=" ")
self.myShowTree(self.syntaxTree, self.syntaxTree.children(rootNID)[0])
print(self.syntaxTree.get_node(rootNID).tag, end=" ")
if len(self.syntaxTree.children(rootNID)) >= 1:
if len(self.syntaxTree.children(rootNID)) == 1:
print("(", end=" ")
self.myShowTree(self.syntaxTree, self.syntaxTree.children(rootNID)[0])
else:
self.myShowTree(self.syntaxTree, self.syntaxTree.children(rootNID)[1])
print(")", end=" ")
def iiif_recurse(uri, tr=None, parent_nid=None, separator='/'):
'''
Treelib nodes have a tag (human readable), and a nid (node id).
Sanitised uris are used in the nids, throughout, concatenated
with the parent id, with a separator, so the path segments can
be recreated.
Comments: Nid and denid adds an extra level of list handling, as it joins
and the splits. Might be better to create the path (no processing)
and then create the nid.
'''
try:
obj = IIIF_Object(uri)
if not tr:
tr = Tree()
if parent_nid:
root_nid = separator.join([parent_nid, sanitise_uri(uri)])
if not tr.get_node(root_nid):
tr.create_node(uri, root_nid, parent=parent_nid, data=obj.data)
# function here to pass to exteral db
else:
root_nid = sanitise_uri(uri)
tr.create_node(uri, root_nid, data=obj.data)
# function here to pass to exteral db
# is it recursing too much here?
if obj.source_dict:
print 'Parsing: %s' % obj.identifier
if 'manifests' in obj.source_dict:
for manifest in obj.source_dict['manifests']:
manifest_id = sanitise_uri(manifest['@id'])
manifest_nid = separator.join([root_nid, manifest_id])
manifest_data = None
manifest_data = base_data(manifest)
manifest_data['path'] = de_nid(manifest_nid, separator)
if not tr.get_node(manifest_nid):
tr.create_node(
manifest['@id'], manifest_nid,
parent=root_nid, data=manifest_data)
# function here to pass to exteral db
if 'collections' in obj.source_dict:
collection_lists = get_recursively(
obj.source_dict, 'collections')
for collection_list in collection_lists:
for item in collection_list:
coll_id = sanitise_uri(item['@id'])
coll_nid = separator.join([root_nid, coll_id])
coll_data = None
coll_data = base_data(item)
coll_data['path'] = de_nid(coll_nid, separator)
if not tr.get_node(coll_nid):
tr.create_node(
item['@id'],
coll_nid,
parent=root_nid,
data=coll_data
)
# function here to pass to exteral db
if 'manifests' in item:
for manifest in item['manifests']:
manifest_id = sanitise_uri(manifest['@id'])
manifest_nid = separator.join(
[coll_nid, manifest_id]
)
manifest_data = None
manifest_data = base_data(manifest)
manifest_data['path'] = de_nid(
manifest_nid, separator
)
if not tr.get_node(manifest_nid):
tr.create_node(
manifest['@id'],
manifest_nid,
parent=coll_nid,
data=manifest_data
)
# function here to pass to exteral db
else:
iiif_recurse(item['@id'], tr, root_nid)
except:
pass
return tr
for i in range(0,len(etiquetas)-1):
entropias.append(calEntropia(contenido,i))
menorColumna=buscarMenor(entropias)
tree=Tree()
tree.create_node({"Etiqueta":etiquetas[menorColumna] , "Camino": "Raiz", "CamElg":0},0)
historicoContenido = []
historicoEtiquetas =[]
hijo=tree.get_node(0)
histFunc=[]
histFunc.append(contenido)
histHij=[]
histHij.append(hijo)
hisEtiq=[]
hisEtiq.append(etiquetas)
hijosRaiz=0
ide=1
print UnaPos(contenido,"0",3)
while(hijosRaiz<=len(valoresResultados)):
aaa = raw_input("")
print ("------------------")
tree.show()
if(len(hijo._fpointer)<valoresResultados and len(contenido)>2):
class RST_DT:
def load(self, path2file):
self.id_EDUs = []
self.EDU = {}
self.treeNS = Tree()
self.tree = Tree()
# nombre max d'espace pour init id_parents
with open(path2file, "r") as f:
max_space = 0
nb_line = 0
for i, line in enumerate(f):
nb_space = 0
for c in line:
if c == " ":
nb_space += 1
else:
break
if nb_space > max_space:
max_space = nb_space
nb_line += 1
with open(path2file, "r") as f:
id_parents = [0] * max_space
NS_parents = [0] * max_space
for i, line in enumerate(f):
# nombre d'espace détermine le parent
nb_space = 0
for c in line:
if c == " ":
nb_space += 1
else:
break
space = nb_space / 2
id_parents[space] = i
parent = id_parents[space - 1]
reg = "\(([\w\-\[\]]+)|(_!.+!_)" # récupération du contenu
match = re.findall(reg, line)[0]
if match[0] == "":
content = match[1] # feuille EDU
self.id_EDUs.append(i)
# print content
self.EDU[i] = re.findall("_!(.*)!_", content)
else:
content = match[0]
reg2 = "\[(N|S)\]" # récupération NS
match2 = re.findall(reg2, content)
NS_parents[space] = match2 # ['N','S']
# création du noeud
if i == 0:
self.tree.create_node(content, 0)
self.treeNS.create_node("Root", 0)
else:
id_NS = len(self.tree.is_branch(parent)) # 0 ou 1 car arbre binaire
self.tree.create_node(content, i, parent=parent)
self.treeNS.create_node(NS_parents[space - 1][id_NS], i, parent=parent)
def toDEP(self):
###############################
# Etape 1 : construction du head_tree
# parcours en largeur de tree afin de récupérer chaque id_node
# pour chaque profondeur (init à 0) _! sans compter !_ les feuilles (EDUs)
nodes_depth = [-1] * self.tree.size()
for i in xrange(self.tree.size()):
id_nodes = [0]
depth = [999] * self.tree.size()
while id_nodes: # False if empty
id_node = id_nodes.pop(0)
node = self.tree.get_node(id_node)
if node.bpointer != None:
node_parent = self.tree.get_node(node.bpointer)
depth[node.identifier] = depth[node_parent.identifier] + 1
else:
depth[node.identifier] = 0
if id_node == i:
# print 'noeud ',i,' en profondeur', depth[node.identifier]
if node.fpointer:
nodes_depth[i] = depth[i]
break
if node.fpointer:
id_nodes.append(node.fpointer[0])
id_nodes.append(node.fpointer[1])
# print nodes_depth
id_nodes_depth = []
for d in xrange(self.tree.depth()):
id_nodes_depth.append([])
for i in xrange(self.tree.size()):
if nodes_depth[i] == d:
id_nodes_depth[d].append(i)
# print id_nodes_depth
#
# construction du head_tree
head_tree = [-1] * self.treeNS.size()
# pour chaque noeud (non EDU/feuille) en partant de la plus grande profondeur dans l'arbre
for d in range(len(id_nodes_depth) - 1, -1, -1):
for id_node in id_nodes_depth[d]:
node = self.treeNS.get_node(id_node)
node_left = self.treeNS.get_node(node.fpointer[0])
node_right = self.treeNS.get_node(node.fpointer[1])
if node_left.tag == "N":
if head_tree[node_left.identifier] == -1:
identifier = node_left.identifier
else:
identifier = head_tree[node_left.identifier]
else:
if head_tree[node_right.identifier] == -1:
identifier = node_right.identifier
else:
identifier = head_tree[node_right.identifier]
head_tree[id_node] = identifier
# print head_tree
###############################
# Etape 2 : construction du DEP
#
# construction du DEP
# init
# root est le premier noeud de head
# pour chaque EDU son père est le root dans DEP
dep_tree = Tree()
id_root = head_tree[0]
root = self.tree.get_node(id_root)
# dep_tree.create_node(root.tag, root.identifier)
dep_tree.create_node(root.tag, root.identifier)
for id_EDU in xrange(len(head_tree)):
if head_tree[id_EDU] == -1 and id_EDU != id_root:
node = self.tree.get_node(id_EDU)
# dep_tree.create_node(node.tag, node.identifier, parent=id_root)
# dep_tree.create_node(str(id_EDU), node.identifier, parent=id_root)
dep_tree.create_node(node.tag, node.identifier, parent=id_root)
# print '//////////////////////'
# print 'EDU', id_root
# pour chaque EDU
for id_EDU in xrange(len(head_tree)):
if head_tree[id_EDU] == -1 and id_EDU != id_root:
EDU_NS = self.treeNS.get_node(id_EDU)
# print '.......................'
# print 'EDU', id_EDU
# print 'TAG', EDU_NS.tag
if EDU_NS.tag == "N":
# parcours en largeur jusqu'à trouver un S avec un head donc qui soit pas EDU
id_nodes = [EDU_NS.identifier]
visited = [False] * self.treeNS.size()
while id_nodes:
id_node = id_nodes.pop(0)
EDU = self.tree.get_node(id_node)
# print 'visited EDU', EDU.identifier
visited[EDU.identifier] = True
# cas d'arret
head_EDU = head_tree[EDU.identifier] == -1
head_EDU = False
node_tag = self.treeNS.get_node(EDU.identifier).tag
# print ' head_EDU', head_EDU
# print ' node_tag', node_tag
if not head_EDU and node_tag == "S":
break
if EDU.bpointer:
if not visited[EDU.bpointer]:
id_nodes.append(EDU.bpointer)
if EDU.fpointer: # sécurité
if not visited[EDU.fpointer[0]]:
id_nodes.append(EDU.fpointer[0])
if not visited[EDU.fpointer[1]]:
id_nodes.append(EDU.fpointer[1])
# puis ajouter au DEP comme enfant du head du parent du noeud S
id_head = head_tree[EDU.bpointer]
# si parent S
else:
# parcours en largeur des ancêtre jusqu'à trouver un ancêtre avec un head
parent = self.treeNS.get_node(EDU_NS.bpointer)
id_head = head_tree[parent.identifier]
# puis ajouter au DEP comme enfant de ce head
if id_EDU != id_head:
dep_tree.move_node(id_EDU, id_head)
EDU = self.tree.get_node(id_EDU)
# print '---- ajout de',EDU.identifier,' à',id_head
# if id_EDU == id_head:
# dep_tree.show()
return dep_tree
# showDepth(dep_tree, 4)
# dep_tree.show()
# node = dep_tree.
def toString(self):
""" affiche comme la sortie de Hilda """
showDepth(self.tree, 0)
class WikipediaResolver(object):
""" WikipediaResolver class
"""
_base_wiki_url = 'https://pl.wikipedia.org'
_base_wiki_url_search = _base_wiki_url + '/wiki/'
def __init__(self, start_from, finish_at):
"""
:param start_from: Wikipedia entry like "Chrześcijaństwo"
:param finish_at: Wikipedia entry like "Biblia"
:return: WikipediaResolver instance
"""
self._tree = Tree()
self._finish_at_url = requests.get(self._base_wiki_url_search + finish_at).url
self._to_check_list = [(None, self._base_wiki_url_search + start_from)]
self._known_resources = set()
def solve(self):
"""
:return: returns ordered list (start point first) of found links (steps)
"""
while True:
found_node = self._process_level()
if found_node:
break
# logging.debug("Formating tree... This may take a while.")
# self._show_tree()
path = self._backtrack_path()
return path
def _process_level(self):
"""
:return: Bool. True if target link was found in dispached breadth graph search iteration, else False
"""
to_check_list = []
while self._to_check_list:
parent_url, url = self._to_check_list.pop(0)
self._known_resources.add(url)
if not parent_url:
self._tree.create_node(url, url) # this is root node
set_to_check = set((self._base_wiki_url + a.attr('href') for a in self._get_a_items(requests.get(url).text) if a.attr('href')))
set_to_check -= self._known_resources
map((lambda _url: self._tree.create_node(_url, _url, parent=url)), set_to_check)
if self._finish_at_url in set_to_check:
return True
to_check_list += [(url, _url) for _url in set_to_check]
self._known_resources.update(set_to_check)
self._to_check_list += to_check_list
def _show_tree(self):
""" This simply print current state of the tree
:return: Nothing
"""
self._tree.show()
def _backtrack_path(self):
""" Backtracks found link among the tree.
:return: returns ordered list (start point first) of found links (steps)
"""
node = self._tree.get_node(self._finish_at_url)
nodes = [node]
while node.bpointer:
nodes.append(self._tree.get_node(node.bpointer))
node = self._tree.get_node(node.bpointer)
return list(reversed([node.identifier for node in nodes]))
@staticmethod
def _get_a_items(html):
dom = PQ(html) # this parses html into tree
return dom.find('a[href^="/wiki/"]:not([href*=":"])').items() # Neat jquery style selectors. #PDK
