def build_tree(self):
print("Building product tree for " + str(len(self.data)) + " products")
if os.path.exists('trees/' + str(self.num_rows)):
return self.load_tree(self.num_rows)
"""
Constructing the tree structure with unique data from each column,
the hierarchy goes like :
- Products
|- Department
|- Commodity-Desc
|- SubCommodity-Desc
|- Size of the product
|- Brand name
|- Product ID
"""
r = Resolver('name')
for row in self.data.iterrows():
row = row[1]
for index, node_name in enumerate(row):
if index == 0:
# The first index will be attached to the root node
try:
# If we find the child, we do nothing
r.get(self.tree, node_name)
except ChildResolverError:
# If we don't (that raises an exception), we add the child to the root
Node(node_name, self.tree)
else:
parent = r.get(self.tree,
self.get_path_for_index_at_row(row, index))
Node(node_name, parent=parent)
self.save_tree(self.num_rows)
return self
def has_path(self, path):
r = Resolver('id')
try:
r.get(self.root, path)
return True
except:
return False
class MessageListModel(QAbstractItemModel):
def __init__(self, parent=None):
super(MessageListModel, self).__init__(parent)
self.message_list = []
self.name_resolver = Resolver('name')
def set_message_list(self, message_list):
self.message_list = message_list
def headerData(self,
section: int,
orientation: PySide2.QtCore.Qt.Orientation,
role: int = ...):
if role == Qt.DisplayRole and orientation == Qt.Horizontal:
if section == 0:
return "Server"
elif section == 1:
return "Code"
elif section == 2:
return "Type"
elif section == 3:
return "Subtype"
def data(self, index: PySide2.QtCore.QModelIndex, role: int = ...):
if not index.isValid():
return None
node = index.internalPointer()
if role == Qt.DisplayRole:
column = index.column()
if column == 0:
return self.name_resolver.get(node,
'../../../../../../server').value
elif column == 1:
return node.value.code
elif column == 2:
return self.name_resolver.get(node, './content').class_name
elif column == 3:
return self.name_resolver.get(node,
'./content/content').class_name
def index(self,
row: int,
column: int,
parent: PySide2.QtCore.QModelIndex = ...):
return self.createIndex(row, column, self.message_list[row])
def parent(self, index: PySide2.QtCore.QModelIndex):
return QModelIndex()
def columnCount(self, parent: PySide2.QtCore.QModelIndex = ...):
return 4
def rowCount(self, parent: PySide2.QtCore.QModelIndex = ...):
if not parent.isValid():
return len(self.message_list)
return 0
def read_putty_registry(self):
aReg = ConnectRegistry(None, HKEY_CURRENT_USER)
aKey = OpenKey(aReg, r"Software\SimonTatham\PuTTY\Sessions")
res = Resolver('name')
for i in range(0, QueryInfoKey(aKey)[0]):
try:
asubkey_name = EnumKey(aKey, i)
if str(asubkey_name) == 'WinSSHTerm':
continue
asubkey = OpenKey(aKey, asubkey_name)
try:
sessionPath = str(
QueryValueEx(asubkey, "PsmPath")[0].encode('utf-8'))
except Exception as e:
sessionPath = "Sessions"
pass
list = sessionPath.split('\\')
tmp = self.root
counter = 1
for i in list:
pathB64 = base64.b64encode(i)
try:
if res.get(tmp, pathB64):
tmp = res.get(tmp, pathB64)
if counter >= len(list):
self.saveSessionData(
node=tmp,
name=str(asubkey_name),
username=str(
QueryValueEx(asubkey, "UserName")[0]),
privateKey=str(
QueryValueEx(asubkey,
"PublicKeyFile")[0]),
hostname=str(
QueryValueEx(asubkey, "HostName")[0]),
port=str(
QueryValueEx(asubkey,
"PortNumber")[0]))
except ChildResolverError as e:
tmp = Node(pathB64, parent=tmp, type="Container")
if counter >= len(list):
self.saveSessionData(
node=tmp,
name=str(asubkey_name),
username=str(
QueryValueEx(asubkey, "UserName")[0]),
privateKey=str(
QueryValueEx(asubkey, "PublicKeyFile")[0]),
hostname=str(
QueryValueEx(asubkey, "HostName")[0]),
port=str(
QueryValueEx(asubkey, "PortNumber")[0]))
counter = counter + 1
except EnvironmentError as e:
break
def node_exists(root, node_name) -> bool:
"""Checks if a node with the name provided to this method exists
Args:
root: root node of tree or root of search if search is applied to subtree
node_name: name of the node that is searched for.
"""
try:
r = Resolver()
r.get(root, node_name)
return True
except ChildResolverError:
return False
def read_mobaxterm_ini(self):
style = wx.FD_OPEN | wx.FD_FILE_MUST_EXIST
dialog = wx.FileDialog(self,
message='Open MobaXterm.ini',
wildcard='(*.ini)|*.ini',
style=style)
if dialog.ShowModal() == wx.ID_OK:
file = dialog.GetPath()
else:
return False
dialog.Destroy()
try:
config = configparser.RawConfigParser()
config.optionxform = str
config.read(file)
res = Resolver('name')
for s in config.sections():
if s.startswith('Bookmarks'):
if config[s]['SubRep'] == 'PuTTY sessions':
continue
tmp = self.root
for (key, val) in config.items(s):
if key == 'ImgNum':
continue
if key == 'SubRep' and val:
sessionPath = config[s]['SubRep']
list = sessionPath.encode('utf-8').split('\\')
counter = 1
for i in list:
pathB64 = base64.b64encode(i)
try:
if res.get(tmp, pathB64):
tmp = res.get(tmp, pathB64)
except ChildResolverError as e:
node = Node(pathB64,
parent=tmp,
type='Container')
tmp = node
counter = counter + 1
break
for (key, val) in config.items(s):
if key == 'ImgNum' or key == 'SubRep':
continue
sessionData = val.encode('utf-8').split('%')
if sessionData[0] == '#109#0':
self.saveSessionData(tmp, key, sessionData[3], '',
sessionData[1],
sessionData[2])
return True
except Exception as e:
wx.MessageBox(str(e), "Error")
return False
def read_superputty_xml(self):
style = wx.FD_OPEN | wx.FD_FILE_MUST_EXIST
dialog = wx.FileDialog(self, message='Open Sessions.XML', wildcard='(*.XML)|*.XML', style=style)
if dialog.ShowModal() == wx.ID_OK:
file = dialog.GetPath()
else:
return False
dialog.Destroy()
try:
tree = ET.parse(file)
for item in tree.iter():
if item.tag == "SessionData":
sessionPath = item.attrib.get('SessionId')
list = sessionPath.encode('utf-8').split('/')
tmp = self.root
res = Resolver('name')
counter = 1
for i in list:
pathB64 = base64.b64encode(i)
try:
if res.get(tmp, pathB64):
tmp = res.get(tmp, pathB64)
if counter >= len(list):
print pathB64
self.saveSessionData(
node=tmp,
name=str(item.attrib.get('SessionName').encode('utf-8')),
username=str(item.attrib.get('Username').encode('utf-8')),
privateKey='',
hostname=str(item.attrib.get('Host').encode('utf-8')),
port=str(item.attrib.get('Port').encode('utf-8'))
)
print pathB64
except ChildResolverError as e:
if counter < len(list):
tmp = Node(pathB64, parent=tmp, type="Container")
if counter >= len(list):
self.saveSessionData(
node=tmp,
name=str(item.attrib.get('SessionName').encode('utf-8')),
username=str(item.attrib.get('Username').encode('utf-8')),
privateKey='',
hostname=str(item.attrib.get('Host').encode('utf-8')),
port=str(item.attrib.get('Port').encode('utf-8'))
)
counter = counter + 1
return True
except Exception as e:
wx.MessageBox(str(e), "Error")
return False
def build_tree_from_files(files: List[Dict]) -> Node:
"""Builds a directory tree out of the results from the metadata-hub query.
The tree is build upon a "Root" node which is returned at the end.
Each dict contains the entire result but only the file paths are used.
Args:
files (List[Dict]): list containing the results from the mdh query
Returns:
Node: The root node (anytree.Node) of the resulting tree
"""
# '/home/dome_/test_tree'
root_node = Node("Root")
parent_finder = Resolver("name")
file_paths = _extract_file_paths_parts(files)
max_index = _length_of_longest_path(file_paths)
for i in range(max_index):
# In every iteration of the outer loop we only work on parts up to position i
for file_path in file_paths:
if i >= len(file_path):
# After reaching the last part of a path it can be skipped
continue
last_path_node = file_path[i]
path_without_last_node = _create_path_from_parts(file_path[:i])
parent_node = parent_finder.get(root_node, path_without_last_node)
if not _parent_has_child(parent_node, last_path_node):
Node(last_path_node, parent_node)
return root_node
def getattr(self, path, fh=None):
path_stat = SFS_Stat()
print(f"getattr called with: {path}")
os_path = os.stat(path)
path_stat.st_size = os_path.st_size
path_stat.st_blocks
now = time.time()
path_stat.st_atime = now
path_stat.st_mtime = now
path_stat.st_ctime = now
if path in [".", "..", "/"]:
path_stat.st_mode = stat.S_IFDIR | 0o755
return path_stat.__dict__
file_finder = Resolver("name")
path = path[1:] # strip leading "/"
path_node: Node = file_finder.get(self.directory_tree, path)
if len(path_node.children) == 0:
print("got regular file")
path_stat.st_mode = stat.S_IFREG | 0o755
else:
path_stat.st_mode = stat.S_IFDIR | 0o755
return path_stat.__dict__
def __init__(self, texture_info_list):
self.walker = Walker()
r = Resolver('name')
id = 0
self.root = TreeNode('Root', id)
id += 1
for info in texture_info_list:
respath = info.respath
cur_parent = self.root
cur_parent.filesize.bytes += info.filesize.bytes
cur_node = None
for chunk in respath.chunks:
try:
cur_node = r.get(cur_parent, chunk)
cur_parent = cur_node
except ChildResolverError:
cur_node = TreeNode(chunk, id, cur_parent)
cur_parent = cur_node
id += 1
if chunk == respath.chunks[-1]:
cur_node.horizontal_desc = str(info)
finally:
cur_node.filesize.bytes += info.filesize.bytes
def merge_private_into_main_tree(tree_root: VSSNode):
r = Resolver()
try:
private = r.get(tree_root, "/Vehicle/Private")
detect_and_merge(tree_root, private)
private.parent = None
except ChildResolverError:
print("No private Attribute branch detected")
def resolve_rel(self, path) -> 'AnalNode':
"""
>>> f.resolve_rel("./obl/amod")
:param path:
:return:
"""
r = Resolver('dependency_relation')
return r.get(self, path)
class XsdTree:
"""Represents a tree structure containing the important
XSD information for all the elements and attributes.
"""
def __init__(self, root: XsdNode):
self.root: XsdNode = root
self.node_resolver = Resolver("name")
self.expand_element = self._build_expand_element()
def find_node_by_stack(self, node_stack: List[str]) -> Optional[XsdNode]:
"""Finds the node definition in the tree that matches the given stack of tags.
Args:
node_stack (List[str]): The stack of tag names composing a tree branch.
Like: ['root', 'node', 'subnode', 'leaf']
Returns:
Optional[XsdNode]: The node definition matching the leaf in the path or
None if the node could not be found.
"""
result_node = None
if node_stack:
try:
path = self._get_path_from_stack(node_stack)
if path.endswith(self.expand_element.name):
return self.expand_element
result_node = self.node_resolver.get(self.root, path)
except ResolverError as e:
print(e)
return result_node
def render(self) -> str:
"""Gets an ascii representation of this tree.
Returns:
str: An ascii representation of the tree.
"""
return self.root.render()
def _get_path_from_stack(self, node_stack: List[str]) -> str:
if node_stack[0] == self.root.name:
node_stack[0] = "."
return "/".join(node_stack)
def _build_expand_element(self) -> XsdNode:
"""Creates a XsdNode representing the <expand> element and it's attributes.
The <expand> element is a special element that is not defined in the XSD schema but allows to use macros in the tool.
Returns:
XsdNode: The node with information about the special <expand> element.
"""
expand_node = XsdNode("expand", None)
attr_name = "macro"
attr = XsdAttribute(attr_name, None, type_name=None, is_required=True)
expand_node.attributes[attr_name] = attr
return expand_node
def merge_tree(base: VSSNode, overlay: VSSNode):
r = Resolver()
overlay_element: VSSNode
for overlay_element in LevelOrderIter(overlay):
element_name = "/" + overlay_element.qualified_name("/")
if not VSSNode.node_exists(base, element_name):
#The node in the overlay does not exist, so we connect it
#print(f"Not exists {overlay_element.qualified_name()} does not exist, creating.")
new_parent_name = "/" + overlay_element.parent.qualified_name("/")
new_parent = r.get(base, new_parent_name)
overlay_element.parent = new_parent
else:
# else we merge the node. The merge function of VSSNode is not recursive
# so children in base will not be overwritten
#print(f"Merging {overlay_element.qualified_name()}")
other_node: VSSNode = r.get(base, element_name)
other_node.merge(overlay_element)
def get(self, path):
"""
>>> from sagas.nlu.warehouse import warehouse as wh
>>> wh.get("/_/ent:Person")
:param path:
:return:
"""
r = Resolver('qualified')
return r.get(self, path)
def _get_class(self, row):
"""get the class of a row in a dataframe"""
node = self._tree
r = Resolver('name')
while not node.is_leaf:
if node.feat != None:
print "the " + node.feat + " is " + row[node.feat]
n = r.get(node, row[node.feat])
node = n
print "\tso we predict " + node.target
return node.target
def handle_message(self, message):
#m = 0
if m == 0:
self.value = message["value"]
self.decide()
#m > 0
else:
path = message["path"]
#Commander message, empty tree
if len(message["path"]) == 1:
self.root = AnyNode(id=path[0],
value=message["value"],
decide_value=message["value"],
name=path[0])
#Message from a lieutenant
else:
#Get last known node on path and add new node as its child
with open("./debug/messages_handled_lieutenant.txt", "a") as f:
print(self.id, message["path"], path[-1], file=f)
r = Resolver('id')
last_node = r.get(self.root, '/'.join(path[1:-1]))
AnyNode(id=path[-1],
value=message["value"],
decide_value=message["value"],
parent=last_node,
name=path[-1])
with open("./debug/all_tree_operations.txt", "a") as f:
print(self.id,
message["path"],
last_node.id,
path[-1],
file=f)
if self.id not in path and len(path) <= m:
#If possible, apend its own id to the path and broadcast the message
path.append(self.id)
message = self.construct_message(path, message["value"])
self.multicast(self.process_list[1:], message)
leafs = [
node for node in LevelOrderIter(self.root) if node.is_leaf
]
if len(leafs) == n_leafs:
#If already received all the messages, start phase 2
reverse_nodes = [node
for node in LevelOrderIter(self.root)][::-1]
for node in reverse_nodes:
if not node.is_leaf:
node.decide_value = majority(
[n.decide_value for n in node.children])
self.value = self.root.decide_value
print("Process", self.id, "has decided", self.value)
#print("Process", self.id, "has decided", self.value)
with open('./results/' + self.id + ".txt", "w") as f:
print(self.id, RenderTree(self.root), file=f)
def detect_and_merge(tree_root: VSSNode, private_root: VSSNode):
r = Resolver()
private_element: VSSNode
for private_element in LevelOrderIter(private_root):
if private_element == private_root:
continue
if not private_element.is_private():
continue
element_name = "/" + private_element.qualified_name()
candidate_name = element_name.replace("Private/", "")
if not VSSNode.node_exists(tree_root, candidate_name):
new_parent_name = "/" + private_element.parent.qualified_name().replace("/Private", "")
new_parent = r.get(tree_root, new_parent_name)
private_element.parent = new_parent
elif private_element.is_leaf:
other_node = r.get(tree_root, candidate_name)
other_node.merge(private_element)
private_element.parent = None
def on_message_list_activated(self, index: QModelIndex):
# Gets activated node.
activated_node = index.internalPointer()
# Recover node message message content
r = Resolver('name')
message = r.get(activated_node, './content')
object_tree_index = self.object_tree.model().index_from_node(message)
self.object_tree.setRootIndex(object_tree_index)
# Object tree first element
object_tree_first_item = self.object_tree.model().index(0, 0, object_tree_index)
self.on_object_tree_clicked(object_tree_first_item)
def readdir(self, path, fh):
print(f"readdir called with {path}")
children = [".", ".."]
file_finder = Resolver("name")
path = path[1:] # strip leading "/"
path_node: Node = file_finder.get(self.directory_tree, path)
child: Node
for child in path_node.children:
children.append(child.name)
print(f"added {child.name}")
return children
def main():
# part 1
print("# part1:")
f = open("input7.txt", "r")
lines = f.readlines()
f.close()
root = Node("root")
# generate first level
for n_line, line in enumerate(lines):
splited_line = line.replace("\n","").split(" contain ")
Node(splited_line[0].strip().replace("bags", "bag").replace(".", ""), parent=root, bag_value=0)
# generate n-th levels
for pre, fill, node in RenderTree(root):
for n_line, line in enumerate(lines):
splited_line = line.replace("\n","").split(" contain ")
if(node.name in splited_line[0]):
desc = splited_line[1].split(",")
for d in desc:
d_value = ''.join([i for i in d if i.isdigit()]).strip()
#if(len(d_value) == 0): d_value = 0
d_name = ''.join([i for i in d if not i.isdigit()]).strip()
d_name = d_name.replace("bags", "bag").replace(".", "")
if(d_name not in "no other bag"):
Node(d_name, parent=node, bag_value=d_value)
#print the tree
# for pre, fill, node in RenderTree(root):
# print("%s%s" % (pre, node.name))
results = []
for pre, fill, node in RenderTree(root, maxlevel=2):
finds = findall(node, filter_=lambda n: "shiny gold" in n.name)
if(len(finds) > 0): results.append(finds)
print("total part1 :", len(results) - 2)
#part 2
r = Resolver('name')
shiny_gold_bag_node = r.get(root, "shiny gold bag")
shiny_gold_bag_node.bag_value = 1
# print the tree
# for pre, fill, node in RenderTree(shiny_gold_bag_node):
# print("%s%s(%s)" % (pre, node.name, node.bag_value))
print("total part2 :", count_bags(shiny_gold_bag_node))
def on_network_event_list_activated(self, index: QModelIndex):
# Gets activated node.
activated_node = index.internalPointer()
# Recover node packet list.
r = Resolver('name')
packets = r.get(activated_node, './packets')
# Sets packet list as root index on object tree.
object_tree_index = self.object_tree.model().index_from_node(packets)
self.object_tree.setRootIndex(object_tree_index)
# Object tree first element
object_tree_first_item = self.object_tree.model().index(0, 0, object_tree_index)
self.on_object_tree_clicked(object_tree_first_item)
def add_to_structure(base, resource):
try:
r = Resolver("name")
try:
if '?' in resource[0]:
return
existing_node = r.get(base, resource[0])
if len(resource) > 1:
add_to_structure(existing_node, resource[1:])
except Exception as e:
new_node = Node(resource[0], parent=base)
if len(resource) > 1:
add_to_structure(new_node, resource[1:])
except Exception as e:
return {'web_structure': e}
def get_file(self, filename):
'''
Get the node with the given file
:param filename: path to file relative to the current node
:return: node with the given file or None
'''
if filename[0] == '/':
filename = '/root' + filename
r = Resolver("name")
try:
node = r.get(self.cur_node, filename)
if node:
if node.is_file:
return node
else:
return None
else:
return None
except Exception as e:
return None
def get_dir(self, dirname):
'''
Get the node with the given directory
:param dirname: path to direcroty relative to the current node
:return: node with the given directory or None
'''
if len(dirname) >= 1:
if dirname[0] == '/':
dirname = '/root' + dirname
r = Resolver("name")
try:
node = r.get(self.cur_node, dirname)
if node:
if node.is_file:
return None
else:
return node
else:
return None
except Exception as e:
return None
def resolve_tree_node(tree: Node, path_str: str) -> Node:
"""Given a path string, return the corresponding node from a tree
Parameters
----------
tree : Node
anytree Node object
path_str : str
String representation of anytree node path
Returns
-------
Node
Referenced anytree Node object
"""
resolver = Resolver('name')
try:
node = resolver.get(tree, path_str)
except (ResolverError, ChildResolverError):
parent = resolve_tree_node(tree, path_str.rsplit('/', 1)[0])
node = Node(path_str.split('/')[-1], parent=parent)
return node
def getNode(self, path):
resolver = Resolver("name")
return resolver.get(self.tree, "/GGPK" + path)
class SuffixTree(Node, BaseEstimator):
"""Suffix tree representation of subsequences.
Parameters
----------
max_width: int
Maximum width of the tree, or alternatively maximum length of subsequence.
min_support: int, or float, optional
Minimum occurence of subsequence to take into account. This parameter can be:
- None, in which case no filtering is done at all
- An int, giving the exact number of minum subsequnce occurences in sequences
- A float, giving the ratio of occurences from total occurrences of subsequences of length ``max_width`` in sequences
"""
def __init__(self, max_width, min_support=None):
super().__init__(name='root')
self._max_width = max_width
self._min_support = min_support
self._resolver = Resolver('name')
self._fitted = False
def _safe_resolve(self, sequence, parent=True):
"""Resolves path in the contructed tree.
Paramters
---------
sequence: list of chars
Sequence defining the node.
parent: bool, default True
If return parent node or the node defined by the ``sequence`` parameter.
Returns
-------
node: anytree.node
Parent node or current node of the tree. None, if the node defined by the ``sequence`` parameter does not exist in the tree.
"""
path = '/' + self.name + '/' + '/'.join([sequence[::-1][:i][::-1]
for i in range(1, len(sequence) + int(parent == False))])
try:
return self._resolver.get(self, path)
except ChildResolverError as e:
return None
def _safe_resolve_parent(self, sequence):
return self._safe_resolve(sequence, parent=True)
def _safe_resolve_node(self, sequence):
return self._safe_resolve(sequence, parent=False)
def fit(self, sequences):
"""Fit the suffix tree.
Paramters
--------
sequences: list of pandas.Series
List of sequences(possibly with diffferent lengths).
Returns
-------
self : SuffixTree
Returns an instance of self.
"""
all_sequences = np.hstack(sequences)
self.count = len(all_sequences)
self._symbols = np.unique(all_sequences)
self.states = ['^']
for width in range(1, self._max_width + 1):
counts = _get_subsequence_counts(
sequences, width, self._min_support)
for sequence, count in counts.items():
self.states.append(sequence)
parent = self._safe_resolve_parent(sequence)
thenode = Node(sequence, parent=parent, count=count)
self._fitted = True
return self
def render(self):
"""Returns pretty representation of the tree."""
if not self._fitted:
raise NotFittedError()
return RenderTree(self)
# In[220]:
state_number = 0
top = CustomNode("root", state_number)
r = Resolver('name')
with open(sys.argv[1]) as f:
words = f.readlines()
for word in words:
trimmed_word = word.rstrip()
root = top
for i in range(len(trimmed_word)):
try:
n = r.get(root, trimmed_word[i])
except ChildResolverError:
state_number += 1
n = CustomNode(trimmed_word[i], state_number, root)
root = n
# In[221]:
#for pre, _, node in RenderTree(top):
# treestr = u"%s%s" % (pre, node.name)
# print(treestr.ljust(8), node.state_number)
def print_as_tabs(prev_state, next_state, input_label, output_label):
print(
str(prev_state) + "\t" + str(next_state) + "\t" + str(input_label) +
"\t" + str(output_label))
def get_config(self, namepath):
'''Get config from store by attribute.'''
r = Resolver('name')
results = r.get(self, namepath)
return results
