def create_graph(self, file_path): num = 0 nodes = {} if file_path == "" or not os.path.isdir(file_path): return [None] for path, dirs, files in os.walk(file_path): if num > NODE_LIMIT: break num += 1 if path in nodes: node = nodes[path] else: dir_name = os.path.basename(path) node = Node(path, "Directory", dir_name) self.set_dir_attributes(path, node) for file_name in files: num += 1 full_path = os.path.join(path, file_name) new_node = Node(full_path, "File", file_name) node.neighbours.append(new_node) self.set_file_attributes(full_path, new_node) for dir_name in dirs: full_path = os.path.join(path, dir_name) new_node = Node(full_path, "Directory", dir_name) node.neighbours.append(new_node) nodes[full_path] = new_node if path == file_path: self.root = node return [self.root]
def read_recursively(self, parent, data, name): # Based on the type of data, we determine how to proceed with parsing. if isinstance(data, dict): new_node = Node(str(uuid.uuid4()), name, "JSON Object") parent.children.append(new_node) # If this is a json object, traverse it's fields. for key in data.keys(): self.read_recursively(new_node, data[key], key) elif isinstance(data, list): new_node = Node(str(uuid.uuid4()), name, "JSON Array") parent.children.append(new_node) # If this is a json array, traverse all the elements. for child in data: self.read_recursively(new_node, child, "Array element") # If this is a simple type, we just copy it's value into the attribute field. # For demo purposes, string and number cases are implemented. elif isinstance(data, str): new_node = Node(str(uuid.uuid4()), name, "JSON String") new_node.attributes["value"] = data parent.children.append(new_node) elif isinstance(data, (int, float)): new_node = Node(str(uuid.uuid4()), name, "JSON Number") new_node.attributes["value"] = data parent.children.append(new_node)
def read(self, file_path): soup = "" if ".html" not in file_path: raise TypeError("Must be .html file") return try: with open(file_path, encoding='utf8') as fp: soup = BeautifulSoup(fp, "html.parser") except FileNotFoundError: raise FileNotFoundError("HTML file " + file_path + " not found!") return if soup.html is None: raise TypeError("Invalid html document.") descendants = soup.html.descendants descendants_list = list(descendants) my_list = [x for x in descendants_list if x != '\n'] model_list = [] html_node = Node(str(uuid.uuid4()), 'html', 'tag') model_list.append(html_node) for child in my_list: if str(child).strip() == "": # skip whitespace continue node_name = child.name if child.name is not None else child node_type = 'tag' if child.name is not None else 'text' node = Node(str(uuid.uuid4()), node_name, node_type) if isinstance(child, Tag): node.attributes = child.attrs else: node.name = str(child).strip() if node.name != ' ' and node.name is not None and node.name != "\n" and node.name != '\t': model_list.append(node) for i, v in zip(range(len(model_list) - 1, -1, -1), reversed(model_list)): if i == model_list.__len__() - 1: continue if v.name == child.parent.name: model_list[i].children.append(node) break return [model_list[0]]
def _register_node(): """Create a simple node to register content to""" """This creates an empty node that a piece of content can be attached to. All rows should have a corresponding node created first to be attached to. """ if current_user.is_authenticated: node = Node(_version=1, _timestamp=datetime.utcnow(), user_id=current_user._id) else: node = Node(_version=1, _timestamp=datetime.utcnow(), user_id=0) db.session.add(node) db.session.commit() return node
def handle_starttag(self, tag, attrs): if self.root is None: self.root = Node(str(uuid.uuid4()), "tag", tag) self.current = self.root for attr in attrs: self.current.attributes[attr[0]] = attr[1] self.stack.append(self.current) else: new_node = Node(str(uuid.uuid4()), "tag", tag) for attr in attrs: new_node.attributes[attr[0]] = attr[1] self.current.neighbours.append(new_node) if tag not in self.void_elements: self.current = new_node self.stack.append(self.current)
def read(self, file_path): # Based on the file path type, try to read from a remote or local source. try: if "https://" in file_path or "http://" in file_path or ".com" in file_path: response = urllib.request.urlopen(file_path) str_response = response.read().decode('utf-8') data = json.loads(str_response) else: with open(file_path, encoding="utf8") as data_file: data = json.load(data_file) except Exception: traceback.print_exc() raise FileNotFoundError("JSON file " + file_path + " not found!") return [None] # Create root node. root_node = Node(str(uuid.uuid4()), "JSON Object", "Root") if isinstance(data, dict): # Traverse root node's children, passing the root as a top parent. for key in data.keys(): self.read_recursively(root_node, data[key], key) else: # Consider the possibility of a empty root. self.read_recursively(root_node, data, "Root") return [root_node]
def handle_data(self, data): if self.current is not None: data = data.replace("\\n", "") data = data.strip() if not data.isspace() and data != "": text_node = Node(str(uuid.uuid4()), "text", "text") text_node.attributes["content"] = data self.current.neighbours.append(text_node)
def init_regions(self) -> List[Node]: """Network initialization""" nodes = [] for region in Node.REGIONS: node = Node(name='%s1' % region, region=region) self.start(node) nodes.append(node) return nodes
def create_graph_recursively(self, parent, data, name): if isinstance(data, dict): new_node = Node(str(uuid.uuid4()), "JSON Object", name) parent.neighbours.append(new_node) for key in data.keys(): self.create_graph_recursively(new_node, data[key], key) elif isinstance(data, list): new_node = Node(str(uuid.uuid4()), "JSON Array", name) parent.neighbours.append(new_node) for item in data: self.create_graph_recursively(new_node, item, "Array element") elif isinstance(data, str): new_node = Node(str(uuid.uuid4()), "JSON String", name) new_node.attributes["value"] = data parent.neighbours.append(new_node) elif isinstance(data, (int, float)): new_node = Node(str(uuid.uuid4()), "JSON Number", name) new_node.attributes["value"] = data parent.neighbours.append(new_node)
def create(self): xos_args = self.get_xos_args() if not xos_args.get("site", None): raise Exception("Site is a required field of Node") if not xos_args.get("site_deployment", None): raise Exception("Deployment is a required field of Node") node = Node(**xos_args) node.caller = self.user node.save() self.postprocess(node) self.info("Created Node '%s' on Site '%s' Deployment '%s'" % (str(node), str(node.site), str(node.site_deployment.deployment)))
def create_graph(self, file_path): try: if "http://" in file_path or "https://" in file_path or ".com" in file_path: response = urllib.request.urlopen(file_path) str_response = response.read().decode('utf-8') data = json.loads(str_response) else: with open(file_path, encoding="utf8") as data_file: data = json.load(data_file) except Exception: traceback.print_exc() return [None] root_node = Node(str(uuid.uuid4()), "JSON Object", "Root") if isinstance(data, dict): for key in data.keys(): self.create_graph_recursively(root_node, data[key], key) else: self.create_graph_recursively(root_node, data, "Unnamed") return [root_node]
def node_classifier(self, request): certname = request.query_params.get("certname", "") master_id = request.query_params.get("master_id", "") error_message = "" try: node = Node.objects.get(certname=certname, master_zone__id=master_id) except Node.DoesNotExist: # Instantiate a dummy node to produce an empty answer node = Node() except MasterZone.DoesNotExist: error_message = 'Master with id "%s" not found' % master_id except ValidationError: error_message = "Invalid certname or master_id parameters" if error_message: return JsonResponse( {"error": error_message}, status=status.HTTP_400_BAD_REQUEST ) serializer = self.get_serializer(node) response = Response(serializer.data, content_type="text/yaml") cd = 'attachment; filename="%s_classifier.yml"' % node.certname response["Content-Disposition"] = cd return response
def __init__(self): self.links = [] self.found = [] self.row_chars = 30 self.graph = Node()
def handle_startendtag(self, tag, attrs): new_node = Node(str(uuid.uuid4()), "tag", tag) for attr in attrs: new_node.attributes[attr[0]] = attr[1] self.current.neighbours.append(new_node)