def addtorow(self, xpath, data, elem=None): fullp='/'.join(xpath) path=None if elem!=None: s=self.schemagetall if fullp in s: path=fullp else: shortp=pathwithoutns(xpath) if shortp in s: path=shortp if path == None: return try: data = cleansubtree.match(etree.tostring(elem)).groups()[0] except AttributeError: data = etree.tostring(elem) else: s=self.schema if fullp in s: path=fullp else: shortp=pathwithoutns(xpath) if shortp in s: path=shortp if path==None: if self.strict==2 and elem==None: path=xpath self.resetrow() msg='Undeclared path in XML-prototype was found in the input data. The path is:\n' shortp='/'+pathwithoutns(path) fullp='/'+'/'.join(path) if shortp!=fullp: msg+=shortp+'\n' msg+=fullp+'\nThe data to insert into path was:\n'+functions.mstr(data) raise etree.ParseError(msg) else: if self.row[s[path][0]]=='': self.row[s[path][0]]=data.replace('\t', self.tabreplace) return i=1 attribnum=path+'1' oldattribnum=path while attribnum in s: if self.row[s[attribnum][0]]=='': self.row[s[attribnum][0]]=data.replace('\t', self.tabreplace) return i+=1 oldattribnum=attribnum attribnum=path+str(i) self.row[s[oldattribnum][0]]+='\t'+data.replace('\t', self.tabreplace)
def read_traffic_matrix(path, encoding='utf-8'): """ Parses a traffic matrix from a traffic matrix XML file. If the XML file contains more than one traffic matrix, it returns a TrafficMatrixSequence object, otherwise a TrafficMatrixObject. Parameters ---------- path: str The path of the XML file to parse encoding : str, optional The encoding of the file Returns ------- tm : TrafficMatrix or TrafficMatrixSequence """ def parse_single_matrix(head): """ Parses a single traffic matrix from the XML file """ traffic_matrix = TrafficMatrix() for prop in head.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) if name == 'volume_unit' and value not in capacity_units: raise ET.ParseError(\ 'Invalid volume_unit property in time node') traffic_matrix.attrib[name] = value for origin in head.findall('origin'): o = util.xml_cast_type(origin.attrib['id.type'], origin.attrib['id']) for destination in origin.findall('destination'): d = util.xml_cast_type(destination.attrib['id.type'], destination.attrib['id']) volume = float(destination.text) traffic_matrix.add_flow(o, d, volume) return traffic_matrix tree = ET.parse(path) head = tree.getroot() matrix_type = head.attrib['type'] if matrix_type == 'single': traffic_matrix = parse_single_matrix(head.find('time')) elif matrix_type == 'sequence': traffic_matrix = TrafficMatrixSequence() for prop in head.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) traffic_matrix.attrib[name] = value for matrix in head.findall('time'): traffic_matrix.append(parse_single_matrix(matrix)) else: raise ET.ParseError('Invalid TM type attribute in XML file') return traffic_matrix
def parse_single_matrix(head): """ Parses a single traffic matrix from the XML file """ traffic_matrix = TrafficMatrix() for prop in head.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) if name == 'volume_unit' and value not in capacity_units: raise ET.ParseError(\ 'Invalid volume_unit property in time node') traffic_matrix.attrib[name] = value for origin in head.findall('origin'): o = util.xml_cast_type(origin.attrib['id.type'], origin.attrib['id']) for destination in origin.findall('destination'): d = util.xml_cast_type(destination.attrib['id.type'], destination.attrib['id']) volume = float(destination.text) traffic_matrix.add_flow(o, d, volume) return traffic_matrix
def report_parser(path_or_file, ignore_log_info=True): """ This functions transform XML OpenVas file report to OpenVASResult object structure. To pass StringIO file as parameter, you must do that: >>> import StringIO >>> xml='<report extension="xml" type="scan" id="aaaa" content_type="text/xml" format_id="a994b278-1f62-11e1-96ac-406186ea4fc5"></report>' >>> f=StringIO.StringIO(xml) >>> report_parser(f) [OpenVASResult] To pass a file path: >>> xml_path='/home/my_user/openvas_result.xml' >>> report_parser(xml_path) [OpenVASResult] Language specification: http://www.openvas.org/omp-4-0.html :param path_or_file: path or file descriptor to xml file. :type path_or_file: str | file | StringIO :param ignore_log_info: Ignore Threats with Log and Debug info :type ignore_log_info: bool :raises: etree.ParseError, IOError, TypeError :return: list of OpenVASResult structures. :rtype: list(OpenVASResult) """ if isinstance(path_or_file, str): if not os.path.exists(path_or_file): raise IOError("File %s not exits." % path_or_file) if not os.path.isfile(path_or_file): raise IOError("%s is not a file." % path_or_file) else: if not getattr(getattr(path_or_file, "__class__", ""), "__name__", "") in ("file", "StringIO", "StringO", "StringI"): raise TypeError("Expected str or file, got '%s' instead" % type(path_or_file)) # Parse XML file try: xml_parsed = etree.parse(path_or_file) except etree.ParseError: raise etree.ParseError( "Invalid XML file. Ensure file is correct and all tags are properly closed." ) # Use this method, because API not exposes real path and if you write isisntance(xml_results, Element) # doesn't works if type(xml_parsed).__name__ == "Element": xml = xml_parsed elif type(xml_parsed).__name__ == "ElementTree": xml = xml_parsed.getroot() else: raise TypeError("Expected ElementTree or Element, got '%s' instead" % type(xml_parsed)) if "status" in xml.keys(): xml = xml[0] # Check valid xml format if "id" not in xml.keys(): raise ValueError( "XML format is not valid, doesn't contains id attribute.") # Regex port_regex_specific = re.compile("([\w\d\s]*)\(([\d]+)/([\w\W\d]+)\)") port_regex_generic = re.compile("([\w\d\s]*)/([\w\W\d]+)") cvss_regex = re.compile("(cvss_base_vector=[\s]*)([\w:/]+)") vulnerability_IDs = ("cve", "bid", "bugtraq") m_return = [] m_return_append = m_return.append # All the results for l_results in xml.findall(".//result"): l_partial_result = OpenVASResult() # Id l_vid = None try: l_vid = l_results.get("id") l_partial_result.id = l_vid except TypeError as e: logging.warning( "%s is not a valid vulnerability ID, skipping vulnerability..." % l_vid) logging.debug(e) continue # -------------------------------------------------------------------------- # Filter invalid vulnerability # -------------------------------------------------------------------------- threat = l_results.find("threat") if threat is None: logging.warning( "Vulnerability %s can't has 'None' as thread value, skipping vulnerability..." % l_vid) continue else: # Valid threat? if threat.text not in OpenVASResult.risk_levels: logging.warning( "%s is not a valid risk level for %s vulnerability. skipping vulnerability..." % (threat.text, l_vid)) continue # Ignore log/debug messages, only get the results if threat.text in ("Log", "Debug") and ignore_log_info is True: continue # For each result for l_val in l_results.getchildren(): l_tag = l_val.tag # -------------------------------------------------------------------------- # Common properties: subnet, host, threat, raw_description # -------------------------------------------------------------------------- if l_tag in ("subnet", "host", "threat"): # All text vars can be processes both. try: setattr(l_partial_result, l_tag, l_val.text) except (TypeError, ValueError) as e: logging.warning( "%s is not a valid value for %s property in %s vulnerability. skipping vulnerability..." % (l_val.text, l_tag, l_partial_result.id)) logging.debug(e) continue elif l_tag == "description": try: setattr(l_partial_result, "raw_description", l_val.text) except TypeError as e: logging.warning( "%s is not a valid description for %s vulnerability. skipping vulnerability..." % (l_val.text, l_vid)) logging.debug(e) continue # -------------------------------------------------------------------------- # Port # -------------------------------------------------------------------------- elif l_tag == "port": # Looking for port as format: https (443/tcp) l_port = port_regex_specific.search(l_val.text) if l_port: l_service = l_port.group(1) l_number = int(l_port.group(2)) l_proto = l_port.group(3) try: l_partial_result.port = OpenVASPort( l_service, l_number, l_proto) except (TypeError, ValueError) as e: logging.warning( "%s is not a valid port for %s vulnerability. skipping vulnerability..." % (l_val.text, l_vid)) logging.debug(e) continue else: # Looking for port as format: general/tcp l_port = port_regex_generic.search(l_val.text) if l_port: l_service = l_port.group(1) l_proto = l_port.group(2) try: l_partial_result.port = OpenVASPort( l_service, 0, l_proto) except (TypeError, ValueError) as e: logging.warning( "%s is not a valid port for %s vulnerability. skipping vulnerability..." % (l_val.text, l_vid)) logging.debug(e) continue # -------------------------------------------------------------------------- # NVT # -------------------------------------------------------------------------- elif l_tag == "nvt": # The NVT Object l_nvt_object = OpenVASNVT() try: l_nvt_object.oid = l_val.attrib['oid'] except TypeError as e: logging.warning( "%s is not a valid NVT oid for %s vulnerability. skipping vulnerability..." % (l_val.attrib['oid'], l_vid)) logging.debug(e) continue # Sub nodes of NVT tag l_nvt_symbols = [ x for x in dir(l_nvt_object) if not x.startswith("_") ] for l_nvt in l_val.getchildren(): l_nvt_tag = l_nvt.tag # For each xml tag... if l_nvt_tag in l_nvt_symbols: # For tags with content, like: <cert>blah</cert> if l_nvt.text: # For filter tags like <cve>NOCVE</cve> if l_nvt.text.startswith("NO"): try: setattr(l_nvt_object, l_nvt_tag, "") except (TypeError, ValueError) as e: logging.warning( "Empty value is not a valid NVT value for %s property in %s vulnerability. skipping vulnerability..." % (l_nvt_tag, l_vid)) logging.debug(e) continue # Tags with valid content else: # -------------------------------------------------------------------------- # Vulnerability IDs: CVE-..., BID..., BugTraq... # -------------------------------------------------------------------------- if l_nvt_tag.lower() in vulnerability_IDs: l_nvt_text = getattr(l_nvt, "text", "") try: setattr(l_nvt_object, l_nvt_tag, l_nvt_text.split(",")) except (TypeError, ValueError) as e: logging.warning( "%s value is not a valid NVT value for %s property in %s vulnerability. skipping vulnerability..." % (l_nvt_text, l_nvt_tag, l_vid)) logging.debug(e) continue else: l_nvt_text = getattr(l_nvt, "text", "") try: setattr(l_nvt_object, l_nvt_tag, l_nvt_text) except (TypeError, ValueError) as e: logging.warning( "%s value is not a valid NVT value for %s property in %s vulnerability. skipping vulnerability..." % (l_nvt_text, l_nvt_tag, l_vid)) logging.debug(e) continue # For filter tags without content, like: <cert/> else: try: setattr(l_nvt_object, l_nvt_tag, "") except (TypeError, ValueError) as e: logging.warning( "Empty value is not a valid NVT value for %s property in %s vulnerability. skipping vulnerability..." % (l_nvt_tag, l_vid)) logging.debug(e) continue # Get CVSS cvss_candidate = l_val.find("tags") if cvss_candidate is not None and getattr( cvss_candidate, "text", None): # Extract data cvss_tmp = cvss_regex.search(cvss_candidate.text) if cvss_tmp: l_nvt_object.cvss_base_vector = cvss_tmp.group( 2) if len(cvss_tmp.groups()) >= 2 else "" # Add to the NVT Object try: l_partial_result.nvt = l_nvt_object except (TypeError, ValueError) as e: logging.warning( "NVT oid %s is not a valid NVT value for %s vulnerability. skipping vulnerability..." % (l_nvt_object.oid, l_vid)) logging.debug(e) continue # -------------------------------------------------------------------------- # Unknown tags # -------------------------------------------------------------------------- else: # Unrecognised tag logging.warning("%s tag unrecognised" % l_tag) # Add to the return values m_return_append(l_partial_result) return m_return
def report_parser(path_or_file, ignore_log_info=True): """ This functions transform XML OpenVas file report to OpenVASResult object structure. To pass StringIO file as parameter, you must do that: >>> import StringIO >>> xml='<report extension="xml" type="scan" id="aaaa" content_type="text/xml" format_id="a994b278-1f62-11e1-96ac-406186ea4fc5"></report>' >>> f=StringIO.StringIO(xml) >>> report_parser(f) [OpenVASResult] To pass a file path: >>> xml_path='/home/my_user/openvas_result.xml' >>> report_parser(xml_path) [OpenVASResult] Language specification: http://www.openvas.org/omp-4-0.html :param path_or_file: path or file descriptor to xml file. :type path_or_file: str | file | StringIO :param ignore_log_info: Ignore Threats with Log and Debug info :type ignore_log_info: bool :raises: etree.ParseError, IOError, TypeError :return: list of OpenVASResult structures. :rtype: list(OpenVASResult) """ if isinstance(path_or_file, basestring): if not os.path.exists(path_or_file): raise IOError("File %s not exits." % path_or_file) if not os.path.isfile(path_or_file): raise IOError("%s is not a file." % path_or_file) else: if not getattr(getattr(path_or_file, "__class__", ""), "__name__", "") in ("file", "StringIO", "StringO"): raise TypeError("Expected basestring or file, got '%s' instead" % type(path_or_file)) # Parse XML file try: xml_parsed = etree.parse(path_or_file) except etree.ParseError: raise etree.ParseError( "Invalid XML file. Ensure file is correct and all tags are properly closed." ) # Use this method, because API not exposes real path and if you write isisntance(xml_results, Element) # doesn't works if type(xml_parsed).__name__ == "Element": xml = xml_parsed elif type(xml_parsed).__name__ == "ElementTree": xml = xml_parsed.getroot() else: raise TypeError("Expected ElementTree or Element, got '%s' instead" % type(xml_parsed)) # Check valid xml format if "id" not in xml.keys(): raise ValueError( "XML format is not valid, doesn't contains id attribute.") # Regex port_regex_specific = re.compile("([\w\d\s]*)\(([\d]+)/([\w\W\d]+)\)") port_regex_generic = re.compile("([\w\d\s]*)/([\w\W\d]+)") cvss_regex = re.compile("(cvss_base_vector=[\s]*)([\w:/]+)") vulnerability_IDs = ("cve", "bid", "bugtraq") m_return = [] m_return_append = m_return.append # All the results for l_results in xml.findall(".//result"): l_partial_result = OpenVASResult() # Id l_vid = None try: l_vid = l_results.get("id") l_partial_result.id = l_vid except TypeError, e: logging.warning( "%s is not a valid vulnerability ID, skipping vulnerability..." % l_vid) logging.debug(e) continue # -------------------------------------------------------------------------- # Filter invalid vulnerability # -------------------------------------------------------------------------- threat = l_results.find("threat") if threat is None: logging.warning( "Vulnerability %s can't has 'None' as thread value, skipping vulnerability..." % l_vid) continue else: # Valid threat? if threat.text not in OpenVASResult.risk_levels: logging.warning( "%s is not a valid risk level for %s vulnerability. skipping vulnerability..." % (threat.text, l_vid)) continue # Ignore log/debug messages, only get the results if threat.text in ("Log", "Debug") and ignore_log_info is True: continue # For each result for l_val in l_results.getchildren(): l_tag = l_val.tag # -------------------------------------------------------------------------- # Common properties: subnet, host, threat, raw_description #-------------------------------------------------------------------------- if l_tag in ("subnet", "host", "threat"): # All text vars can be processes both. try: setattr(l_partial_result, l_tag, l_val.text) except (TypeError, ValueError), e: logging.warning( "%s is not a valid value for %s property in %s vulnerability. skipping vulnerability..." % (l_val.text, l_tag, l_partial_result.id)) logging.debug(e) continue elif l_tag == "description": try: setattr(l_partial_result, "raw_description", l_val.text) except TypeError, e: logging.warning( "%s is not a valid description for %s vulnerability. skipping vulnerability..." % (l_val.text, l_vid)) logging.debug(e) continue
def read_topology(path, encoding='utf-8'): """Read a topology from an XML file and returns either a Topology or a DirectedTopology object Parameters ---------- path : str The path of the topology XML file to parse encoding : str, optional The encoding of the file Returns ------- topology: Topology or DirectedTopology The parsed topology """ tree = ET.parse(path) head = tree.getroot() topology = Topology() if head.attrib['linkdefault'] == 'undirected' \ else DirectedTopology() for prop in head.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) topology.graph[name] = value for node in head.findall('node'): v = util.xml_cast_type(node.attrib['id.type'], node.attrib['id']) topology.add_node(v) for prop in node.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) topology.node[v][name] = value if len(node.findall('stack')) > 0: if len(node.findall('stack')) > 1: raise ET.ParseError('Invalid topology. ' \ 'A node has more than one stack.') stack = node.findall('stack')[0] stack_name = util.xml_cast_type(stack.attrib['name.type'], stack.attrib['name']) stack_props = {} for prop in stack.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) stack_props[name] = value topology.node[v]['stack'] = (stack_name, stack_props) if len(node.findall('application')) > 0: topology.node[v]['application'] = {} for application in node.findall('application'): app_name = util.xml_cast_type(application.attrib['name.type'], application.attrib['name']) app_props = {} for prop in application.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) app_props[name] = value topology.node[v]['application'][app_name] = app_props for edge in head.findall('link'): u = util.xml_cast_type( edge.find('from').attrib['type'], edge.find('from').text) v = util.xml_cast_type( edge.find('to').attrib['type'], edge.find('to').text) topology.add_edge(u, v) for prop in edge.findall('property'): name = prop.attrib['name'] value = util.xml_cast_type(prop.attrib['type'], prop.text) topology.adj[u][v][name] = value return topology