def scan_from_xml(self): """ Returns a dict compatible with database schema for scan and insert scan data. """ debug("Building scan table..") parsedsax = self.parsed scan_d = { } scan_d["args"] = parsedsax.nmap_command timestamp_start = parsedsax.start scan_d["start"] = datetime.fromtimestamp(float(timestamp_start)) scan_d["startstr"] = empty() # ToFix: Parser isnt storing this scan_d["finish"] = datetime(*parsedsax.finish_epoch_time[:6]) scan_d["finishstr"] = parsedsax.finish_time scan_d["xmloutputversion"] = ( parsedsax.nmap["nmaprun"]["xmloutputversion"]) if self.store_original: scan_d["xmloutput"] = '\n'.join(open(self.xml_file, 'r').readlines()) else: scan_d["xmloutput"] = empty() scan_d["verbose"] = parsedsax.verbose_level scan_d["debugging"] = parsedsax.debugging_level scan_d["hosts_up"] = parsedsax.hosts_up scan_d["hosts_down"] = parsedsax.hosts_down scanner_name = parsedsax.scanner scanner_version = parsedsax.scanner_version # get fk_scanner scanner_id = self.get_scanner_id_from_db(scanner_name, scanner_version) if not scanner_id: self.insert_scanner_db(scanner_name, scanner_version) scanner_id = self.get_id_for("scanner") scan_d["scanner"] = scanner_id self.insert_scan_db(scan_d) # get pk for the just inserted scan. scan_d["pk"] = self.get_id_for("scan") return scan_d
def insert_osmatch_db(self, host, osmatch): """ Create new record in osmatch with data from osmatch dict. """ debug("Inserting new osmatch into database") osmatch["line"] = empty() # ToFix: Parser isnt storing this self.cursor.execute("INSERT INTO osmatch (name, accuracy, line, " "fk_host) VALUES (?, ?, ?, ?)", (osmatch["name"], osmatch["accuracy"], osmatch["line"], host))
def insert_osmatch_db(self, host, osmatch): """ Create new record in osmatch with data from osmatch dict. """ debug("Inserting new osmatch into database") osmatch["line"] = empty() # ToFix: Parser isnt storing this self.cursor.execute( "INSERT INTO osmatch (name, accuracy, line, " "fk_host) VALUES (?, ?, ?, ?)", (osmatch["name"], osmatch["accuracy"], osmatch["line"], host))
def get_service_info_id_from_db(self, info, service_name_id): """ Get service_info id based on data from info and service_name_id. """ debug("Getting pk for service_info..") normalize(info) info["ostype"] = empty() # ToFix: Parser isnt storing this data = ( info["product"], info["version"], info["extrainfo"], info["method"], info["conf"], service_name_id) s_id = self.cursor.execute("SELECT pk FROM service_info " "WHERE product = ? AND version = ? AND extrainfo = ? AND " "method = ? AND conf = ? AND fk_service_name = ?", data).fetchone() if s_id: return s_id[0]
def get_service_info_id_from_db(self, info, service_name_id): """ Get service_info id based on data from info and service_name_id. """ debug("Getting pk for service_info..") normalize(info) info["ostype"] = empty() # ToFix: Parser isnt storing this data = (info["product"], info["version"], info["extrainfo"], info["method"], info["conf"], service_name_id) s_id = self.cursor.execute( "SELECT pk FROM service_info " "WHERE product = ? AND version = ? AND extrainfo = ? AND " "method = ? AND conf = ? AND fk_service_name = ?", data).fetchone() if s_id: return s_id[0]
def hosts_from_xml(self): """ Returns a list of dicts compatible with database schema for host, and insert hosts data. """ debug("Building host table...") hosts_l = [ ] for host in self.parsed.nmap["hosts"]: host_d = { } host_d["distance"] = empty() # ToFix: Parser not storing this. # get host_state fk host_state_id = self.get_host_state_id_from_db(host.state) if not host_state_id: self.insert_host_state_db(host.state) host_state_id = self.get_id_for("host_state") host_d["fk_host_state"] = host_state_id host_d["fk_scan"] = self.scan["pk"] # insert host self.insert_host_db(host_d) host_d["pk"] = self.get_id_for("host") hosts_l.append(host_d) # host fingerprint fp_d = { } fp_d["uptime"] = host.uptime["seconds"] fp_d["lastboot"] = host.uptime["lastboot"] if host.tcpsequence: fp_d["tcp_sequence_class"] = host.tcpsequence.get("class", '') fp_d["tcp_sequence_index"] = host.tcpsequence["index"] fp_d["tcp_sequence_value"] = host.tcpsequence["values"] fp_d["tcp_sequence_difficulty"] = host.tcpsequence["difficulty"] if host.tcptssequence: fp_d["tcp_ts_sequence_class"] = host.tcptssequence["class"] fp_d["tcp_ts_sequence_value"] = host.tcptssequence.get( "values", '') if host.ipidsequence: fp_d["ip_id_sequence_class"] = host.ipidsequence["class"] fp_d["ip_id_sequence_value"] = host.ipidsequence["values"] # insert fingerprint_info if len(fp_d) > 2: fp_d["fk_host"] = host_d["pk"] self.insert_fingerprint_info_db(fp_d) # insert hostnames for _host in host.hostnames: fk_hostname = self.get_hostname_id_from_db(_host) if not fk_hostname: self.insert_hostname_db(_host) fk_hostname = self.get_id_for("hostname") self.insert_host_hostname_db(host_d["pk"], fk_hostname) # insert host addresses (ipv4) if host.ip: normalize(host.ip) # get fk_vendor if not host.ip.get("vendor"): vendor = empty() else: vendor = host.ip["vendor"] fk_vendor = self.get_vendor_id_from_db(vendor) if not fk_vendor: self.insert_vendor_db(vendor) fk_vendor = self.get_id_for("vendor") # get fk_address fk_address = self.get_address_id_from_db(host.ip["addr"], host.ip["addrtype"], fk_vendor) if not fk_address: self.insert_address_db(host.ip["addr"], host.ip["addrtype"], fk_vendor) fk_address = self.get_id_for("address") # insert _host_address self.insert_host_address_db(host_d["pk"], fk_address) # insert host addresses (ipv6) if host.ipv6: normalize(host.ipv6) # get fk_vendor if not host.ipv6.get("vendor"): vendor = empty() else: vendor = host.ipv6["vendor"] fk_vendor = self.get_vendor_id_from_db(vendor) if not fk_vendor: self.insert_vendor_db(vendor) fk_vendor = self.get_id_for("vendor") # get fk_address fk_address = self.get_address_id_from_db(host.ipv6["addr"], host.ipv6["addrtype"], fk_vendor) if not fk_address: self.insert_address_db(host.ipv6["addr"], host.ipv6["addrtype"], fk_vendor) fk_address = self.get_id_for("address") # insert _host_address self.insert_host_address_db(host_d["pk"], fk_address) # insert host addresses (mac) if host.mac: normalize(host.mac) # get fk_vendor if not host.ip.get("vendor"): vendor = empty() else: vendor = host.mac["vendor"] fk_vendor = self.get_vendor_id_from_db(vendor) if not fk_vendor: self.insert_vendor_db(vendor) fk_vendor = self.get_id_for("vendor") # get fk_address fk_address = self.get_address_id_from_db(host.mac["addr"], host.mac["addrtype"], fk_vendor) if not fk_address: self.insert_address_db(host.mac["addr"], host.mac["addrtype"], fk_vendor) fk_address = self.get_id_for("address") # insert _host_address self.insert_host_address_db(host_d["pk"], fk_address) # insert host os match if host.osmatch: # ToFix: Parser is returning only last osmatch ? # XXX the new parser stores all osmatches but this is still # using only the last osmatch -- for now. self.insert_osmatch_db(host_d["pk"], host.osmatch[-1]) # insert os classes for osclass in host.osclass: # get fk_osgen osclass_osgen = osclass.get('osgen', '') osgen_id = self.get_osgen_id_from_db(osclass_osgen) if not osgen_id: self.insert_osgen_db(osclass_osgen) osgen_id = self.get_id_for("osgen") # get fk_osfamily osfamily_id = self.get_osfamily_id_from_db(osclass["osfamily"]) if not osfamily_id: self.insert_osfamily_db(osclass["osfamily"]) osfamily_id = self.get_id_for("osfamily") # get fk_osvendor osvendor_id = self.get_osvendor_id_from_db(osclass["vendor"]) if not osvendor_id: self.insert_osvendor_db(osclass["vendor"]) osvendor_id = self.get_id_for("osvendor") # get fk_ostype osclass_type = osclass.get('type', '') ostype_id = self.get_ostype_id_from_db(osclass_type) if not ostype_id: self.insert_ostype_db(osclass_type) ostype_id = self.get_id_for("ostype") self.insert_osclass_db(osclass["accuracy"], osgen_id, osfamily_id, osvendor_id, ostype_id, host_d["pk"]) # insert ports used if host.portused: for portused in host.portused: # get fk_port_state port_state_id = self.get_port_state_id_from_db( portused["state"]) if not port_state_id: self.insert_port_state_db(portused["state"]) port_state_id = self.get_id_for("port_state") # get fk_protocol port_protocol_id = self.get_protocol_id_from_db( portused["proto"]) if not port_protocol_id: self.insert_protocol_db(portused["proto"]) port_protocol_id = self.get_id_for("protocol") # insert portused self.insert_portused_db(portused["portid"], port_state_id, port_protocol_id, host_d["pk"]) # some scan may not return any ports if host.extraports: # insert extraports for extraport in host.extraports: port_state = self.get_port_state_id_from_db( extraport["state"]) if not port_state: self.insert_port_state_db(extraport["state"]) port_state = self.get_id_for("port_state") self.insert_extraports_db(extraport["count"], host_d["pk"], port_state) if host.ports: # insert ports for port in host.ports: # get fk_protocol protocol_id = self.get_protocol_id_from_db(port["protocol"]) if not protocol_id: self.insert_protocol_db(port["protocol"]) protocol_id = self.get_id_for("protocol") # get fk_port_state port_state_id = self.get_port_state_id_from_db( port["state"]) if not port_state_id: self.insert_port_state_db(port["state"]) port_state_id = self.get_id_for("port_state") if not "name" in port: port["name"] = empty() service_name_id = self.get_service_name_id_from_db( port["name"]) if not service_name_id: self.insert_service_name_db(port["name"]) service_name_id = self.get_id_for("service_name") # get fk_service_info keys = ["product", "version", "extrainfo", "method", "conf"] for k in keys: if not k in port: port[k] = empty() service_info_id = self.get_service_info_id_from_db(port, service_name_id) if not service_info_id: data = ( port["product"], port["version"], port["extrainfo"], port["method"], port["conf"], service_name_id) self.insert_service_info_db(data) service_info_id = self.get_id_for("service_info") # get fk_port port_id = self.get_port_id_from_db(port["portid"], service_info_id, protocol_id, port_state_id) if not port_id: self.insert_port_db(port["portid"], service_info_id, protocol_id, port_state_id) port_id = self.get_id_for("port") # insert _host_port self.insert_host_port_db(host_d["pk"], port_id) return hosts_l