def scan_from_xml(self):
"""
Returns a dict compatible with database schema for scan and
insert scan data.
"""
debug("Building scan table..")
parsedsax = self.parsed
scan_d = { }
scan_d["args"] = parsedsax.nmap_command
timestamp_start = parsedsax.start
scan_d["start"] = datetime.fromtimestamp(float(timestamp_start))
scan_d["startstr"] = empty() # ToFix: Parser isnt storing this
scan_d["finish"] = datetime(*parsedsax.finish_epoch_time[:6])
scan_d["finishstr"] = parsedsax.finish_time
scan_d["xmloutputversion"] = (
parsedsax.nmap["nmaprun"]["xmloutputversion"])
if self.store_original:
scan_d["xmloutput"] = '\n'.join(open(self.xml_file,
'r').readlines())
else:
scan_d["xmloutput"] = empty()
scan_d["verbose"] = parsedsax.verbose_level
scan_d["debugging"] = parsedsax.debugging_level
scan_d["hosts_up"] = parsedsax.hosts_up
scan_d["hosts_down"] = parsedsax.hosts_down
scanner_name = parsedsax.scanner
scanner_version = parsedsax.scanner_version
# get fk_scanner
scanner_id = self.get_scanner_id_from_db(scanner_name,
scanner_version)
if not scanner_id:
self.insert_scanner_db(scanner_name, scanner_version)
scanner_id = self.get_id_for("scanner")
scan_d["scanner"] = scanner_id
self.insert_scan_db(scan_d)
# get pk for the just inserted scan.
scan_d["pk"] = self.get_id_for("scan")
return scan_d
def insert_osmatch_db(self, host, osmatch):
"""
Create new record in osmatch with data from osmatch dict.
"""
debug("Inserting new osmatch into database")
osmatch["line"] = empty() # ToFix: Parser isnt storing this
self.cursor.execute("INSERT INTO osmatch (name, accuracy, line, "
"fk_host) VALUES (?, ?, ?, ?)", (osmatch["name"],
osmatch["accuracy"], osmatch["line"], host))
def insert_osmatch_db(self, host, osmatch):
"""
Create new record in osmatch with data from osmatch dict.
"""
debug("Inserting new osmatch into database")
osmatch["line"] = empty() # ToFix: Parser isnt storing this
self.cursor.execute(
"INSERT INTO osmatch (name, accuracy, line, "
"fk_host) VALUES (?, ?, ?, ?)",
(osmatch["name"], osmatch["accuracy"], osmatch["line"], host))
def get_service_info_id_from_db(self, info, service_name_id):
"""
Get service_info id based on data from info and service_name_id.
"""
debug("Getting pk for service_info..")
normalize(info)
info["ostype"] = empty() # ToFix: Parser isnt storing this
data = (
info["product"], info["version"],
info["extrainfo"], info["method"],
info["conf"], service_name_id)
s_id = self.cursor.execute("SELECT pk FROM service_info "
"WHERE product = ? AND version = ? AND extrainfo = ? AND "
"method = ? AND conf = ? AND fk_service_name = ?", data).fetchone()
if s_id:
return s_id[0]
def get_service_info_id_from_db(self, info, service_name_id):
"""
Get service_info id based on data from info and service_name_id.
"""
debug("Getting pk for service_info..")
normalize(info)
info["ostype"] = empty() # ToFix: Parser isnt storing this
data = (info["product"], info["version"], info["extrainfo"],
info["method"], info["conf"], service_name_id)
s_id = self.cursor.execute(
"SELECT pk FROM service_info "
"WHERE product = ? AND version = ? AND extrainfo = ? AND "
"method = ? AND conf = ? AND fk_service_name = ?",
data).fetchone()
if s_id:
return s_id[0]
def hosts_from_xml(self):
"""
Returns a list of dicts compatible with database schema for host,
and insert hosts data.
"""
debug("Building host table...")
hosts_l = [ ]
for host in self.parsed.nmap["hosts"]:
host_d = { }
host_d["distance"] = empty() # ToFix: Parser not storing this.
# get host_state fk
host_state_id = self.get_host_state_id_from_db(host.state)
if not host_state_id:
self.insert_host_state_db(host.state)
host_state_id = self.get_id_for("host_state")
host_d["fk_host_state"] = host_state_id
host_d["fk_scan"] = self.scan["pk"]
# insert host
self.insert_host_db(host_d)
host_d["pk"] = self.get_id_for("host")
hosts_l.append(host_d)
# host fingerprint
fp_d = { }
fp_d["uptime"] = host.uptime["seconds"]
fp_d["lastboot"] = host.uptime["lastboot"]
if host.tcpsequence:
fp_d["tcp_sequence_class"] = host.tcpsequence.get("class", '')
fp_d["tcp_sequence_index"] = host.tcpsequence["index"]
fp_d["tcp_sequence_value"] = host.tcpsequence["values"]
fp_d["tcp_sequence_difficulty"] = host.tcpsequence["difficulty"]
if host.tcptssequence:
fp_d["tcp_ts_sequence_class"] = host.tcptssequence["class"]
fp_d["tcp_ts_sequence_value"] = host.tcptssequence.get(
"values", '')
if host.ipidsequence:
fp_d["ip_id_sequence_class"] = host.ipidsequence["class"]
fp_d["ip_id_sequence_value"] = host.ipidsequence["values"]
# insert fingerprint_info
if len(fp_d) > 2:
fp_d["fk_host"] = host_d["pk"]
self.insert_fingerprint_info_db(fp_d)
# insert hostnames
for _host in host.hostnames:
fk_hostname = self.get_hostname_id_from_db(_host)
if not fk_hostname:
self.insert_hostname_db(_host)
fk_hostname = self.get_id_for("hostname")
self.insert_host_hostname_db(host_d["pk"], fk_hostname)
# insert host addresses (ipv4)
if host.ip:
normalize(host.ip)
# get fk_vendor
if not host.ip.get("vendor"):
vendor = empty()
else:
vendor = host.ip["vendor"]
fk_vendor = self.get_vendor_id_from_db(vendor)
if not fk_vendor:
self.insert_vendor_db(vendor)
fk_vendor = self.get_id_for("vendor")
# get fk_address
fk_address = self.get_address_id_from_db(host.ip["addr"],
host.ip["addrtype"], fk_vendor)
if not fk_address:
self.insert_address_db(host.ip["addr"],
host.ip["addrtype"], fk_vendor)
fk_address = self.get_id_for("address")
# insert _host_address
self.insert_host_address_db(host_d["pk"], fk_address)
# insert host addresses (ipv6)
if host.ipv6:
normalize(host.ipv6)
# get fk_vendor
if not host.ipv6.get("vendor"):
vendor = empty()
else:
vendor = host.ipv6["vendor"]
fk_vendor = self.get_vendor_id_from_db(vendor)
if not fk_vendor:
self.insert_vendor_db(vendor)
fk_vendor = self.get_id_for("vendor")
# get fk_address
fk_address = self.get_address_id_from_db(host.ipv6["addr"],
host.ipv6["addrtype"], fk_vendor)
if not fk_address:
self.insert_address_db(host.ipv6["addr"],
host.ipv6["addrtype"], fk_vendor)
fk_address = self.get_id_for("address")
# insert _host_address
self.insert_host_address_db(host_d["pk"], fk_address)
# insert host addresses (mac)
if host.mac:
normalize(host.mac)
# get fk_vendor
if not host.ip.get("vendor"):
vendor = empty()
else:
vendor = host.mac["vendor"]
fk_vendor = self.get_vendor_id_from_db(vendor)
if not fk_vendor:
self.insert_vendor_db(vendor)
fk_vendor = self.get_id_for("vendor")
# get fk_address
fk_address = self.get_address_id_from_db(host.mac["addr"],
host.mac["addrtype"], fk_vendor)
if not fk_address:
self.insert_address_db(host.mac["addr"],
host.mac["addrtype"], fk_vendor)
fk_address = self.get_id_for("address")
# insert _host_address
self.insert_host_address_db(host_d["pk"], fk_address)
# insert host os match
if host.osmatch:
# ToFix: Parser is returning only last osmatch ?
# XXX the new parser stores all osmatches but this is still
# using only the last osmatch -- for now.
self.insert_osmatch_db(host_d["pk"], host.osmatch[-1])
# insert os classes
for osclass in host.osclass:
# get fk_osgen
osclass_osgen = osclass.get('osgen', '')
osgen_id = self.get_osgen_id_from_db(osclass_osgen)
if not osgen_id:
self.insert_osgen_db(osclass_osgen)
osgen_id = self.get_id_for("osgen")
# get fk_osfamily
osfamily_id = self.get_osfamily_id_from_db(osclass["osfamily"])
if not osfamily_id:
self.insert_osfamily_db(osclass["osfamily"])
osfamily_id = self.get_id_for("osfamily")
# get fk_osvendor
osvendor_id = self.get_osvendor_id_from_db(osclass["vendor"])
if not osvendor_id:
self.insert_osvendor_db(osclass["vendor"])
osvendor_id = self.get_id_for("osvendor")
# get fk_ostype
osclass_type = osclass.get('type', '')
ostype_id = self.get_ostype_id_from_db(osclass_type)
if not ostype_id:
self.insert_ostype_db(osclass_type)
ostype_id = self.get_id_for("ostype")
self.insert_osclass_db(osclass["accuracy"], osgen_id,
osfamily_id, osvendor_id, ostype_id, host_d["pk"])
# insert ports used
if host.portused:
for portused in host.portused:
# get fk_port_state
port_state_id = self.get_port_state_id_from_db(
portused["state"])
if not port_state_id:
self.insert_port_state_db(portused["state"])
port_state_id = self.get_id_for("port_state")
# get fk_protocol
port_protocol_id = self.get_protocol_id_from_db(
portused["proto"])
if not port_protocol_id:
self.insert_protocol_db(portused["proto"])
port_protocol_id = self.get_id_for("protocol")
# insert portused
self.insert_portused_db(portused["portid"], port_state_id,
port_protocol_id, host_d["pk"])
# some scan may not return any ports
if host.extraports:
# insert extraports
for extraport in host.extraports:
port_state = self.get_port_state_id_from_db(
extraport["state"])
if not port_state:
self.insert_port_state_db(extraport["state"])
port_state = self.get_id_for("port_state")
self.insert_extraports_db(extraport["count"], host_d["pk"],
port_state)
if host.ports:
# insert ports
for port in host.ports:
# get fk_protocol
protocol_id = self.get_protocol_id_from_db(port["protocol"])
if not protocol_id:
self.insert_protocol_db(port["protocol"])
protocol_id = self.get_id_for("protocol")
# get fk_port_state
port_state_id = self.get_port_state_id_from_db(
port["state"])
if not port_state_id:
self.insert_port_state_db(port["state"])
port_state_id = self.get_id_for("port_state")
if not "name" in port:
port["name"] = empty()
service_name_id = self.get_service_name_id_from_db(
port["name"])
if not service_name_id:
self.insert_service_name_db(port["name"])
service_name_id = self.get_id_for("service_name")
# get fk_service_info
keys = ["product", "version", "extrainfo", "method", "conf"]
for k in keys:
if not k in port:
port[k] = empty()
service_info_id = self.get_service_info_id_from_db(port,
service_name_id)
if not service_info_id:
data = (
port["product"],
port["version"],
port["extrainfo"],
port["method"],
port["conf"], service_name_id)
self.insert_service_info_db(data)
service_info_id = self.get_id_for("service_info")
# get fk_port
port_id = self.get_port_id_from_db(port["portid"],
service_info_id, protocol_id, port_state_id)
if not port_id:
self.insert_port_db(port["portid"], service_info_id,
protocol_id, port_state_id)
port_id = self.get_id_for("port")
# insert _host_port
self.insert_host_port_db(host_d["pk"], port_id)
return hosts_l
