def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nmap XML Scan file
import re
import os
from skaldship.general import get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
from zenmapCore_Kvasir.NmapParser import NmapParser
# output regexes
RE_NETBIOS_NAME = re.compile("NetBIOS computer name: (?P<d>.*),")
RE_NETBIOS_WORKGROUP = re.compile("Workgroup: (?P<d>.*),")
RE_NETBIOS_MAC = re.compile("NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split("\r\n")
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split("\r\n")
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nmap scan file %s" % (filename))
nmap_parsed = NmapParser()
nmap_parsed.parse_file(filename)
# existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the hosts, where all the goodies are
log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts)))
hoststats = {}
hoststats["added"] = 0
hoststats["skipped"] = 0
hoststats["updated"] = 0
hoststats["errored"] = 0
hosts = [] # array of host_id fields
svc_db = db.t_services
for node in nmap_parsed.hosts:
nodefields = {}
if node.ipv6:
ipaddr = node.ipv6
nodefields["f_ipv4"] = ipaddr
elif node.ip.get("type") == "ipv4":
ipaddr = node.ip.get("addr")
nodefields["f_ipv4"] = ipaddr
else:
log(" [!] No IPv4/IPv6 address, skipping")
continue
try:
nodefields["f_macaddr"] = node.mac["addr"]
except TypeError:
nodefields["f_macaddr"] = None
status = node.state
log(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats["skipped"] += 1
continue
if ipaddr in ip_exclude:
log(" [-] Host is in exclude list... skipping")
hoststats["skipped"] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Host is not in the only list... skipping")
hoststats["skipped"] += 1
continue
if not node.ports and not addnoports:
log(" [-] No ports open and not asked to add those kind... skipping")
hoststats["skipped"] += 1
continue
# we'lll just take the last hostname in the names list since it'll usually be the full dns name
for name in node.hostnames:
nodefields["f_hostname"] = name["hostname"]
nodefields["f_engineer"] = engineer
nodefields["f_asset_group"] = asset_group
nodefields["f_confirmed"] = False
# see if host exists, if so update. if not, insert!
query = (db.t_hosts.f_ipv4 == ipaddr) | (db.t_hosts.f_ipv6 == ipaddr)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats["added"] += 1
log(" [-] Adding %s" % (ipaddr))
elif host_rec is not None and update_hosts:
db.commit()
if "f_ipv4" in nodefields:
host_id = db(db.t_hosts.f_ipv4 == nodefields["f_ipv4"]).update(**nodefields)
else:
host_id = db(db.t_hosts.f_ipv6 == nodefields["f_ipv6"]).update(**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats["updated"] += 1
log(" [-] Updating %s" % (ipaddr))
else:
hoststats["skipped"] += 1
db.commit()
log(" [-] Skipped %s" % (ipaddr))
continue
hosts.append(host_id)
# process OS related info
for os in node.osmatches:
os_id = None
host_id = None
f_title = os["name"] # title
for k in os["osclasses"]:
f_cpename = k["cpe"].lstrip("cpe:/o:")
f_vendor = k["vendor"]
f_product = k["osfamily"]
f_version = k["osgen"]
f_class = k["type"]
f_family = k["osfamily"]
f_certainty = k["accuracy"]
cpe_res = db((db.t_os.f_cpename == f_cpename) & (db.t_os.f_title == f_title)).select().first()
if cpe_res is not None:
os_id = cpe_res.id
else:
try:
os_id = db.t_os.insert(
f_cpename=f_cpename,
f_title=f_title,
f_vendor=f_vendor,
f_product=f_product,
f_version=f_version,
)
except Exception, e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
if os_id and (f_class or f_family or f_certainty):
ipaddr = node.ip.get("addr")
host_id = get_host_record(ipaddr)
host_id = host_id.id
try:
db.t_host_os_refs.insert(
f_certainty=f_certainty,
f_family=f_family,
f_class=f_class,
f_hosts_id=host_id,
f_os_id=os_id,
)
except Exception, e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_workspace=False,
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process nMap XML Scan file
import re
from MetasploitAPI import MetasploitAPI
from skaldship.general import get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
from zenmapCore_Kvasir.NmapParser import NmapParser
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS computer name: (?P<d>.*),')
RE_NETBIOS_WORKGROUP = re.compile('Workgroup: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile('NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing nMap scan file %s" % (filename))
nmap_parsed = NmapParser()
nmap_parsed.parse_file(filename)
#existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
user_id = db.auth_user(engineer) or auth.user.id
# parse the hosts, where all the goodies are
log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts)))
hoststats = {}
hoststats['added'] = 0
hoststats['skipped'] = 0
hoststats['updated'] = 0
hoststats['errored'] = 0
hosts = [] # array of host_id fields
svc_db = db.t_services
for node in nmap_parsed.hosts:
nodefields = {}
if node.ipv6:
ipaddr = node.ipv6
nodefields['f_ipv4'] = ipaddr
elif node.ip.get('type') == 'ipv4':
ipaddr = node.ip.get('addr')
nodefields['f_ipv4'] = ipaddr
else:
log(" [!] No IPv4/IPv6 address, skipping")
continue
nodefields['f_macaddr'] = node.mac
status = node.state
log(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats['skipped'] += 1
continue
if ipaddr in ip_exclude:
log(" [-] Host is in exclude list... skipping")
hoststats['skipped'] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Host is not in the only list... skipping")
hoststats['skipped'] += 1
continue
if not node.ports and not addnoports:
log(" [-] No ports open and not asked to add those kind... skipping")
hoststats['skipped'] += 1
continue
# we'lll just take the last hostname in the names list since it'll usually be the full dns name
for name in node.hostnames:
nodefields['f_hostname'] = name
nodefields['f_engineer'] = user_id
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
# check to see if IPv4/IPv6 exists in DB already
if 'f_ipv4' in nodefields:
host_rec = db(db.t_hosts.f_ipv4 == nodefields['f_ipv4']).select().first()
elif 'f_ipv6' in nodefields:
host_rec = db(db.t_hosts.f_ipv6 == nodefields['f_ipv6']).select().first()
else:
log("No IP Address found in record. Skipping", logging.ERROR)
continue
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
log(" [-] Adding %s" % (ipaddr))
elif host_rec is not None and update_hosts:
db.commit()
if 'f_ipv4' in nodefields:
host_id = db(db.t_hosts.f_ipv4 == nodefields['f_ipv4']).update(**nodefields)
else:
host_id = db(db.t_hosts.f_ipv6 == nodefields['f_ipv6']).update(**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats['updated'] += 1
log(" [-] Updating %s" % (ipaddr))
else:
hoststats['skipped'] += 1
db.commit()
log(" [-] Skipped %s" % (ipaddr))
continue
hosts.append(host_id)
# process non-port <hostscript> entries. Add to info/0:
for hostscripts in node.hostscripts:
query = (svc_db.f_proto == 'info') & (svc_db.f_number == 0) & (svc_db.f_hosts_id == host_id)
svc_id = db.t_services.update_or_insert(query, f_proto='info', f_number=0, f_status='open', f_hosts_id=host_id)
if not svc_id:
svc_rec = db(query).select(cache=(cache.ram, 180)).first()
if svc_rec:
svc_id = svc_rec.id
else:
log(" [!] Service record wasn't created", logging.ERROR)
continue
db.commit()
for script in hostscripts:
script_id = script.id
output = script.output
db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=script_id, f_text=output)
db.commit()
if script_id == 'nbstat':
# pull out NetBIOS info from nbstat output
result = RE_NETBIOS_MAC.search(output)
if 'd' in result.groupdict():
host_rec.update(f_macaddr=result.group('d'))
db.commit()
result = RE_NETBIOS_NAME.search(output)
if 'd' in result.groupdict():
host_rec.update(f_netbios_name=result.group('d'))
db.commit()
result = RE_NETBIOS_WORKGROUP.search(output)
if 'd' in result.groupdict():
db(db.t_netbios.update_or_insert(f_hosts_id=host_id, f_domain=result.group('d')))
db.commit()
# add ports and resulting vulndata
for port in node.ports:
f_proto = port.get('protocol')
f_number = port.get('portid')
f_status = port.get('port_state')
f_name = port.get('service_name')
f_product = port.get('service_product')
log(" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name))
svc_id = db.t_services.update_or_insert(f_proto=f_proto, f_number=f_number, f_status=f_status, f_hosts_id=host_id, f_name=f_name)
if f_product:
version = port.get('service_version')
if version:
f_product += " (%s)" % (version)
db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=f_name, f_text=f_product)
db.commit()
# Process <script> service entries
for script in port.get('scripts'):
db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=script.get('id'), f_text=script.get('output'))
db.commit()
# Process <cpe> service entries
for port_cpe in port.get('service_cpe'):
cpe_id = port_cpe.text.lstrip('cpe:/')
if cpe_id[0] == "a":
# process CPE Applications
#log(" [-] Found Application CPE data: %s" % (cpe_id))
db.t_service_info.update_or_insert(f_services_id=svc_id, f_name='cpe.app', f_text="cpe:/%s" % (cpe_id))
db.commit()
elif cpe_id[0] == "o":
# process CPE Operating System
os_id = lookup_cpe(cpe_id[2:])
if os_id is not None:
db.t_host_os_refs.insert(f_certainty='0.9',
f_family='Unknown',
f_class='Other',
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
# So no CPE or existing OS data, lets split up the CPE data and make our own
log(" [!] No os_id found, this is odd !!!")
if msf_workspace:
msf = MetasploitAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key)
if msf.login():
try:
res = msf.pro_import_file(
msf_workspace,
filename,
{
'DS_REMOVE_FILE': False,
'tag': asset_group,
},
)
log(" [*] Added file to MSF Pro: %s" % (res))
except MetasploitAPI.MSFAPIError, e:
logging.error("MSFAPI Error: %s" % (e))
pass
else:
log(" [!] Unable to login to Metasploit PRO, check your API key", logging.ERROR)
msf = None
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nmap XML Scan file
import re
import os
from skaldship.general import get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
from zenmapCore_Kvasir.NmapParser import NmapParser
# output regexes
RE_NETBIOS_NAME = re.compile("NetBIOS computer name: (?P<d>.*),")
RE_NETBIOS_WORKGROUP = re.compile("Workgroup: (?P<d>.*),")
RE_NETBIOS_MAC = re.compile("NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split("\r\n")
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split("\r\n")
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nmap scan file %s" % (filename))
nmap_parsed = NmapParser()
nmap_parsed.parse_file(filename)
# existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the hosts, where all the goodies are
log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts)))
hoststats = {}
hoststats["added"] = 0
hoststats["skipped"] = 0
hoststats["updated"] = 0
hoststats["errored"] = 0
hosts = [] # array of host_id fields
svc_db = db.t_services
for node in nmap_parsed.hosts:
nodefields = {}
if node.ipv6:
ipaddr = node.ipv6
nodefields["f_ipv4"] = ipaddr
elif node.ip.get("type") == "ipv4":
ipaddr = node.ip.get("addr")
nodefields["f_ipv4"] = ipaddr
else:
log(" [!] No IPv4/IPv6 address, skipping")
continue
try:
nodefields["f_macaddr"] = node.mac["addr"]
except TypeError:
nodefields["f_macaddr"] = None
status = node.state
log(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats["skipped"] += 1
continue
if ipaddr in ip_exclude:
log(" [-] Host is in exclude list... skipping")
hoststats["skipped"] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Host is not in the only list... skipping")
hoststats["skipped"] += 1
continue
if not node.ports and not addnoports:
log(" [-] No ports open and not asked to add those kind... skipping")
hoststats["skipped"] += 1
continue
# we'lll just take the last hostname in the names list since it'll usually be the full dns name
for name in node.hostnames:
nodefields["f_hostname"] = name["hostname"]
nodefields["f_engineer"] = engineer
nodefields["f_asset_group"] = asset_group
nodefields["f_confirmed"] = False
# see if host exists, if so update. if not, insert!
query = (db.t_hosts.f_ipv4 == ipaddr) | (db.t_hosts.f_ipv6 == ipaddr)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats["added"] += 1
log(" [-] Adding %s" % (ipaddr))
elif host_rec is not None and update_hosts:
db.commit()
if "f_ipv4" in nodefields:
host_id = db(db.t_hosts.f_ipv4 == nodefields["f_ipv4"]).update(**nodefields)
else:
host_id = db(db.t_hosts.f_ipv6 == nodefields["f_ipv6"]).update(**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats["updated"] += 1
log(" [-] Updating %s" % (ipaddr))
else:
hoststats["skipped"] += 1
db.commit()
log(" [-] Skipped %s" % (ipaddr))
continue
hosts.append(host_id)
# process non-port <hostscript> entries. Add to info/0:
for hostscripts in node.hostscripts:
query = (svc_db.f_proto == "info") & (svc_db.f_number == 0) & (svc_db.f_hosts_id == host_id)
svc_id = db.t_services.update_or_insert(
query, f_proto="info", f_number=0, f_status="open", f_hosts_id=host_id
)
if not svc_id:
svc_rec = db(query).select(cache=(cache.ram, 180)).first()
if svc_rec:
svc_id = svc_rec.id
else:
log(" [!] Service record wasn't created", logging.ERROR)
continue
db.commit()
for script in hostscripts:
script_id = script.id
output = script.output
db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=script_id, f_text=output)
db.commit()
if script_id == "nbstat":
# pull out NetBIOS info from nbstat output
result = RE_NETBIOS_MAC.search(output)
if "d" in result.groupdict():
host_rec.update(f_macaddr=result.group("d"))
db.commit()
result = RE_NETBIOS_NAME.search(output)
if "d" in result.groupdict():
host_rec.update(f_netbios_name=result.group("d"))
db.commit()
result = RE_NETBIOS_WORKGROUP.search(output)
if "d" in result.groupdict():
db(db.t_netbios.update_or_insert(f_hosts_id=host_id, f_domain=result.group("d")))
db.commit()
# add ports and resulting vulndata
for port in node.ports:
f_proto = port.get("protocol")
f_number = port.get("portid")
f_status = port.get("port_state")
f_name = port.get("service_name")
f_product = port.get("service_product")
log(" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name))
svc_id = db.t_services.update_or_insert(
f_proto=f_proto, f_number=f_number, f_status=f_status, f_hosts_id=host_id, f_name=f_name
)
if f_product:
version = port.get("service_version")
if version:
f_product += " (%s)" % (version)
db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=f_name, f_text=f_product)
db.commit()
# Process <script> service entries
for script in port.get("scripts"):
db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=script.get("id"), f_text=script.get("output")
)
db.commit()
# Process <cpe> service entries
port_cpe = port.get("service_cpe")
if port_cpe:
cpe_id = port_cpe.lstrip("cpe:/")
if cpe_id.startswith("a"):
# process CPE Applications
# log(" [-] Found Application CPE data: %s" % (cpe_id))
db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name="cpe.app", f_text="cpe:/%s" % (cpe_id)
)
db.commit()
elif cpe_id.startswith("o"):
# process CPE Operating System
os_id = lookup_cpe(cpe_id[2:])
if os_id is not None:
db.t_host_os_refs.insert(
f_certainty="0.9", f_family="Unknown", f_class="Other", f_hosts_id=host_id, f_os_id=os_id
)
db.commit()
else:
# So no CPE or existing OS data, lets split up the CPE data and make our own
log(" [!] No os_id found, this is odd !!!")
if msf_settings.get("workspace"):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitAPI import MetasploitAPI
msf_api = MetasploitAPI(host=msf_settings.get("url"), apikey=msf_settings.get("key"))
working_msf_api = msf_api.login()
except Exception, error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error), logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception, error:
log(" [!] Error loading scan data to send to Metasploit: %s" % str(error), logging.ERROR)
scan_data = None
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nmap XML Scan file
import re
import os
from skaldship.hosts import get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
from zenmapCore_Kvasir.NmapParser import NmapParser
from gluon import current
T = current.T
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS computer name: (?P<d>.*),')
RE_NETBIOS_WORKGROUP = re.compile('Workgroup: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile(
'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nmap scan file %s" % (filename))
nmap_parsed = NmapParser()
nmap_parsed.parse_file(filename)
#existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the hosts, where all the goodies are
log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts)))
hoststats = {}
hoststats['added'] = 0
hoststats['skipped'] = 0
hoststats['updated'] = 0
hoststats['errored'] = 0
hosts = [] # array of host_id fields
svc_db = db.t_services
for node in nmap_parsed.hosts:
nodefields = {}
if node.ipv6:
ipaddr = node.ipv6
nodefields['f_ipaddr'] = ipaddr
elif node.ip.get('type') == 'ipv4':
ipaddr = node.ip.get('addr')
nodefields['f_ipaddr'] = ipaddr
else:
log(" [!] No IPv4/IPv6 address, skipping")
continue
try:
nodefields['f_macaddr'] = node.mac['addr']
except TypeError:
nodefields['f_macaddr'] = None
status = node.state
log(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats['skipped'] += 1
continue
if ipaddr in ip_exclude:
log(" [-] Host is in exclude list... skipping")
hoststats['skipped'] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Host is not in the only list... skipping")
hoststats['skipped'] += 1
continue
if not node.ports and not addnoports:
log(" [-] No ports open and not asked to add those kind... skipping"
)
hoststats['skipped'] += 1
continue
# we'll just take the last hostname in the names list since it'll usually be the full dns name
for name in node.hostnames:
nodefields['f_hostname'] = name['hostname']
nodefields['f_engineer'] = engineer
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
# see if host exists, if so update. if not, insert!
query = (db.t_hosts.f_ipaddr == ipaddr)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
log(" [-] Adding %s" % (ipaddr))
elif host_rec is not None and update_hosts:
db.commit()
host_id = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats['updated'] += 1
log(" [-] Updating %s" % (ipaddr))
else:
hoststats['skipped'] += 1
db.commit()
log(" [-] Skipped %s" % (ipaddr))
continue
hosts.append(host_id)
# process OS related info
for os in node.osmatches:
os_id = None
host_id = None
f_title = os['name'] #title
for k in os['osclasses']:
if k.get('cpe') != None: #fixes error on loading cpe:/os
f_cpename = k['cpe'].replace('cpe:/o:', '')
f_vendor = k['vendor']
f_product = k['osfamily']
f_version = k['osgen']
f_class = k['type']
f_family = k['osfamily']
f_certainty = k['accuracy']
cpe_res = db((db.t_os.f_cpename == f_cpename) & (
db.t_os.f_title == f_title)).select().first()
if cpe_res is not None:
os_id = cpe_res.id
else:
try:
os_id = db.t_os.insert(
f_cpename=f_cpename,
f_title=f_title,
f_vendor=f_vendor,
f_product=f_product,
f_version=f_version,
)
except Exception, e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
if os_id and (f_class or f_family or f_certainty):
ipaddr = node.ip.get('addr')
host_id = get_host_record(ipaddr)
host_id = host_id.id
try:
db.t_host_os_refs.insert(f_certainty=f_certainty,
f_family=f_family,
f_class=f_class,
f_hosts_id=host_id,
f_os_id=os_id)
except Exception, e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nmap XML Scan file
import re
import os
from skaldship.hosts import get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
from zenmapCore_Kvasir.NmapParser import NmapParser
from gluon import current
T = current.T
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS computer name: (?P<d>.*),')
RE_NETBIOS_WORKGROUP = re.compile('Workgroup: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile(
'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nmap scan file %s" % (filename))
nmap_parsed = NmapParser()
nmap_parsed.parse_file(filename)
#existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the hosts, where all the goodies are
log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts)))
hoststats = {}
hoststats['added'] = 0
hoststats['skipped'] = 0
hoststats['updated'] = 0
hoststats['errored'] = 0
hosts = [] # array of host_id fields
svc_db = db.t_services
for node in nmap_parsed.hosts:
nodefields = {}
if node.ipv6:
ipaddr = node.ipv6
nodefields['f_ipaddr'] = ipaddr
elif node.ip.get('type') == 'ipv4':
ipaddr = node.ip.get('addr')
nodefields['f_ipaddr'] = ipaddr
else:
log(" [!] No IPv4/IPv6 address, skipping")
continue
try:
nodefields['f_macaddr'] = node.mac['addr']
except TypeError:
nodefields['f_macaddr'] = None
status = node.state
log(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats['skipped'] += 1
continue
if ipaddr in ip_exclude:
log(" [-] Host is in exclude list... skipping")
hoststats['skipped'] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Host is not in the only list... skipping")
hoststats['skipped'] += 1
continue
if not node.ports and not addnoports:
log(" [-] No ports open and not asked to add those kind... skipping"
)
hoststats['skipped'] += 1
continue
# we'll just take the last hostname in the names list since it'll usually be the full dns name
for name in node.hostnames:
nodefields['f_hostname'] = name['hostname']
nodefields['f_engineer'] = engineer
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
# see if host exists, if so update. if not, insert!
query = (db.t_hosts.f_ipaddr == ipaddr)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
log(" [-] Adding %s" % (ipaddr))
elif host_rec is not None and update_hosts:
db.commit()
host_id = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats['updated'] += 1
log(" [-] Updating %s" % (ipaddr))
else:
hoststats['skipped'] += 1
db.commit()
log(" [-] Skipped %s" % (ipaddr))
continue
hosts.append(host_id)
# process OS related info
for os in node.osmatches:
os_id = None
host_id = None
f_title = os['name'] #title
for k in os['osclasses']:
if k.get('cpe') != None: #fixes error on loading cpe:/os
f_cpename = k['cpe'].replace('cpe:/o:', '')
f_vendor = k['vendor']
f_product = k['osfamily']
f_version = k['osgen']
f_class = k['type']
f_family = k['osfamily']
f_certainty = k['accuracy']
cpe_res = db((db.t_os.f_cpename == f_cpename) & (
db.t_os.f_title == f_title)).select().first()
if cpe_res is not None:
os_id = cpe_res.id
else:
try:
os_id = db.t_os.insert(
f_cpename=f_cpename,
f_title=f_title,
f_vendor=f_vendor,
f_product=f_product,
f_version=f_version,
)
except Exception as e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
if os_id and (f_class or f_family or f_certainty):
ipaddr = node.ip.get('addr')
host_id = get_host_record(ipaddr)
host_id = host_id.id
try:
db.t_host_os_refs.insert(f_certainty=f_certainty,
f_family=f_family,
f_class=f_class,
f_hosts_id=host_id,
f_os_id=os_id)
except Exception as e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
# process non-port <hostscript> entries. Add to info/0:
for hostscripts in node.hostscripts:
query = (svc_db.f_proto == 'info') & (svc_db.f_number == 0) & (
svc_db.f_hosts_id == host_id)
svc_id = db.t_services.update_or_insert(query,
f_proto='info',
f_number=0,
f_status='open',
f_hosts_id=host_id)
if not svc_id:
svc_rec = db(query).select(cache=(cache.ram, 180)).first()
if svc_rec:
svc_id = svc_rec.id
else:
log(" [!] Service record wasn't created", logging.ERROR)
continue
db.commit()
for script in hostscripts:
script_id = script.id
output = script.output
db.t_service_info.update_or_insert(f_services_id=svc_id,
f_name=script_id,
f_text=output)
db.commit()
if script_id == 'nbstat':
# pull out NetBIOS info from nbstat output
result = RE_NETBIOS_MAC.search(output)
if 'd' in result.groupdict():
host_rec.update(f_macaddr=result.group('d'))
db.commit()
result = RE_NETBIOS_NAME.search(output)
if 'd' in result.groupdict():
host_rec.update(f_netbios_name=result.group('d'))
db.commit()
result = RE_NETBIOS_WORKGROUP.search(output)
if 'd' in result.groupdict():
db(
db.t_netbios.update_or_insert(
f_hosts_id=host_id,
f_domain=result.group('d')))
db.commit()
# add ports and resulting vulndata
for port in node.ports:
f_proto = port.get('protocol')
f_number = port.get('portid')
f_status = port.get('port_state')
f_name = port.get('service_name')
f_product = port.get('service_product')
query = (svc_db.f_proto == f_proto) & (
svc_db.f_number == f_number) & (svc_db.f_hosts_id == host_id)
svc = db(query).select().first()
# if record does not exist, query returns None so add the record:
if svc is None:
log(" [-] Adding port: %s/%s (%s)" %
(f_proto, f_number, f_name))
svc_id = db.t_services.update_or_insert(f_proto=f_proto,
f_number=f_number,
f_status=f_status,
f_name=f_name,
f_hosts_id=host_id)
if not (svc is None):
log(" [-] Updating port: %s/%s (%s)" %
(f_proto, f_number, f_name))
if svc.f_name != f_name and f_name not in svc.f_name:
svc_id = db.t_services.validate_and_update(
_key=svc.id, f_name=(f_name + ' | ' + svc.f_name))
else:
svc_id = db.t_services.validate_and_update(
_key=svc.id, f_name=(svc.f_name))
svc_id = svc_id.id
if f_product:
version = port.get('service_version')
if version:
f_product += " (%s)" % (version)
db.t_service_info.update_or_insert(f_services_id=svc_id,
f_name=f_name,
f_text=f_product)
db.commit()
# Process <script> service entries
for script in port.get('scripts'):
try:
if script.get('id') == 'ssl-cert':
text = script.get('output')
sslcert = text.replace("/", "\n")
db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=script.get('id'),
f_text=sslcert)
else:
db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=script.get('id'),
f_text=script.get('output'))
except Exception as e:
logger.error("Error inserting Script: %s" % (e))
db.commit()
# check for banner id and update t_services banner field with the output
if script.get('id') == "banner":
try:
db.t_services.update_or_insert(
(db.t_services.id == svc_id),
f_banner=script.get('output'))
except Exception as e:
logger.error("Error inserting Banner: %s" % (e))
db.commit()
# Process <cpe> service entries
port_cpe = port.get('service_cpe')
if port_cpe:
cpe_id = port_cpe.replace('cpe:/', '')
if cpe_id.startswith('a'):
# process CPE Applications
#log(" [-] Found Application CPE data: %s" % (cpe_id))
db.t_service_info.update_or_insert(f_services_id=svc_id,
f_name='cpe.app',
f_text="cpe:/%s" %
(cpe_id))
db.commit()
elif cpe_id.startswith('o'):
# process CPE Operating System
os_id = lookup_cpe(cpe_id[2:])
if os_id is not None:
db.t_host_os_refs.insert(f_certainty='0.9',
f_family='Unknown',
f_class='Other',
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
# So no CPE or existing OS data, lets split up the CPE data and make our own
log(" [!] No os_id found, this is odd !!!")
# Adding uptime. Needed to add a table "f_uptime" in t_hosts db!
if node.uptime['lastboot']:
db.t_hosts.update_or_insert((db.t_hosts.f_ipaddr == ipaddr),
f_uptime=node.uptime['lastboot'])
if not node.uptime['lastboot']:
db.t_hosts.update_or_insert((db.t_hosts.f_ipaddr == ipaddr),
f_uptime=T("no entry found"))
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'),
apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception as error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error),
logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception as error:
log(
" [!] Error loading scan data to send to Metasploit: %s" %
str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
task = msf_api.pro_import_data(
msf_settings.get('workspace'),
"".join(scan_data),
{
#'preserve_hosts': form.vars.preserve_hosts,
'blacklist_hosts': "\n".join(ip_ignore_list)
},
)
msf_workspace_num = session.msf_workspace_num or 'unknown'
msfurl = os.path.join(msf_settings.get('url'), 'workspaces',
msf_workspace_num, 'tasks', task['task_id'])
log(" [*] Added file to MSF Pro: %s" % msfurl)
# any new nexpose vulns need to be checked against exploits table and connected
log(" [*] Connecting exploits to vulns and performing do_host_status")
do_host_status(asset_group=asset_group)
log(" [*] Import complete: hosts: %s added, %s updated, %s skipped" % (
hoststats['added'],
hoststats['updated'],
hoststats['skipped'],
))