def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_workspace=False,
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Qualys XML Scan file
import os, time, re, html.parser
from io import StringIO
from MetasploitProAPI import MetasploitProAPI
from skaldship.hosts import html_to_markmin, get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
parser = html.parser.HTMLParser()
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS name: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile(
'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
RE_IPV4 = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$')
if msf_workspace:
msf = MetasploitProAPI(host=user_id.f_msf_pro_url,
apikey=user_id.f_msf_pro_key)
if msf.login():
logger.info(" [-] Authenticated to Metasploit PRO")
else:
logger.error(
" [!] Unable to login to Metasploit PRO, check your API key")
msf = None
else:
logger.warn(" [-] No Metasploit workspace provided!")
msf = None
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
print((" [*] Processing Qualys scan file %s" % (filename)))
try:
nmap_xml = etree.parse(filename)
except etree.ParseError as e:
print((" [!] Invalid XML file (%s): %s " % (filename, e)))
return
root = nmap_xml.getroot()
db = current.globalenv['db']
cache = current.globalenv['cache']
existing_vulnids = db(db.t_vulndata()).select(
db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
#print(" [*] Found %d vulnerabilities in the database already." % (len(existing_vulnids)))
# check for any CPE OS data
if db(db.t_cpe_os).count() > 0:
have_cpe = True
else:
have_cpe = False
user_id = db.auth_user(engineer) or auth.user.id
# parse the hosts, where all the goodies are
nodes = root.findall('IP')
print((" [-] Parsing %d hosts" % (len(nodes))))
hoststats = {}
hoststats['added'] = 0
hoststats['skipped'] = 0
hoststats['updated'] = 0
hoststats['errored'] = 0
hosts = [] # array of host_id fields
vulns_added = 0
vulns_skipped = 0
for node in nodes:
nodefields = {}
ipaddr = node.get('value')
nodefields['f_ipaddr'] = ipaddr
nodefields['f_hostname'] = node.get('hostname')
nodefields['f_netbios_name'] = node.findtext('NETBIOS_HOSTNAME')
# nodefields['f_macaddr'] = address.get('addr')
"""
status = node.find('status').get('state')
print(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats['skipped'] += 1
continue
"""
if ipaddr in ip_exclude:
print(" [-] Host is in exclude list... skipping")
hoststats['skipped'] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
print(" [-] Host is not in the only list... skipping")
hoststats['skipped'] += 1
continue
ports = node.findall('INFOS')
if len(ports) < 1 and not addnoports:
print(
" [-] No ports open and not asked to add those kind... skipping"
)
hoststats['skipped'] += 1
continue
nodefields['f_engineer'] = user_id
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
# check to see if IPv4/IPv6 exists in DB already
if 'f_ipaddr' in nodefields:
host_rec = db(db.t_hosts.f_ipaddr ==
nodefields['f_ipaddr']).select().first()
else:
logging.warn("No IP Address found in record. Skipping")
continue
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
print((" [-] Adding %s" % (ipaddr)))
elif host_rec is not None and update_hosts:
db.commit()
host_id = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats['updated'] += 1
print((" [-] Updating %s" % (ipaddr)))
else:
hoststats['skipped'] += 1
db.commit()
print((" [-] Skipped %s" % (ipaddr)))
continue
hosts.append(host_id)
# :
for hostscripts in node.findall('hostscript/script'):
svc_id = db.t_services.update_or_insert(f_proto='info',
f_number=0,
f_status='open',
f_hosts_id=host_id)
db.commit()
for script in hostscripts:
script_id = script.get('id')
output = script.get('output')
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=script_id, f_text=output)
db.commit()
# add ports and resulting vulndata
for port in node.findall("ports/port"):
f_proto = port.get('protocol')
f_number = port.get('portid')
f_status = port.find('state').get('state')
port_svc = port.find('service')
if port_svc:
f_name = port_svc.get('name')
f_product = port_svc.get('product')
svc_fp = port_svc.get('servicefp')
else:
f_name = None
f_product = None
svc_fp = None
print(
(" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name)))
svc_id = db.t_services.update_or_insert(f_proto=f_proto,
f_number=f_number,
f_status=f_status,
f_hosts_id=host_id,
f_name=f_name)
if f_product:
version = port.find('service').get('version', None)
if version:
f_product += " (%s)" % (version)
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=f_name, f_text=f_product)
db.commit()
if svc_fp:
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=svc_fp, f_text=svc_fp)
db.commit()
# Process <script> service entries
for script in port.findall('service/script'):
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=script.get('id'),
f_text=script.get('output'))
db.commit()
# Process <cpe> service entries
for port_cpe in port.findall('service/cpe'):
cpe_id = port_cpe.text.replace('cpe:/', '')
if cpe_id[0] == "a":
# process CPE Applications
print((" [-] Found Application CPE data: %s" % (cpe_id)))
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name='CPE ID',
f_text="cpe:/%s" % (cpe_id))
db.commit()
elif cpe_id[0] == "o":
# process CPE Operating System
os_id = lookup_cpe(cpe_id[2:])
if os_id is not None:
db.t_host_os_refs.insert(f_certainty='0.9',
f_family='Unknown',
f_class='Other',
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
# So no CPE or existing OS data, lets split up the CPE data and make our own
print(" [!] No os_id found, this is odd !!!")
for config in port.findall("configuration/config"):
cfg_id = db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=config.attrib['name'],
f_text=config.text)
db.commit()
if re.match('\w+.banner$', config.attrib['name']):
db.t_services[svc_id] = dict(f_banner=config.text)
db.commit()
if config.attrib['name'] == 'mac-address':
# update the mac address of the host
db.t_hosts[host_id] = dict(f_macaddr=config.text)
db.commit()
if "advertised-name" in config.attrib['name']:
# netbios computer name
d = config.text.split(" ")[0]
if "Computer Name" in config.text:
data = {}
data['f_netbios_name'] = d
# if hostname isn't defined then lowercase netbios name and put it in
if db.t_hosts[host_id].f_hostname is None:
data['f_hostname'] = d.lower()
db(db.t_hosts.id == host_id).update(**data)
elif "Domain Name" in config.text:
db(db.t_netbios.f_hosts_id == host_id).update(
f_domain=d) or db.t_netbios.insert(
f_hosts_id=host_id, f_domain=d)
db.commit()
for script in port.findall("script"):
# process <script> results. This data contains both info
# and vulnerability data. For now we'll take a list of
# known nmap vuln checks from private/nmap_vulns.csv and
# use that to separate between service_info and vulndata.
pass
if msf is not None:
# send the downloaded nexpose file to MSF for importing
try:
res = msf.pro_import_file(
msf_workspace,
filename,
{
'DS_REMOVE_FILE': False,
'tag': asset_group,
},
)
print((" [*] Added file to MSF Pro: %s" % (res)))
except MSFAPIError as e:
logging.error("MSFAPI Error: %s" % (e))
pass
# any new nexpose vulns need to be checked against exploits table and connected
print(" [*] Connecting exploits to vulns and performing do_host_status")
do_host_status(asset_group=asset_group)
print((
" [*] Import complete: hosts: %s added, %s skipped, %s errors - vulns: %s added, %s skipped"
% (hoststats['added'], hoststats['skipped'], hoststats['errored'],
vulns_added, vulns_skipped)))
