def process_mass_password(pw_file=None,
pw_type=None,
message=None,
proto=None,
portnum=None,
add_hosts=False,
user_id=1):
"""
Process a medusa/hydra mass password run
"""
import fileinput
db = current.globalenv['db']
cache = current.globalenv['cache']
added = 0
updated = 0
new_hosts = 0
ip_dict = {}
if pw_file is not None:
try:
fIN = fileinput.input(files=pw_file)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return "Error opening %s: %s" % (pw_file, e)
def process_medusa(line):
"""
Process a medusa line and return a dictionary
:param line: A line from a medusa output file
:returns dict: A dictionary based upon the content
{ 'ip' : ip address
'port': port info - can be port # or module name
'user': username,
'pass': password,
'hash': ntlm hash if smbnt hash used
'msg' : status message
}
>>> process_medusa("# Medusa v.2.0a_rc1 (2012-12-06 10:44:10)")
{}
>>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.23 User: spauser Password: password [SUCCESS]")
{'ip': '10.89.172.23', 'error': False, 'user': '******', 'pass': '******', 'msg': '[SUCCESS]', 'type': 'cleartext', 'port': '[smbnt]'}
>>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.24 User: spauser Password: AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0::: [SUCCESS]")
{'ip': '10.89.172.23', 'port': 'smbnt', 'user': '******', 'password': '', 'hash': 'AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0', 'msg': '[SUCCESS]'}
"""
retval = {}
try:
data = line.split()
except Exception, e:
log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR)
return retval
def process_password_file(pw_file=None,
pw_data=None,
file_type=None,
source=None):
"""
Process a password file and return a dictionary fit for t_accounts
:param pw_file: Filename to process
:param pw_data: List of password lines instead of processing a file
:param file_type: 'PWDUMP', 'MSCa$h Dump', 'UNIX Passwd', 'UNIX Shadow', 'Medusa',
'Hydra', 'Username:Password', 'Usernames', 'AccountDB', 'Metasploit Creds CSV'
:param source: Source to add to f_source field
"""
import fileinput
from skaldship.passwords.medusa import process_medusa
from skaldship.passwords.hydra import process_hydra
from skaldship.passwords.metasploit import process_msfcsv
accounts = {}
if pw_file is not None:
try:
pw_data = []
for line in fileinput.input(files=pw_file):
pw_data.append(line)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return accounts
def scan_template():
from lxml import etree
from StringIO import StringIO
response.title = "%s :: Nexpose Scan Templates" % (settings.title)
formupload = SQLFORM.factory(
Field('f_filename', 'upload', uploadfolder=os.path.join(request.folder, 'data', 'misc'), label=T('Import Nexpose Scan Template')),
_formname='uploader')
if formupload.accepts(request.vars, formname='uploader'):
najax = NXAJAX(session.najaxsession)
template_class = ScanTemplates()
filename = os.path.join(request.folder,'data','misc',formupload.vars.f_filename)
template_xml = etree.parse(filename, etree.XMLParser())
imported = template_class.importtemplate(etree.tostring(template_xml), najax)
response.flash = imported
templates = ScanTemplates.listscantemps(True, najax)
parse_templates = DIV(TAG(templates).elements('templateid'))
return dict(form="", form2=formupload, html=parse_templates)
najax = NXAJAX(session.najaxsession)
najax.host = auth.user.f_nexpose_host
najax.port = auth.user.f_nexpose_port
if najax.login(user_id=auth.user.f_nexpose_user, password=auth.user.f_nexpose_pw):
log("Logged in to Nexpose API. Session cached.")
session.najaxsession = najax.getsession()
template_class = ScanTemplates()
templates = template_class.listscantemps(True, najax)
response.flash = "Loaded %s scan templates" % (templates.count('<templateid>'))
parse_templates = DIV(TAG(templates).elements('templateid'))
return dict(form="", form2=formupload, html=parse_templates)
else:
response.flash = "Unable to login to Nexpose"
return dict(form=formlogin, form2="", html="")
def run_scan(
blacklist=None,
target_list=None,
scan_options=None,
):
'''
Executes nmap scan
'''
from zenmapCore_Kvasir.NmapCommand import NmapCommand
from zenmapCore_Kvasir.NmapOptions import NmapOptions
from time import sleep
if scan_options[0] is not 'nmap':
if 'nmap' in settings:
scan_options.insert(0, settings.nmap)
else:
scan_options.insert(0, 'nmap')
if target_list:
data = []
for ip in target_list:
data.append(ip.strip(' \t\n\r'))
target_list = data
if blacklist:
data = []
for ip in blacklist:
data.append(ip.strip(' \t\n\r'))
blacklist = [','.join(map(str, data))]
blacklist.insert(0, "--exclude")
ops = NmapOptions()
try:
ops.parse(scan_options + target_list + blacklist)
except Exception as e:
log("[!] %s" % e)
cmd = NmapCommand(ops.render_string())
log(" [*] Starting Nmap Scan: %s" % (cmd.command))
cmd.run_scan()
try:
cmd.scan_state()
except Exception as e:
log("[!] %s" % e)
full_output = ""
while cmd.scan_state():
sleep(5)
result = cmd.get_output()
start = len(full_output) - len(result)
output = result[start:]
full_output = "%s%s" % (full_output, output)
log(output)
log(" [*] Nmap Scan Complete")
filename = cmd.get_xml_output_filename()
return filename
def process_medusa(line):
"""
Process a medusa line and return a dictionary
:param line: A line from a medusa output file
:returns dict: A dictionary based upon the content
{ 'ip' : ip address
'port': port info - can be port # or module name
'user': username,
'pass': password,
'hash': ntlm hash if smbnt hash used
'msg' : status message
}
>>> process_medusa("# Medusa v.2.0a_rc1 (2012-12-06 10:44:10)")
{}
>>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.23 User: spauser Password: password [SUCCESS]")
{'ip': '10.89.172.23', 'error': False, 'user': '******', 'pass': '******', 'msg': '[SUCCESS]', 'type': 'cleartext', 'port': '[smbnt]'}
>>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.24 User: spauser Password: AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0::: [SUCCESS]")
{'ip': '10.89.172.23', 'port': 'smbnt', 'user': '******', 'password': '', 'hash': 'AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0', 'msg': '[SUCCESS]'}
"""
retval = {}
try:
data = line.split()
except Exception, e:
log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR)
return retval
def get_or_create_record(argument, **defaults):
"""
Returns a t_host record based on the argument. If argument is an ipv4/ipv6 address it looks it up. If it's an
integer it returns it. If none exist and argument is an ipv4/ipv6 address it creates a new record using the
defaults provided.
:param argument: ip address or db.t_hosts.id
:param defaults: dictionary of db.t_hosts fields, validated before inserting
:returns: Row with id
>>> get_or_create_record('2.2.2.2')
<Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}>
>>> get_or_create_record('9.9.9.9', f_engineer=9999)
None
>>> get_or_create_record(1)
<Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}>
>>> get_or_create_record(9999)
None
"""
if argument is None:
return None
from gluon.validators import IS_IPADDRESS
db = current.globalenv['db']
auth = current.globalenv['auth']
record = get_host_record(argument)
if not record:
fields = {}
for k in list(defaults.keys()):
if k in db.t_hosts.fields:
fields[k] = defaults[k]
# set defaults for assetgroup/engineer if not set
if 'f_asset_group' not in fields:
fields['f_asset_group'] = 'undefined'
if 'f_engineer' not in fields:
fields['f_engineer'] = auth.user_id or 1
if IS_IPADDRESS()(argument)[1] == None:
fields['f_ipaddr'] = argument
else:
# invalid ip address, clear the fields
fields = None
if fields:
host_rec = db.t_hosts.validate_and_insert(**fields)
if host_rec.errors:
log("Error creating host record: %s" % host_rec.errors,
logging.ERROR)
else:
db.commit()
record = db.t_hosts(host_rec.get('id'))
return record
def get_or_create_record(argument, **defaults):
"""
Returns a t_host record based on the argument. If argument is an ipv4/ipv6 address it looks it up. If it's an
integer it returns it. If none exist and argument is an ipv4/ipv6 address it creates a new record using the
defaults provided.
:param argument: ip address or db.t_hosts.id
:param defaults: dictionary of db.t_hosts fields, validated before inserting
:returns: Row with id
>>> get_or_create_record('2.2.2.2')
<Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}>
>>> get_or_create_record('9.9.9.9', f_engineer=9999)
None
>>> get_or_create_record(1)
<Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}>
>>> get_or_create_record(9999)
None
"""
if argument is None:
return None
from gluon.validators import IS_IPADDRESS
db = current.globalenv["db"]
auth = current.globalenv["auth"]
record = get_host_record(argument)
if not record:
fields = {}
for k in defaults.keys():
if k in db.t_hosts.fields:
fields[k] = defaults[k]
# set defaults for assetgroup/engineer if not set
if "f_asset_group" not in fields:
fields["f_asset_group"] = "undefined"
if "f_engineer" not in fields:
fields["f_engineer"] = auth.user_id or 1
if IS_IPADDRESS()(argument)[1] == None:
fields["f_ipaddr"] = argument
else:
# invalid ip address, clear the fields
fields = None
if fields:
host_rec = db.t_hosts.validate_and_insert(**fields)
if host_rec.errors:
log("Error creating host record: %s" % host_rec.errors, logging.ERROR)
else:
db.commit()
record = db.t_hosts(host_rec.get("id"))
return record
def process_report(
filename=None,
host_list=[],
query=None,
ip_ignore_list=None,
ip_include_list=None,
engineer=1,
asset_group="ShodanHQ Import",
):
"""
Processes a ShodanHQ XML Report adding records to the db
"""
settings = current.globalenv['settings']
#try:
# from shodan import WebAPI
# from shodan.api import WebAPIError
# webapi = WebAPI(settings.shodanhq_apikey)
#except ImportError:
# webapi = None
sd = ShodanData()
sd.engineer = engineer
sd.asset_group = asset_group
# build the hosts only/exclude list
if ip_ignore_list:
sd.ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if ip_include_list:
sd.ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
hosts = []
if filename:
log(" [*] Processing ShodanHQ report file: %s" % (filename))
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
try:
xml = etree.parse(filename)
except etree.ParseError, e:
raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
return
root = xml.getroot()
hosts = root.findall("host")
def run_scan(blacklist=None, target_list=None, scan_options=None):
"""
Executes nmap scan
"""
from zenmapCore_Kvasir.NmapCommand import NmapCommand
from zenmapCore_Kvasir.NmapOptions import NmapOptions
from time import sleep
if scan_options[0] is not "nmap":
if "nmap" in settings:
scan_options.insert(0, settings.nmap)
else:
scan_options.insert(0, "nmap")
if target_list:
data = []
for ip in target_list:
data.append(ip.strip(" \t\n\r"))
target_list = data
if blacklist:
data = []
for ip in blacklist:
data.append(ip.strip(" \t\n\r"))
blacklist = [",".join(map(str, data))]
blacklist.insert(0, "--exclude")
ops = NmapOptions()
try:
ops.parse(scan_options + target_list + blacklist)
except Exception as e:
log("[!] %s" % e)
cmd = NmapCommand(ops.render_string())
log(" [*] Starting Nmap Scan: %s" % (cmd.command))
cmd.run_scan()
try:
cmd.scan_state()
except Exception as e:
log("[!] %s" % e)
full_output = ""
while cmd.scan_state():
sleep(5)
result = cmd.get_output()
start = len(full_output) - len(result)
output = result[start:]
full_output = "%s%s" % (full_output, output)
log(output)
log(" [*] Nmap Scan Complete")
filename = cmd.get_xml_output_filename()
return filename
def process_report(
filename=None,
host_list=[],
query=None,
ip_ignore_list=None,
ip_include_list=None,
engineer=1,
asset_group="ShodanHQ Import",
):
"""
Processes a ShodanHQ XML Report adding records to the db
"""
settings = current.globalenv['settings']
#try:
# from shodan import WebAPI
# from shodan.api import WebAPIError
# webapi = WebAPI(settings.shodanhq_apikey)
#except ImportError:
# webapi = None
sd = ShodanData()
sd.engineer = engineer
sd.asset_group = asset_group
# build the hosts only/exclude list
if ip_ignore_list:
sd.ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if ip_include_list:
sd.ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
hosts = []
if filename:
log(" [*] Processing ShodanHQ report file: %s" % (filename))
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
try:
xml = etree.parse(filename)
except etree.ParseError, e:
raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
return
root = xml.getroot()
hosts = root.findall("host")
def grab_screenshot(url=None, outfile=None, phantomjs="/usr/bin/phantomjs"):
"""
Capture a PNG image of a URL using phantomjs
@args:
url: Website URL to retrieve
outfile: Output filename, will overwrite but not remove failures
phantomjs: Full path to phantomjs binary
@output:
[True/False, png image data]
"""
import os
db = current.globalenv['db']
if not outfile:
raise Exception("No output filename provided")
try:
os.stat(phantomjs)
except OSError:
phantomjs = "/usr/local/bin/phantomjs"
try:
os.stat(phantomjs)
except OSError:
logging.error("Unable to locate phantomjs binary")
return [False, None]
# encode the url to make sure it passes cleanly to phantomjs
url = urllib.quote(url, safe='/:')
folder = current.globalenv['request'].folder
from sys import platform
if platform in ["linux", "linux2"]:
timeout = ["/usr/bin/timeout", "-k", "2", "5"]
elif platform in ["darwin", "freebsd"]:
timeout = [os.path.join(folder, 'private/timeout3'), "-t" "5"]
else:
timeout = []
phantom = timeout + [
phantomjs, "--ignore-ssl-errors=true",
"%s/modules/skaldship/valkyries/webimaging.js" % (folder), url, outfile
]
log("calling: %s" % str(phantom), logging.DEBUG)
call(phantom)
try:
f = file(outfile)
imgdata = f.read()
f.close()
result = True
except:
result = False
imgdata = None
return [result, imgdata]
def process_report_xml(
filename=None,
ip_ignore_list=None,
ip_include_list=None,
engineer=1,
asset_group="Metasploit Import",
update_hosts=True,
):
"""
Processes a Metasploit XML Export for the following data and adds to the db:
- Hosts and services
- Credentials
Generate the XML report by using db_export -t xml filename.xml or through WebUI
TODO: Auto-exploits successful exploit attempts if matching CVE/VulnDB entry found
"""
from gluon.validators import IS_IPADDRESS
from skaldship.passwords.utils import lookup_hash
from skaldship.hosts import get_host_record, get_or_create_record
from skaldship.services import Services
services = Services()
db = current.globalenv['db']
#cache = current.globalenv['cache']
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Metasploit Pro report file: %s" % (filename))
try:
xml = etree.parse(filename)
except etree.ParseError, e:
raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
def process_exploits(filename=None):
"""
Process Nexpose exploits.xml file into the database
"""
log("Processing Nexpose exploits file: %s ..." % filename)
try:
exploits = etree.parse(filename)
except etree.ParseError, e:
raise Exception("Error processing file: %s" % e)
def process_report_xml(
filename=None,
ip_ignore_list=None,
ip_include_list=None,
engineer=1,
asset_group="Metasploit Import",
update_hosts=True,
):
"""
Processes a Metasploit XML Export for the following data and adds to the db:
- Hosts and services
- Credentials
Generate the XML report by using db_export -t xml filename.xml or through WebUI
TODO: Auto-exploits successful exploit attempts if matching CVE/VulnDB entry found
"""
from gluon.validators import IS_IPADDRESS
from skaldship.passwords.utils import lookup_hash
from skaldship.hosts import get_host_record, get_or_create_record
from skaldship.services import Services
services = Services()
db = current.globalenv['db']
#cache = current.globalenv['cache']
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Metasploit Pro report file: %s" % (filename))
try:
xml = etree.parse(filename)
except etree.ParseError, e:
raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
def grab_screenshot(url=None, outfile=None, phantomjs="/usr/bin/phantomjs"):
"""
Capture a PNG image of a URL using phantomjs
@args:
url: Website URL to retrieve
outfile: Output filename, will overwrite but not remove failures
phantomjs: Full path to phantomjs binary
@output:
[True/False, png image data]
"""
import os
db = current.globalenv['db']
if not outfile:
raise Exception("No output filename provided")
try:
os.stat(phantomjs)
except OSError:
phantomjs = "/usr/local/bin/phantomjs"
try:
os.stat(phantomjs)
except OSError:
logging.error("Unable to locate phantomjs binary")
return [False, None]
# encode the url to make sure it passes cleanly to phantomjs
url = urllib.quote(url, safe='/:')
folder = current.globalenv['request'].folder
from sys import platform
if platform in ["linux", "linux2"]:
timeout = ["/usr/bin/timeout", "-k", "2", "5"]
elif platform in ["darwin", "freebsd"]:
timeout = [os.path.join(folder, 'private/timeout3'), "-t" "5"]
else:
timeout = []
phantom = timeout + [phantomjs, "--ignore-ssl-errors=true", "%s/modules/skaldship/valkyries/webimaging.js" % (folder), url, outfile]
log("calling: %s" % str(phantom), logging.DEBUG)
call(phantom)
try:
f = file(outfile)
imgdata = f.read()
f.close()
result = True
except:
result = False
imgdata = None
return [result, imgdata]
def process_hydra(line):
"""
Process a hydra line and return a dictionary:
{ 'ip' : ip address
'port': port info - can be port # or module name
'user': username,
'pass': password,
'hash': ntlm hash if smbnt hash used
'msg' : status message
}
"""
# line: [22][ssh] host: 1.1.1.1 login: username password: pw1234
retval = {}
try:
data = line.split()
except Exception as e:
log("Error processing hydra line: %s -- %s" % (e, line), logging.ERROR)
return retval
if data[1] == "host:":
# these fields are always there.. sometimes password is not
retval['port'] = data[0][1:data[0].find("]")]
retval['ip'] = data[2]
retval['user'] = data[4]
if "password:"******"[smb]" in data and "Error:" in data:
if len(retval['pass']) == 68 and retval['pass'][65:68] == ":::":
# we have an ntlm hash cap'n
retval['hash'] = ":".join(retval['pass'].split(':')[:2])
retval['pass'] = lookup_hash(retval['hash'])
retval['type'] = 'smb'
else:
retval['type'] = 'cleartext'
retval['hash'] = None
retval['error'] = False
return retval
def add_or_update(hostfields, update=False):
"""
Add a host and return the record. If update is True and host already exists
then the record is updated and returned
"""
if not isinstance(hostfields, dict()):
log(" [!] Hostfields is not a dictionary", logging.ERROR)
return None
host_rec = db(db.t_hosts.f_ipaddr == hostfields.get('f_ipaddr'))
if not host_rec:
try:
host_id = db.t_hosts.insert(**hostfields)
db.commit()
except Exception as e:
log("Error adding host: %s" % strerror(e))
return None
host_rec = db.t_hosts[host_id]
log(" [*] Added host: %s" % host_title_maker(host_rec))
else:
if update:
host_rec.update(**hostfields)
log(" [*] Updated host: %s" % host_title_maker(host_rec))
return host_rec
def vuln_time_convert(vtime=''):
"""Converts Nexpose timetsamp (YYYYMMDDTHHMMSSUUU) into python datetime"""
if not vtime:
tval = datetime(1970, 1, 1)
else:
if isinstance(vtime, str):
if vtime[8] == "T":
tstr = "%%Y%%m%%dT%%H%%M%%S%s" % vtime[15:]
tval = time.strptime(vtime, tstr)
else:
log("Unknown datetime value: %s" % vtime, logging.ERROR)
else:
log("Invalid datetime value provided: %s" % vtime, logging.ERROR)
tval = datetime(1970, 1, 1)
return datetime.fromtimestamp(time.mktime(tval))
def process_xml(
filename=None,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
"""
Process a Nessus XML Report file
Args:
filename: A local filename to process
asset_group: Asset group to assign hosts to
engineer: Engineer record number to assign hosts to
msf_workspace: If set a Metasploit workspace to send the scanfile to via the API
ip_ignore_list: List of IP addresses to ignore
ip_include_list: List of IP addresses to ONLY import (skip all others)
update_hosts: Boolean to update/append to hosts, otherwise hosts are skipped
Returns:
msg: A string status message
"""
from skaldship.cpe import lookup_cpe
db = current.globalenv['db']
cache = current.globalenv['cache']
settings = current.globalenv['settings']
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nessus scan file %s" % filename)
try:
nessus_xml = etree.parse(filename)
except etree.ParseError, e:
msg = " [!] Invalid Nessus scan file (%s): %s " % (filename, e)
log(msg, logging.ERROR)
return msg
def process_cracked_file(pw_file=None, file_type=None, message=""):
"""
Process a file of cracked passwords and update the cleartext with
the new results.
"""
import fileinput
db = current.globalenv['db']
cache = current.globalenv['cache']
if pw_file is not None:
try:
fIN = fileinput.input(files=pw_file)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return "Error opening %s: %s" % (pw_file, e)
def process_file(filename=None, asset_group=None, engineer=None):
# Upload and process hping Scan file
from skaldship.hosts import get_host_record, do_host_status, add_or_update
from gluon.validators import IS_IPADDRESS
log(" [*] Processing hping scan file %s" % filename)
hoststats = 0
nodefields = {'f_engineer': engineer, 'f_asset_group': asset_group, 'f_confirmed': False}
svc_db = db.t_services
host_ip = None
ICMP_type = ''
answer_ip = ''
with open(filename) as f:
for line in f:
if "IP: " in line:
host_ip = line.split()[1]
if IS_IPADDRESS()(host_ip)[1] == None:
nodefields['f_ipaddr'] = host_ip
db.t_hosts.update_or_insert(**nodefields)
db.commit()
hoststats += 1
else:
log(" [!] ERROR: Not a valid IP Address (%s)" % host_ip, logging.ERROR)
if "[*] " in line:
ICMP_type = line.split()[1]
if "ip=" in line:
ip = line.split('=')[2]
answer_ip = ip.split()[0]
if "transmitted" in line:
packets = line.split()
if packets[0] == packets[3]:
if answer_ip != host_ip:
response = "No"
else:
response = "Yes"
else:
response = "No"
get_id = get_host_record(host_ip)
svc_db.update_or_insert(
f_hosts_id=get_id.id, f_proto='ICMP', f_number='0', f_status=response, f_name=ICMP_type
)
db.commit()
def process_loot_files(loot_list=[]):
"""
Processes locally stored (to web2py) MSF password loot files into the
account database.
Args:
loot_list: an array of [filename, settings.password_file_types, port, host_id]
Returns:
An array of [filename, result text]
"""
from skaldship.passwords.utils import process_password_file, insert_or_update_acct
#import os
db = current.globalenv['db']
data = []
for loot in loot_list:
if isinstance(loot, []):
(filename, file_type, port) = loot
else:
log("Invalid loot sent: %s" % loot, logging.ERROR)
continue
try:
(proto, number) = port.split('/')
except AttributeError as e:
log("Invalid port sent: %s", port, logging.ERROR)
try:
pw_data = open(filename, "rb").readlines().split('\n')
except IOError as e:
log("Error opening %s: %s" % (filename, e), logging.ERROR)
accounts = process_password_file(
pw_data=pw_data,
file_type=file_type,
source='Metasploit',
)
# find the info/0 service id for the host
host = get_host_record(loot['host'])
query = (db.t_services.f_number == number)
query &= (db.t_services.f_proto == proto)
query &= (db.t_services.f_hosts_id == host.id)
svc_id = db(query).select().first()
if svc_id is None:
# info/0 not found.. add it!
svc_id = db.t_services.insert(f_proto=proto,
f_number=number,
f_hosts_id=host.id)
db.commit()
# insert or update the account records
resp_text = insert_or_update_acct(svc_id.id, accounts)
log("Added loot accounts for host: %s" % host.f_ipaddr)
data.append({loot['host']: resp_text})
def process_pwdump_loot(loot_list=[], msf=None):
"""
Takes an array of loot records in loot_list, downloads the pwdump file and
adds the users.
"""
from skaldship.passwords.utils import process_password_file, insert_or_update_acct
db = current.globalenv['db']
#cache = current.globalenv['cache']
data = []
for loot_id in loot_list:
loot = msf.loot_download(loot_id)
if loot['ltype'] not in ['host.windows.pwdump', 'windows.hashes']:
log("Loot is not a pwdump, it is a %s" % loot['ltype'],
logging.ERROR)
continue
else:
# process the pwdump file
pw_data = loot['data'].split('\n')
accounts = process_password_file(
pw_data=pw_data,
file_type='PWDUMP',
source='Metasploit',
)
# find the info/0 service id for the host
host = get_host_record(loot['host'])
query = (db.t_services.f_number == '0') & (
db.t_services.f_proto == 'info') & (db.t_services.f_hosts_id
== host.id)
svc_id = db(query).select().first()
if svc_id is None:
# info/0 not found.. add it!
svc_id = db.t_services.insert(f_proto="info",
f_number="0",
f_status="info",
f_hosts_id=host.id)
db.commit()
# insert or update the account records
resp_text = insert_or_update_acct(svc_id.id, accounts)
log("Added pwdump records for host: %s" % host.f_ipaddr)
data.append({loot['host']: resp_text})
return data
def clean_html(htmldata):
"""Cleans up the HTML using lxml.html clean_html for now."""
try:
from lxml.html.clean import clean_html
except ImportError:
log("You don't have lxml installed", logging.ERROR)
return htmldata
if htmldata is None:
return htmldata
newdata = clean_html(htmldata)
newdata = newdata.replace('\n', ' ')
newdata = newdata.replace('<div>', '')
newdata = newdata.replace('</div>', '')
#newdata = re.compile('\s*\n\s*').sub('\n', newdata)
return newdata
def clean_html(htmldata):
"""Cleans up the HTML using lxml.html clean_html for now."""
try:
from lxml.html.clean import clean_html
except ImportError:
log("You don't have lxml installed", logging.ERROR)
return htmldata
if htmldata is None:
return htmldata
newdata = clean_html(htmldata)
newdata = newdata.replace('\n', ' ')
newdata = newdata.replace('<div>', '')
newdata = newdata.replace('</div>', '')
#newdata = re.compile('\s*\n\s*').sub('\n', newdata)
return newdata
def process_password_file(pw_file=None, pw_data=None, file_type=None, source=None):
"""
Process a password file and return a dictionary fit for t_accounts
file_type values:
('PWDUMP', 'MSCa$h Dump', 'UNIX Passwd', 'UNIX Shadow', 'Medusa', 'Hydra', 'Username:Password', 'AccountDB')
"""
import fileinput
accounts = {}
if pw_file is not None:
try:
pw_data = []
for line in fileinput.input(files=pw_file):
pw_data.append(line)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return accounts
def _update_or_insert(self, **fields):
"""
Our own update_or_insert routine
:param fields: Matching db.t_services fields (f_proto, f_number, etc)
:returns: t_services record id
"""
if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']:
log("No protocol, number or hosts_id sent", logging.ERROR)
return None
svc_id = self.svc_db.update_or_insert(**fields)
if not svc_id:
# update_or_insert will not return an id if a record is updated.
record = self._get_record(**fields)
if record:
svc_id = record.id
return svc_id
def process_mass_password(pw_file=None, pw_type=None, message=None, proto=None, portnum=None, add_hosts=False, user_id=1):
"""
Process a medusa/hydra mass password run
"""
import fileinput
db = current.globalenv['db']
cache = current.globalenv['cache']
added = 0
updated = 0
new_hosts = 0
ip_dict = {}
if pw_file is not None:
try:
fIN = fileinput.input(files=pw_file)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return "Error opening %s: %s" % (pw_file, e)
def process_hydra(line):
"""
Process a hydra line and return a dictionary:
{ 'ip' : ip address
'port': port info - can be port # or module name
'user': username,
'pass': password,
'hash': ntlm hash if smbnt hash used
'msg' : status message
}
"""
# line: [22][ssh] host: 1.1.1.1 login: username password: pw1234
retval = {}
try:
data = line.split()
except Exception, e:
log("Error processing hydra line: %s -- %s" % (e, line), logging.ERROR)
return retval
def clean_html(htmldata):
"""Cleans up the HTML using lxml.html clean_html for now."""
import re
try:
from lxml.html.clean import clean_html
except ImportError:
log("You don't have lxml installed", logging.ERROR)
return htmldata
if htmldata is None:
return htmldata
newdata = clean_html(htmldata)
newdata = newdata.replace('\n', ' ')
newdata = newdata.replace('<div>', '')
newdata = newdata.replace('</div>', '')
newdata = newdata.replace('\t', '') # tabs? not needed, no never
newdata = re.sub(' +', ' ', newdata)
return newdata
def process_cracked_file(pw_file=None, file_type=None, message=""):
"""
Process a file of cracked passwords and update the cleartext with
the new results.
:param pw_file: Filename to process
:param file_type: String of a file type
:param message: Message string to add to f_message field
"""
import fileinput
db = current.globalenv['db']
if pw_file is not None:
try:
fIN = fileinput.input(files=pw_file)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return "Error opening %s: %s" % (pw_file, e)
def process_cracked_file(pw_file=None, file_type=None, message=""):
"""
Process a file of cracked passwords and update the cleartext with
the new results.
:param pw_file: Filename to process
:param file_type: String of a file type
:param message: Message string to add to f_message field
"""
import fileinput
db = current.globalenv['db']
if pw_file is not None:
try:
fIN = fileinput.input(files=pw_file)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return "Error opening %s: %s" % (pw_file, e)
def process_medusa(line):
"""
Process a medusa line and return a dictionary:
{ 'ip' : ip address
'port': port info - can be port # or module name
'user': username,
'pass': password,
'hash': ntlm hash if smbnt hash used
'msg' : status message
}
"""
retval = {}
try:
data = line.split()
except Exception, e:
log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR)
return retval
def scan_template():
from lxml import etree
from StringIO import StringIO
response.title = "%s :: Nexpose Scan Templates" % (settings.title)
formupload = SQLFORM.factory(Field(
'f_filename',
'upload',
uploadfolder=os.path.join(request.folder, 'data', 'misc'),
label=T('Import Nexpose Scan Template')),
_formname='uploader')
if formupload.accepts(request.vars, formname='uploader'):
najax = NXAJAX(session.najaxsession)
template_class = ScanTemplates()
filename = os.path.join(request.folder, 'data', 'misc',
formupload.vars.f_filename)
template_xml = etree.parse(filename, etree.XMLParser())
imported = template_class.importtemplate(etree.tostring(template_xml),
najax)
response.flash = imported
templates = ScanTemplates.listscantemps(True, najax)
parse_templates = DIV(TAG(templates).elements('templateid'))
return dict(form="", form2=formupload, html=parse_templates)
nexpose_config = nexpose_get_config()
najax = NXAJAX(session.najaxsession)
najax.host = nexpose_config['host']
najax.port = nexpose_config['port']
if najax.login(user_id=nexpose_config['user'],
password=nexpose_config['password']):
log("Logged in to Nexpose API. Session cached.")
session.najaxsession = najax.getsession()
template_class = ScanTemplates()
templates = template_class.listscantemps(True, najax)
response.flash = "Loaded %s scan templates" % (
templates.count('<templateid>'))
parse_templates = DIV(TAG(templates).elements('templateid'))
return dict(form="", form2=formupload, html=parse_templates)
else:
response.flash = "Unable to login to Nexpose"
return dict(form=formlogin, form2="", html="")
def _update_or_insert(self, **fields):
"""
Our own update_or_insert routine
:param fields: Matching db.t_services fields (f_proto, f_number, etc)
:returns: t_services record id
"""
if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']:
log("No protocol, number or hosts_id sent", logging.ERROR)
return None
svc_id = self.svc_db.update_or_insert(**fields)
if not svc_id:
# update_or_insert will not return an id if a record is updated.
record = self._get_record(**fields)
if record:
svc_id = record.id
return svc_id
def run_scanner(
scanner=None,
asset_group=None,
engineer=None,
target_list=None,
blacklist=None,
scan_options=None,
addnoports=False,
update_hosts=False,
**kwargs
):
'''
Schedule handler to process nmap scan
'''
from skaldship.log import log
if not isinstance(scanner, str):
return False
scanner = scanner.upper()
logger.info(" [*] Processing Nmap scan ")
if scanner == 'NMAP':
from skaldship.nmap import run_scan
nmap_xml_file = run_scan(
blacklist=blacklist,
target_list=target_list,
scan_options=scan_options,
)
if nmap_xml_file:
from skaldship.nmap import process_xml
log("Processing nmap xml file: %s" % (nmap_xml_file))
process_xml(
filename=nmap_xml_file,
addnoports=addnoports,
asset_group=asset_group,
engineer=engineer,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=update_hosts,
)
def scan_template():
from lxml import etree
from StringIO import StringIO
response.title = "%s :: Nexpose Scan Templates" % (settings.title)
formupload = SQLFORM.factory(
Field(
"f_filename",
"upload",
uploadfolder=os.path.join(request.folder, "data", "misc"),
label=T("Import Nexpose Scan Template"),
),
_formname="uploader",
)
if formupload.accepts(request.vars, formname="uploader"):
najax = NXAJAX(session.najaxsession)
template_class = ScanTemplates()
filename = os.path.join(request.folder, "data", "misc", formupload.vars.f_filename)
template_xml = etree.parse(filename, etree.XMLParser())
imported = template_class.importtemplate(etree.tostring(template_xml), najax)
response.flash = imported
templates = ScanTemplates.listscantemps(True, najax)
parse_templates = DIV(TAG(templates).elements("templateid"))
return dict(form="", form2=formupload, html=parse_templates)
nexpose_config = nexpose_get_config()
najax = NXAJAX(session.najaxsession)
najax.host = nexpose_config["host"]
najax.port = nexpose_config["port"]
if najax.login(user_id=nexpose_config["user"], password=nexpose_config["password"]):
log("Logged in to Nexpose API. Session cached.")
session.najaxsession = najax.getsession()
template_class = ScanTemplates()
templates = template_class.listscantemps(True, najax)
response.flash = "Loaded %s scan templates" % (templates.count("<templateid>"))
parse_templates = DIV(TAG(templates).elements("templateid"))
return dict(form="", form2=formupload, html=parse_templates)
else:
response.flash = "Unable to login to Nexpose"
return dict(form=formlogin, form2="", html="")
def run_scanner(
scanner=None,
asset_group=None,
engineer=None,
target_list=None,
blacklist=None,
scan_options=None,
addnoports=False,
update_hosts=False,
**kwargs
):
'''
Schedule handler to process nmap scan
'''
from skaldship.log import log
if not isinstance(scanner, str):
return False
scanner = scanner.upper()
logger.info(" [*] Processing Nmap scan ")
if scanner == 'NMAP':
from skaldship.nmap import run_scan
nmap_xml_file = run_scan(
blacklist=blacklist,
target_list=target_list,
scan_options=scan_options,
)
if nmap_xml_file:
from skaldship.nmap import process_xml
log("Processing nmap xml file: %s" % (nmap_xml_file))
process_xml(
filename=nmap_xml_file,
addnoports=addnoports,
asset_group=asset_group,
engineer=engineer,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=update_hosts,
)
def _get_record(self, **fields):
"""
Obtain a record from specified fields. Requires f_proto, f_number and f_hosts_id
:param fields: Matching db.t_services fields (f_proto, f_number, etc)
:returns: t_services record
"""
if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']:
log("No protocol, number or hosts_id sent", logging.ERROR)
return None
query = (self.svc_db.f_proto == fields['f_proto']) &\
(self.svc_db.f_number == fields['f_number']) &\
(self.svc_db.f_hosts_id == fields['f_hosts_id'])
record = self.db(query).select().first()
if record and record.id not in self.services:
self.services[record.id] = record
return record
def _get_record(self, **fields):
"""
Obtain a record from specified fields. Requires f_proto, f_number and f_hosts_id
:param fields: Matching db.t_services fields (f_proto, f_number, etc)
:returns: t_services record
"""
if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']:
log("No protocol, number or hosts_id sent", logging.ERROR)
return None
query = (self.svc_db.f_proto == fields['f_proto']) &\
(self.svc_db.f_number == fields['f_number']) &\
(self.svc_db.f_hosts_id == fields['f_hosts_id'])
record = self.db(query).select().first()
if record and record.id not in self.services:
self.services[record.id] = record
return record
def process_xml(
filename=None,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nexpose XML Scan file
from skaldship.cpe import lookup_cpe
from skaldship.hosts import get_host_record
from gluon.validators import IS_IPADDRESS
import os
db = current.globalenv['db']
session = current.globalenv['session']
parser = HTMLParser.HTMLParser()
user_id = db.auth_user(engineer)
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nexpose scan file %s" % filename)
try:
nexpose_xml = etree.parse(filename)
except etree.ParseError, e:
msg = " [!] Invalid Nexpose XML file (%s): %s " % (filename, e)
log(msg, logging.ERROR)
return msg
def process_pwdump_loot(loot_list=[], msf=None):
"""
Takes an array of loot records in loot_list, downloads the pwdump file and
adds the users.
"""
from skaldship.passwords.utils import process_password_file, insert_or_update_acct
db = current.globalenv['db']
#cache = current.globalenv['cache']
data = []
for loot_id in loot_list:
loot = msf.loot_download(loot_id)
if loot['ltype'] not in ['host.windows.pwdump', 'windows.hashes']:
log("Loot is not a pwdump, it is a %s" % loot['ltype'], logging.ERROR)
continue
else:
# process the pwdump file
pw_data = loot['data'].split('\n')
accounts = process_password_file(
pw_data=pw_data,
file_type='PWDUMP',
source='Metasploit',
)
# find the info/0 service id for the host
host = get_host_record(loot['host'])
query = (db.t_services.f_number == '0') & (db.t_services.f_proto == 'info') & (db.t_services.f_hosts_id == host.id)
svc_id = db(query).select().first()
if svc_id is None:
# info/0 not found.. add it!
svc_id = db.t_services.insert(f_proto="info", f_number="0", f_status="info", f_hosts_id=host.id)
db.commit()
# insert or update the account records
resp_text = insert_or_update_acct(svc_id.id, accounts)
log("Added pwdump records for host: %s" % host.f_ipaddr)
data.append({loot['host']: resp_text})
return data
def process_loot_files(loot_list=[]):
"""
Processes locally stored (to web2py) MSF password loot files into the
account database.
Args:
loot_list: an array of [filename, settings.password_file_types, port, host_id]
Returns:
An array of [filename, result text]
"""
from skaldship.passwords.utils import process_password_file, insert_or_update_acct
#import os
db = current.globalenv['db']
data = []
for loot in loot_list:
if isinstance(loot, []):
(filename, file_type, port) = loot
else:
log("Invalid loot sent: %s" % loot, logging.ERROR)
continue
try:
(proto, number) = port.split('/')
except AttributeError, e:
log("Invalid port sent: %s", port, logging.ERROR)
try:
pw_data = open(filename, "rb").readlines().split('\n')
except IOError, e:
log("Error opening %s: %s" % (filename, e), logging.ERROR)
def import_all_vulndata(overwrite=False, nexpose_server={}):
"""
Uses the NexposeAPI and imports each and every vulnerability to Kvasir. Can take a looooong time.
Args:
overwrite: Whether or not to overwrite an existing t_vulndata record
Returns:
msg: A string message of status.
"""
from NexposeAPI import VulnData
db = current.globalenv['db']
vuln_class = VulnData()
vuln_class.host = nexpose_server.get('host', 'localhost')
vuln_class.port = nexpose_server.get('port', '3780')
if vuln_class.login(user_id=nexpose_server.get('user'), password=nexpose_server.get('pw')):
log(" [*] Populating list of Nexpose vulnerability ID summaries")
try:
vuln_class.populate_summary()
except Exception, e:
log(" [!] Error populating summaries: %s" % str(e), logging.ERROR)
return False
try:
vulnxml = etree.parse(StringIO(vuln_class.vulnxml))
except Exception, e:
log(" [!] Error parsing summary XML: %s" % str(e), logging.ERROR)
return False
def process_loot_files(loot_list=[]):
"""
Processes locally stored (to web2py) MSF password loot files into the
account database.
Args:
loot_list: an array of [filename, settings.password_file_types, port, host_id]
Returns:
An array of [filename, result text]
"""
from skaldship.passwords.utils import process_password_file, insert_or_update_acct
#import os
db = current.globalenv['db']
data = []
for loot in loot_list:
if isinstance(loot, []):
(filename, file_type, port) = loot
else:
log("Invalid loot sent: %s" % loot, logging.ERROR)
continue
try:
(proto, number) = port.split('/')
except AttributeError, e:
log("Invalid port sent: %s", port, logging.ERROR)
try:
pw_data = open(filename, "rb").readlines().split('\n')
except IOError, e:
log("Error opening %s: %s" % (filename, e), logging.ERROR)
def process_password_file(pw_file=None, pw_data=None, file_type=None, source=None):
"""
Process a password file and return a dictionary fit for t_accounts
:param pw_file: Filename to process
:param pw_data: List of password lines instead of processing a file
:param file_type: 'PWDUMP', 'MSCa$h Dump', 'UNIX Passwd', 'UNIX Shadow', 'Medusa',
'Hydra', 'Username:Password', 'Usernames', 'AccountDB', 'Metasploit Creds CSV'
:param source: Source to add to f_source field
"""
import fileinput
from skaldship.passwords.medusa import process_medusa
from skaldship.passwords.hydra import process_hydra
from skaldship.passwords.metasploit import process_msfcsv
accounts = {}
if pw_file is not None:
try:
pw_data = []
for line in fileinput.input(files=pw_file):
pw_data.append(line)
except IOError, e:
log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
return accounts
def process_exploits(filename=None):
"""
Process Nexpose exploits.xml file into the database
"""
log("Processing Nexpose exploits file: %s ..." % filename)
try:
exploits = etree.parse(filename)
except etree.ParseError as e:
raise Exception("Error processing file: %s" % e)
except IOError as e:
raise Exception("Error opening file: %s" % e)
r = exploits.getroot()
counter = 0
from .exploits import add_exploit, connect_exploits
for exploit in r.findall('exploit'):
#"adobe-unspec-bof-cve-2010-1297","13787","0day Exploit for Adobe Flash and Reader PoC (from the wild)","Description","1","Expert"
f_name = exploit.findtext('name')
f_title = exploit.findtext('id')
f_description = str(exploit.findtext('description')).encode(
'iso-8859-1').decode('cp1252')
f_description = f_description.replace("\\'", "'").replace('\\x', "0x")
f_source = exploit.findtext('source')
f_level = exploit.findtext(
'rank') or 'Unknown' # exploiter experience level estimate
f_rank = exploit.findtext(
'exploitrank') or 'Unknown' # rank of the exploit
# exploit records can have multiple Nexpose vulnerabilitiy identifiers
f_vulnid = []
for nex_id in exploit.findall("vulnerabilities/vulnerability"):
f_vulnid.append(nex_id.get('id').lower())
res = add_exploit(
cve=None,
vuln_ids=f_vulnid,
f_name=f_name,
f_title=f_title,
f_description=f_description,
f_source=f_source,
f_level=f_level,
f_rank=f_rank,
)
if res > 0:
counter += 1
else:
log("Error importing exploit: %s" % f_name, logging.ERROR)
connect_exploits()
log("%d exploits added/updated" % counter)
return True
def db_vuln_refs(self, vuln_id=None, vulndata={}, extradata={}):
"""
Add or update vulnerability references such as CPE, MSF Bulletins, OSVDB, Bugtraq, etc.
Args:
vuln_id: The db.t_vulndata reference id
vulndadta: A dictionary of vulnerability data from t_vulndata
extradata: A dictionary of extra vulndata
Returns:
None
"""
if not vulndata:
log(" [!] No vulndata sent!", logging.ERROR)
return
if not extradata:
log(" [!] No extradata sent!", logging.ERROR)
return
if not vuln_id:
log(" [!] No vulnerability record id sent!", logging.ERROR)
return
ref_types = self.ref_types
ref_types.extend(self.single_refs)
# ugh this needs to be more pythonic. it's 1:30am and I'm tired
for refname in ref_types:
if refname in extradata:
for reftext in extradata[refname]:
if reftext:
# add the vuln_ref
ref_id = self.db.t_vuln_refs.update_or_insert(
f_text=reftext,
f_source=refname.upper(),
)
if not ref_id:
ref_id = self.db(self.db.t_vuln_refs.f_text == reftext).select(
cache=(self.cache.ram, 180)
).first().id
# link vuln_ref to vulndata
self.db.t_vuln_references.update_or_insert(
f_vulndata_id=vuln_id,
f_vuln_ref_id=ref_id
)
return
def db_vuln_refs(self, vuln_id=None, vulndata={}, extradata={}):
"""
Add or update vulnerability references such as CPE, MSF Bulletins, OSVDB, Bugtraq, etc.
Args:
vuln_id: The db.t_vulndata reference id
vulndadta: A dictionary of vulnerability data from t_vulndata
extradata: A dictionary of extra vulndata
Returns:
None
"""
if not vulndata:
log(" [!] No vulndata sent!", logging.ERROR)
return
if not extradata:
log(" [!] No extradata sent!", logging.ERROR)
return
if not vuln_id:
log(" [!] No vulnerability record id sent!", logging.ERROR)
return
ref_types = self.ref_types
ref_types.extend(self.single_refs)
# ugh this needs to be more pythonic. it's 1:30am and I'm tired
for refname in ref_types:
if refname in extradata:
for reftext in extradata[refname]:
if reftext:
# add the vuln_ref
ref_id = self.db.t_vuln_refs.update_or_insert(
f_text=reftext,
f_source=refname.upper(),
)
if not ref_id:
ref_id = self.db(
self.db.t_vuln_refs.f_text == reftext).select(
cache=(self.cache.ram, 180)).first().id
# link vuln_ref to vulndata
self.db.t_vuln_references.update_or_insert(
f_vulndata_id=vuln_id, f_vuln_ref_id=ref_id)
return
def add_or_update(hostfields, update=False):
"""
Add a host and return the record. If update is True and host already exists
then the record is updated and returned
"""
if not isinstance(hostfields, dict()):
log(" [!] Hostfields is not a dictionary", logging.ERROR)
return None
host_rec = db(db.t_hosts.f_ipaddr == hostfields.get('f_ipaddr'))
if not host_rec:
try:
host_id = db.t_hosts.insert(**hostfields)
db.commit()
except Exception, e:
log("Error adding host: %s" % strerror(e))
return None
host_rec = db.t_hosts[host_id]
log(" [*] Added host: %s" % host_title_maker(host_rec))
def import_vulnid():
"""
Downloads the detailed vulnerability data from Nexpose based on
a vuln id passed to it
"""
form = SQLFORM.factory(
Field('nexid', 'string', label=T('Nexpose ID')),
Field('nexid_list', 'text', label=T('Nexpose ID List')))
response.title = "%s :: Import Nexpose VulnID" % settings.title
nexpose_config = nexpose_get_config()
if form.process().accepted:
from NexposeAPI import VulnData
from skaldship.nexpose import vuln_parse
nxvulns = VulnData()
nxvulns.host = nexpose_config['host']
nxvulns.port = nexpose_config['port']
nexpose_ids = []
if form.vars.nexid:
nexpose_ids.extend([form.vars.nexid])
if form.vars.nexid_list:
nexpose_ids.extend(form.vars.nexid_list.split('\r\n'))
res = nxvulns.login(user_id=nexpose_config['user'],
password=nexpose_config['password'])
if res:
stats = {'added': 0, 'invalid': 0}
for nexid in nexpose_ids:
vulndetails = nxvulns.detail(nexid)
if vulndetails is not None:
(vulnfields, references) = vuln_parse(
vulndetails.find('Vulnerability'), fromapi=True)
else:
stats['invalid'] += 1
continue
# add the vulnerability to t_vulndata
query = (db.t_vulndata.f_vulnid == nexid)
vulnid = db.t_vulndata.update_or_insert(query, **vulnfields)
if not vulnid:
row = db(query).select().first()
if row:
vulnid = row.id
else:
log(" [!] Could not find %s in database.." % nexid,
logging.WARN)
stats['invalid'] += 1
continue
db.commit()
# add the references
if vulnid is not None and references:
for reference in references:
# check to see if reference exists first
query = (db.t_vuln_refs.f_source == reference[0]) & (
db.t_vuln_refs.f_text == reference[1])
ref_id = db.t_vuln_refs.update_or_insert(
query, f_source=reference[0], f_text=reference[1])
if not ref_id:
ref_id = db(query).select().first().id
# make many-to-many relationship with t_vuln_data
db.t_vuln_references.update_or_insert(
f_vuln_ref_id=ref_id, f_vulndata_id=vulnid)
db.commit()
from skaldship.exploits import connect_exploits
connect_exploits()
log(" [-] Added Nexpose vulnerability: %s" % nexid)
stats['added'] += 1
response.flash = "%s added, %s skipped" % (stats['added'],
stats['invalid'])
return dict(form=form)
else:
response.flash = "Unable to login to Nexpose"
elif form.errors:
response.flash = "Error in form"
return dict(form=form)
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
),
group_name=settings.scheduler_group_name,
sync_output=5,
timeout=settings.scheduler_timeout)
if task.id:
redirect(URL('tasks', 'status', args=task.id))
else:
response.flash = "Error submitting job: %s" % (task.errors)
else:
from skaldship.nexpose import process_xml
log("Starting Nexpose XML Import")
process_xml(
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
)
response.flash = "Nexpose XML upload complete"
redirect(URL('default', 'index'))
return dict(form=form)
