def list_lootfiles():
"""
Lists local loot files for import processing into Kvasir. This does not
use the Metasploit API and depends upon a directory being local to the
web2py server instance. The API is used to check if pro is installed
and sets the loot_dir to Linux or Windows path
"""
import os
import re
response.title = "%s :: Metasploit Loots" % (settings.title)
msf_settings = msf_get_config(session)
dbsvcs = db.t_services
loot_dir = request.args(0)
if not loot_dir:
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
if msf.pro_about():
if platform in ["linux", "linux2"]:
loot_dir = "/opt/metasploit_pro/apps/pro/loot"
else:
loot_dir = "C:\\Metasploit\\apps\\pro\\loot"
except ImportError, error:
pass
def task_list():
"""Obtains a list of tasks"""
msf_settings = msf_get_config(session)
response.title = "%s :: Metasploit Task List" % (settings.title)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(error=str(error), alert=True, tasks=None)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
except MSFProAPIError as error:
return dict(error=str(error), alert=True, tasks=None)
tasks = msf.task_list()
tasklist = []
if 'status' in request.vars:
# only return specific tasks as defined in status
for taskid, task in tasks.items():
if task['status'] == request.vars.status.lower():
tasklist.append({taskid: task})
else:
tasklist = tasks
return dict(tasks=tasklist)
def list_lootfiles():
"""
Lists local loot files for import processing into Kvasir. This does not
use the Metasploit API and depends upon a directory being local to the
web2py server instance. The API is used to check if pro is installed
and sets the loot_dir to Linux or Windows path
"""
import os
import re
response.title = "%s :: Metasploit Loots" % (settings.title)
msf_settings = msf_get_config(session)
dbsvcs = db.t_services
loot_dir = request.args(0)
if not loot_dir:
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
if msf.pro_about():
if platform in ["linux", "linux2"]:
loot_dir = "/opt/metasploit_pro/apps/pro/loot"
else:
loot_dir = "C:\\Metasploit\\apps\\pro\\loot"
except ImportError, error:
pass
def import_pwdump():
"""Downloads a pwdump loot and processes it"""
msf_settings = msf_get_config(session)
alert = False
error = None
response.title = "%s :: Import Metasploit PWDUMP Loot" % (settings.title)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(alert=True, error=str(error), form=None)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
data = msf.loot_list(msf_settings['workspace'])
except MSFProAPIError as error:
return dict(alert=True, error=str(error), form=None)
if not alert:
loot_list = [] # list of loot IDs and IPs
loot_hosts = {} # mapping of IP to loot IDs
for k, v in data.items():
if v['ltype'] == 'host.windows.pwdump' or v[
'ltype'] == 'windows.hashes':
loot_list.append([k, v['host']])
loot_hosts.setdefault(v['host'], k)
form = SQLFORM.factory(
Field('hosts',
'list',
requires=IS_IN_SET(loot_list, multiple=True),
label=T('Host')),
Field('host_text', 'text', label=T('Host list (1 per line)')),
Field('addevidence', 'boolean', label=T('Add to Evidence')),
)
if form.accepts(request, session):
from skaldship.metasploit import process_pwdump_loot
data = []
# based on which form data is entered, make a new loot_list
if len(form.vars.hosts) > 0:
loot_list = form.vars.hosts
elif len(form.vars.host_text) > 0:
for ip in form.vars.host_text.split('\n'):
try:
loot_list.append(loot_hosts[ip])
except:
logging.debug("%s not found in MSF loot list" % (ip))
continue
retval = process_pwdump_loot(loot_list, msf)
response.flash = "PWDUMP files imported\n%s" % (retval)
elif form.errors:
response.flash = "Errors in your form"
else:
form = None
return dict(form=form, alert=alert, error=str(error))
def import_report():
"""
Import a MSF Pro XML Report.
TODO: FINISH HIM!
"""
msf_settings = msf_get_config(session)
if msf_settings['workspace'] is None:
redirect(URL('api_settings'))
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
if not msf.login():
response.flash = "Error logging into Metasploit, check your settings"
redirect(URL('api_settings'))
form = SQLFORM.factory(
Field('whitelist', 'text', label=T('Whitelist hosts/nets')),
Field('blacklist', 'text', label=T('Blacklist hosts/nets')),
)
if form.accepts(request, sesssion):
# build the configuration hash
rpt_data = {}
rpt_data['DS_REPORT_TYPE'] = 'XML'
rpt_data['DS_WHITELIST_HOSTS'] = form.vars.whitelist
rpt_data['DS_BLACKLIST_HOSTS'] = form.vars.blacklist
rpt_data['Workdspace'] = msf_settings['workspace']
# send the report request and get the task id
rpt_taskid = msf.pro_start_report(rpt_data)
def import_report():
"""
Import a MSF Pro XML Report.
TODO: FINISH HIM!
"""
msf_settings = msf_get_config(session)
if msf_settings['workspace'] is None:
redirect(URL('api_settings'))
msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
if not msf.login():
response.flash = "Error logging into Metasploit, check your settings"
redirect(URL('api_settings'))
form = SQLFORM.factory(
Field('whitelist', 'text', label=T('Whitelist hosts/nets')),
Field('blacklist', 'text', label=T('Blacklist hosts/nets')),
)
if form.accepts(request, sesssion):
# build the configuration hash
rpt_data = {}
rpt_data['DS_REPORT_TYPE'] = 'XML'
rpt_data['DS_WHITELIST_HOSTS'] = form.vars.whitelist
rpt_data['DS_BLACKLIST_HOSTS'] = form.vars.blacklist
rpt_data['Workdspace'] = msf_settings['workspace']
# send the report request and get the task id
rpt_taskid = msf.pro_start_report(rpt_data)
def import_screenshots():
"""
Import Screenshot files from Metasploit Pro into Kvasir
"""
response.title = "%s :: Import Metasploit Screenshots" % (settings.title)
msf_settings = msf_get_config(session)
loot_apidata = {}
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(form=None, error=str(error), alert=True)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
loot_apidata = msf.loot_list(msf_settings['workspace'])
except MSFProAPIError as error:
return dict(form=None, error=str(error), alert=True)
loot_list = []
loot_dict = {}
loot_hosts = {}
for k, v in loot_apidata.items():
if v['ltype'] == 'host.windows.screenshot':
loot_list.append([k, v['host']])
loot_dict.setdefault(k, v['host'])
loot_hosts.setdefault(v['host'], k)
form = SQLFORM.factory(
Field('host',
'list',
requires=IS_IN_SET(loot_list, multiple=True),
label=T('Host')),
Field('host_text', 'text', label=T('Host list (1 per line)')),
)
if form.accepts(request, session):
loots = []
# based on which form data is entered, make a new loot_list
if form.vars.hosts:
loot_list = form.vars.hosts
elif form.vars.host_text:
for ip in form.vars.host_text.split('\n'):
try:
loot_list.append(loot_hosts[ip])
except:
logging.debug("%s not found in MSF loot list" % (ip))
continue
loot_count = process_screenshot_loot(loot_list, msf)
repsonse.flash = 'Screenshots added for %s host(s)' % (loot_count)
elif form.errors:
response.flash = "Errors in your form"
return dict(form=form, alert=False, error=None)
def api_settings():
"""Settings Metasploit API"""
msf_settings = msf_get_config(session)
response.title = "%s :: Metasploit API Settings" % (settings.title)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(error=str(error), alert=True, form=None)
error = None
alert = False
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
workspaces = [w for w in list(msf.pro_workspaces().keys())]
users = [u for u in list(msf.pro_users().get('users').keys())]
except MSFProAPIError as e:
error = str(e)
alert = True
workspaces = []
users = []
form = SQLFORM.factory(
Field('workspace',
'string',
default=msf_settings['workspace'],
label=T('Workspace Name'),
requires=IS_IN_SET(workspaces)),
Field('workspace_num',
'string',
default=msf_settings['ws_num'],
label=T('Workspace Number')),
Field('user',
'string',
default=msf_settings['user'],
label=T('MSF User'),
requires=IS_IN_SET(users)),
Field('url', 'string', default=msf_settings['url'],
label=T('MSF URL')),
Field('key', 'string', default=msf_settings['key'],
label=T('API Key')),
)
# NOTE: workspace_num must be manually entered since there's no way for us
# to learn it from the API. We're just guessing otherwise - 1 is the default
# workspace so it's more likely to exist
if form.accepts(request, session):
settings.msf_workspace = form.vars.workspace
settings.msf_workspace_num = form.vars.workspace_num
session.msf_key = form.vars.key
session.msf_url = form.vars.url
session.msf_user = form.vars.user
response.flash = "MSF Settings updated"
elif form.errors:
response.flash = "Errors in your form!"
return dict(form=form, error=str(error), alert=False)
def import_scan():
"""
Upload and import Nexpose Scan file
"""
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
from skaldship.general import check_datadir
import time
import os
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nessus Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
nessus_config = nessus_get_config()
# {'ignored_vulnids': [19506, 11219, 34277],
# 'servers': {'server_1': {'password': '******',
# 'url': 'https://localhost:8834/',
# 'user': '******'}}}
nessusreports = [[0, None]]
import NessusAPI
#if auth.user.f_nessus_host is not None:
servers = nessus_config.get('servers', {})
for k, v in servers.iteritems():
try:
# check to see if NessusAPI is working
nessus = NessusAPI.NessusConnection(v.get('user'),
v.get('password'),
url=v.get('url'))
reports = nessus.list_reports()
for report in reports:
ts = time.ctime(float(report.timestamp))
nessusreports.append([
"%s:%s" % (k, report.name),
"%s: %s - %s (%s)" %
(k, report.readablename, ts, report.status)
])
except Exception, e:
logger.error("Error communicating with %s: %s" % (k, str(e)))
def import_xml_scan():
"""
Upload/import Nexpose XML Scan file via scheduler task
"""
from NexposeAPI import NexposeAPI, Sites, Report
from skaldship.general import check_datadir
from skaldship.metasploit import msf_get_config
import time
import os
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nexpose XML Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
# check to see if nexpose is configured/active and get site listing
nexpose_config = nexpose_get_config()
nxsitelist = []
if nexpose_config['host'] is not None and nexpose_config[
'user'] is not None:
# see if the host is open/active first
if nexpose_config['host'] is not None:
sites = Sites()
sites.host = nexpose_config['host']
sites.port = nexpose_config['port']
try:
if sites.login(user_id=nexpose_config['user'],
password=nexpose_config['password']):
sites = sites.listings()
nxsitelist.append([0, None])
for k, v in sites.iteritems():
nxsitelist.append([int(k), sites[k]['name']])
except Exception, e:
pass
def import_report_xml():
"""
Upload/import Metasploit XML export file
"""
import time
import os
from skaldship.general import check_datadir
msf_settings = msf_get_config(session)
response.title = "%s :: Import Metasploit Pro Report XML" % (settings.title)
filedir = os.path.join(request.folder,'data','scanfiles')
fields = []
alert = False
error = None
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append( [ user.id, user.username ] )
fields.append(Field('f_filename', 'upload', uploadfolder=filedir, label=T('Metasploit XML File')))
# check to see if we have a Metasploit Pro instance configured and talking
# if so pull a list of the workspaces and present them
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
except ImportError, error:
msf = None
def import_xml_scan():
"""
Upload/import Nexpose XML Scan file via scheduler task
"""
from NexposeAPI import NexposeAPI, Sites, Report
from skaldship.general import check_datadir
from skaldship.metasploit import msf_get_config
import time
import os
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings["url"], apikey=msf_settings["key"])
working_msf_api = msf_api.login()
except:
working_msf_api = False
filedir = os.path.join(request.folder, "data", "scanfiles")
response.title = "%s :: Import Nexpose XML Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
# check to see if nexpose is configured/active and get site listing
nexpose_config = nexpose_get_config()
nxsitelist = []
if nexpose_config["host"] is not None and nexpose_config["user"] is not None:
# see if the host is open/active first
if nexpose_config["host"] is not None:
sites = Sites()
sites.host = nexpose_config["host"]
sites.port = nexpose_config["port"]
try:
if sites.login(user_id=nexpose_config["user"], password=nexpose_config["password"]):
sites = sites.listings()
nxsitelist.append([0, None])
for k, v in sites.iteritems():
nxsitelist.append([int(k), sites[k]["name"]])
except Exception, e:
pass
def import_scan():
"""
Upload and import Nessus Scan file
"""
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
from skaldship.general import check_datadir
import time
import os
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nessus Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
nessus_config = nessus_get_config()
# {'ignored_vulnids': [19506, 11219, 34277],
# 'servers': {'server_1': {'password': '******',
# 'url': 'https://localhost:8834/',
# 'user': '******'}}}
nessusreports = [[0, None]]
import NessusAPI
#if auth.user.f_nessus_host is not None:
servers = nessus_config.get('servers', {})
for k, v in servers.iteritems():
try:
# check to see if NessusAPI is working
nessus = NessusAPI.NessusConnection(v.get('user'), v.get('password'), url=v.get('url'))
reports = nessus.list_reports()
for report in reports:
ts = time.ctime(float(report.timestamp))
nessusreports.append(["%s:%s" % (k, report.name), "%s: %s - %s (%s)" % (k, report.readablename, ts, report.status)])
except Exception, e:
logger.error("Error communicating with %s: %s" % (k, str(e)))
def exploit_host():
"""
Build an exploit for a specific target
TODO: Finish metasploit.exploit_host
"""
msf_settings = msf_get_config(session)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(error=str(error), alert=True, form=None)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
except MSFProAPIError as error:
return dict(error=str(error), alert=True, form=None)
target = request.vars.f_target or None
exploit = request.vars.f_exploit or None
form = SQLFORM.factory()
def task_stop():
"""Stop a running task"""
msf_settings = msf_get_config(session)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(error=str(error), alert=True, form=None)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
except MSFProAPIError as error:
return dict(error=str(error), alert=True, form=None)
if 'taskid' not in request.vars:
tasks = msf.task_list()
task_list = []
for taskid, task in tasks.items():
if tasks[taskid]['status'] == 'running':
task_list.append([
taskid,
"%s (%s) :: %s :: %s" % (
taskid,
tasks[taskid]['status'],
tasks[taskid]['description'],
tasks[taskid]['info'],
)
])
form = SQLFORM.factory(
Field('taskid',
'string',
requires=IS_IN_SET(task_list),
label=T('Task ID')))
return dict(form=form)
response.title = "%s :: Stop Metasploit Task" % (settings.title)
data = msf.task_stop(request.vars.taskid)
return dict(data=data)
def import_xml_scan():
"""
Upload/import Nexpose XML Scan file via scheduler task
"""
from NexposeAPI import NexposeAPI, Sites, Report
from skaldship.general import check_datadir
from skaldship.metasploit import msf_get_config
import time
import os
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nexpose XML Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
# check to see if nexpose is configured/active and get site listing
nexpose_config = nexpose_get_config()
nxsitelist = []
if nexpose_config['host'] is not None and nexpose_config[
'user'] is not None:
# see if the host is open/active first
if nexpose_config['host'] is not None:
sites = Sites()
sites.host = nexpose_config['host']
sites.port = nexpose_config['port']
try:
if sites.login(user_id=nexpose_config['user'],
password=nexpose_config['password']):
sites = sites.listings()
nxsitelist.append([0, None])
for k, v in sites.items():
nxsitelist.append([int(k), sites[k]['name']])
except Exception as e:
pass
if nxsitelist:
fields.append(
Field('f_nexpose_site',
type='integer',
label=T('Nexpose Site'),
requires=IS_IN_SET(nxsitelist, zero=None)))
fields.append(
Field('f_filename',
'upload',
uploadfolder=filedir,
label=T('Nexpose XML File')))
fields.append(
Field('f_engineer',
type='integer',
label=T('Engineer'),
default=auth.user.id,
requires=IS_IN_SET(userlist)))
fields.append(
Field('f_asset_group',
type='string',
label=T('Asset Group'),
requires=IS_NOT_EMPTY()))
# If Metasploit available, pull a list of the workspaces and present them
if working_msf_api:
msf_workspaces = []
msf_workspaces.append("None")
for w in list(msf_api.pro_workspaces().keys()):
msf_workspaces.append(w)
fields.append(
Field('f_msf_workspace',
type='string',
label=T('MSF Pro Workspace'),
requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None))))
fields.append(
Field('f_include_list', type='text', label=T('Hosts to Only Include')))
fields.append(
Field('f_ignore_list', type='text', label=T('Hosts to Ignore')))
fields.append(
Field('f_update_hosts',
type='boolean',
label=T('Update Host Information'),
default=False))
fields.append(
Field('f_taskit',
type='boolean',
default=auth.user.f_scheduler_tasks,
label=T('Run in background task')))
form = SQLFORM.factory(*fields, table_name='nexpose_xml')
# form processing
if form.errors:
response.flash = 'Error in form'
elif form.accepts(request.vars, session):
# process a nexpose file
if not nxsitelist:
nexpose_site = '0'
else:
nexpose_site = form.vars.f_nexpose_site
if nexpose_site != '0':
report = Report()
report.host = nexpose_config['host']
report.port = nexpose_config['port']
nx_loggedin = report.login(user_id=nexpose_config['user'],
password=nexpose_config['password'])
if nx_loggedin:
# have nexpose generate the adhoc report
check_datadir(request.folder)
filename = os.path.join(
filedir,
"%s-%s.xml" % (form.vars.f_asset_group, int(time.time())))
fout = open(filename, "w")
fout.write(report.adhoc_generate(filterid=nexpose_site))
fout.close()
else:
response.flash = "Unable to login to Nexpose"
return dict(form=form)
else:
filename = form.vars.f_filename
filename = os.path.join(filedir, form.vars.f_filename)
# build the hosts only/exclude list
ip_exclude = []
data = form.vars.get('f_ignore_list')
if data:
ip_exclude = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_include = []
data = form.vars.get('f_include_list')
if data:
ip_include = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if form.vars.f_msf_workspace:
msf_workspace = form.vars.f_msf_workspace
if msf_workspace == "None":
msf_workspace = None
else:
msf_workspace = None
msf_settings = {
'workspace': msf_workspace,
'url': msf_settings['url'],
'key': msf_settings['key']
}
if form.vars.f_taskit:
task = scheduler.queue_task(
scanner_import,
pvars=dict(
scanner='nexpose',
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
),
group_name=settings.scheduler_group_name,
sync_output=5,
timeout=settings.scheduler_timeout)
if task.id:
redirect(URL('tasks', 'status', args=task.id))
else:
response.flash = "Error submitting job: %s" % (task.errors)
else:
from skaldship.nexpose import process_xml
log("Starting Nexpose XML Import")
process_xml(
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
)
response.flash = "Nexpose XML upload complete"
redirect(URL('default', 'index'))
return dict(form=form)
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nmap XML Scan file
import re
import os
from skaldship.hosts import get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
from zenmapCore_Kvasir.NmapParser import NmapParser
from gluon import current
T = current.T
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS computer name: (?P<d>.*),')
RE_NETBIOS_WORKGROUP = re.compile('Workgroup: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile(
'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nmap scan file %s" % (filename))
nmap_parsed = NmapParser()
nmap_parsed.parse_file(filename)
#existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the hosts, where all the goodies are
log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts)))
hoststats = {}
hoststats['added'] = 0
hoststats['skipped'] = 0
hoststats['updated'] = 0
hoststats['errored'] = 0
hosts = [] # array of host_id fields
svc_db = db.t_services
for node in nmap_parsed.hosts:
nodefields = {}
if node.ipv6:
ipaddr = node.ipv6
nodefields['f_ipaddr'] = ipaddr
elif node.ip.get('type') == 'ipv4':
ipaddr = node.ip.get('addr')
nodefields['f_ipaddr'] = ipaddr
else:
log(" [!] No IPv4/IPv6 address, skipping")
continue
try:
nodefields['f_macaddr'] = node.mac['addr']
except TypeError:
nodefields['f_macaddr'] = None
status = node.state
log(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats['skipped'] += 1
continue
if ipaddr in ip_exclude:
log(" [-] Host is in exclude list... skipping")
hoststats['skipped'] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Host is not in the only list... skipping")
hoststats['skipped'] += 1
continue
if not node.ports and not addnoports:
log(" [-] No ports open and not asked to add those kind... skipping"
)
hoststats['skipped'] += 1
continue
# we'll just take the last hostname in the names list since it'll usually be the full dns name
for name in node.hostnames:
nodefields['f_hostname'] = name['hostname']
nodefields['f_engineer'] = engineer
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
# see if host exists, if so update. if not, insert!
query = (db.t_hosts.f_ipaddr == ipaddr)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
log(" [-] Adding %s" % (ipaddr))
elif host_rec is not None and update_hosts:
db.commit()
host_id = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats['updated'] += 1
log(" [-] Updating %s" % (ipaddr))
else:
hoststats['skipped'] += 1
db.commit()
log(" [-] Skipped %s" % (ipaddr))
continue
hosts.append(host_id)
# process OS related info
for os in node.osmatches:
os_id = None
host_id = None
f_title = os['name'] #title
for k in os['osclasses']:
if k.get('cpe') != None: #fixes error on loading cpe:/os
f_cpename = k['cpe'].replace('cpe:/o:', '')
f_vendor = k['vendor']
f_product = k['osfamily']
f_version = k['osgen']
f_class = k['type']
f_family = k['osfamily']
f_certainty = k['accuracy']
cpe_res = db((db.t_os.f_cpename == f_cpename) & (
db.t_os.f_title == f_title)).select().first()
if cpe_res is not None:
os_id = cpe_res.id
else:
try:
os_id = db.t_os.insert(
f_cpename=f_cpename,
f_title=f_title,
f_vendor=f_vendor,
f_product=f_product,
f_version=f_version,
)
except Exception as e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
if os_id and (f_class or f_family or f_certainty):
ipaddr = node.ip.get('addr')
host_id = get_host_record(ipaddr)
host_id = host_id.id
try:
db.t_host_os_refs.insert(f_certainty=f_certainty,
f_family=f_family,
f_class=f_class,
f_hosts_id=host_id,
f_os_id=os_id)
except Exception as e:
logger.error("Error inserting OS: %s" % (e))
db.commit()
# process non-port <hostscript> entries. Add to info/0:
for hostscripts in node.hostscripts:
query = (svc_db.f_proto == 'info') & (svc_db.f_number == 0) & (
svc_db.f_hosts_id == host_id)
svc_id = db.t_services.update_or_insert(query,
f_proto='info',
f_number=0,
f_status='open',
f_hosts_id=host_id)
if not svc_id:
svc_rec = db(query).select(cache=(cache.ram, 180)).first()
if svc_rec:
svc_id = svc_rec.id
else:
log(" [!] Service record wasn't created", logging.ERROR)
continue
db.commit()
for script in hostscripts:
script_id = script.id
output = script.output
db.t_service_info.update_or_insert(f_services_id=svc_id,
f_name=script_id,
f_text=output)
db.commit()
if script_id == 'nbstat':
# pull out NetBIOS info from nbstat output
result = RE_NETBIOS_MAC.search(output)
if 'd' in result.groupdict():
host_rec.update(f_macaddr=result.group('d'))
db.commit()
result = RE_NETBIOS_NAME.search(output)
if 'd' in result.groupdict():
host_rec.update(f_netbios_name=result.group('d'))
db.commit()
result = RE_NETBIOS_WORKGROUP.search(output)
if 'd' in result.groupdict():
db(
db.t_netbios.update_or_insert(
f_hosts_id=host_id,
f_domain=result.group('d')))
db.commit()
# add ports and resulting vulndata
for port in node.ports:
f_proto = port.get('protocol')
f_number = port.get('portid')
f_status = port.get('port_state')
f_name = port.get('service_name')
f_product = port.get('service_product')
query = (svc_db.f_proto == f_proto) & (
svc_db.f_number == f_number) & (svc_db.f_hosts_id == host_id)
svc = db(query).select().first()
# if record does not exist, query returns None so add the record:
if svc is None:
log(" [-] Adding port: %s/%s (%s)" %
(f_proto, f_number, f_name))
svc_id = db.t_services.update_or_insert(f_proto=f_proto,
f_number=f_number,
f_status=f_status,
f_name=f_name,
f_hosts_id=host_id)
if not (svc is None):
log(" [-] Updating port: %s/%s (%s)" %
(f_proto, f_number, f_name))
if svc.f_name != f_name and f_name not in svc.f_name:
svc_id = db.t_services.validate_and_update(
_key=svc.id, f_name=(f_name + ' | ' + svc.f_name))
else:
svc_id = db.t_services.validate_and_update(
_key=svc.id, f_name=(svc.f_name))
svc_id = svc_id.id
if f_product:
version = port.get('service_version')
if version:
f_product += " (%s)" % (version)
db.t_service_info.update_or_insert(f_services_id=svc_id,
f_name=f_name,
f_text=f_product)
db.commit()
# Process <script> service entries
for script in port.get('scripts'):
try:
if script.get('id') == 'ssl-cert':
text = script.get('output')
sslcert = text.replace("/", "\n")
db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=script.get('id'),
f_text=sslcert)
else:
db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=script.get('id'),
f_text=script.get('output'))
except Exception as e:
logger.error("Error inserting Script: %s" % (e))
db.commit()
# check for banner id and update t_services banner field with the output
if script.get('id') == "banner":
try:
db.t_services.update_or_insert(
(db.t_services.id == svc_id),
f_banner=script.get('output'))
except Exception as e:
logger.error("Error inserting Banner: %s" % (e))
db.commit()
# Process <cpe> service entries
port_cpe = port.get('service_cpe')
if port_cpe:
cpe_id = port_cpe.replace('cpe:/', '')
if cpe_id.startswith('a'):
# process CPE Applications
#log(" [-] Found Application CPE data: %s" % (cpe_id))
db.t_service_info.update_or_insert(f_services_id=svc_id,
f_name='cpe.app',
f_text="cpe:/%s" %
(cpe_id))
db.commit()
elif cpe_id.startswith('o'):
# process CPE Operating System
os_id = lookup_cpe(cpe_id[2:])
if os_id is not None:
db.t_host_os_refs.insert(f_certainty='0.9',
f_family='Unknown',
f_class='Other',
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
# So no CPE or existing OS data, lets split up the CPE data and make our own
log(" [!] No os_id found, this is odd !!!")
# Adding uptime. Needed to add a table "f_uptime" in t_hosts db!
if node.uptime['lastboot']:
db.t_hosts.update_or_insert((db.t_hosts.f_ipaddr == ipaddr),
f_uptime=node.uptime['lastboot'])
if not node.uptime['lastboot']:
db.t_hosts.update_or_insert((db.t_hosts.f_ipaddr == ipaddr),
f_uptime=T("no entry found"))
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'),
apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception as error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error),
logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception as error:
log(
" [!] Error loading scan data to send to Metasploit: %s" %
str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
task = msf_api.pro_import_data(
msf_settings.get('workspace'),
"".join(scan_data),
{
#'preserve_hosts': form.vars.preserve_hosts,
'blacklist_hosts': "\n".join(ip_ignore_list)
},
)
msf_workspace_num = session.msf_workspace_num or 'unknown'
msfurl = os.path.join(msf_settings.get('url'), 'workspaces',
msf_workspace_num, 'tasks', task['task_id'])
log(" [*] Added file to MSF Pro: %s" % msfurl)
# any new nexpose vulns need to be checked against exploits table and connected
log(" [*] Connecting exploits to vulns and performing do_host_status")
do_host_status(asset_group=asset_group)
log(" [*] Import complete: hosts: %s added, %s updated, %s skipped" % (
hoststats['added'],
hoststats['updated'],
hoststats['skipped'],
))
def exploit():
"""
Launches Metasploit Pro Exploit based upon a list of host records
"""
response.title = "%s :: Metasploit Pro Exploit" % (settings.title)
msf_settings = msf_get_config(session)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(alert=True, error=str(error), form=None)
host_records = request.vars.host_records
if host_records:
def host_to_ip(host_rec):
if isinstance(host_rec, (int, str)):
host_rec = get_host_record(host_rec)
if not host_rec:
return None
return host_rec.get('f_ipaddr')
target_ips = '\n'.join(
[host_to_ip(x) for x in host_records.split('|')])
else:
target_ips = ''
module_list = []
alert = False
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
module_list = msf.module_list(modtype='exploits').get('modules')
except MSFProAPIError as error:
return dict(alert=True, error=str(error), form=None)
form = SQLFORM.factory(
Field(
'targets',
'text',
default=target_ips,
label=T('Targets'),
requires=IS_NOT_EMPTY(),
comment=
T('Targets to scan can be IP Addresses, ranged lists or subnets. One per line.'
)),
Field(
'blacklist_hosts',
'text',
label=T('Blacklisted Targets'),
comment=
T('Targets to blacklist can be IP Addresses, ranged lists or subnets. One per line.'
)),
Field(
'ports',
'string',
default='1-65535',
label=T('Ports'),
requires=IS_NOT_EMPTY(),
comment=
T('List of ports to match exploits to. Example: 21-23,80,443,8000-8999'
)),
Field(
'blacklist_ports',
'string',
label=T('Blacklisted Ports'),
comment=T(
'List of ports to not exploit. Example: 21-23,80,443,8000-8999'
)),
Field('min_rank',
'string',
default='great',
label=T('Minmum Exploit Rank'),
requires=IS_IN_SET(
['low', 'average', 'normal', 'good', 'great', 'excellent']),
comment=T('Minimum reliability level of exploits to include')),
Field('exploit_speed',
'integer',
default=5,
label=T('Parallel Exploits'),
requires=IS_INT_IN_RANGE(1, 11),
comment=T('How many exploits to run in parallel (1-10)')),
Field('exploit_timeout',
'integer',
default=5,
label=T('Timeout (in minutes)'),
requires=IS_INT_IN_RANGE(0, 1440),
comment=T(
'Maximum time (in minutes) an exploit is allowed to run')),
Field('limit_sessions',
'boolean',
default=True,
label=T('Limit sessions'),
comment=T('Limit sessions to only one per exploited host')),
Field(
'ignore_fragile',
'boolean',
default=True,
label=T('Skip "fragile" devices'),
comment=
T('Avoid exploit attempts on fragile systems such as network devices and printers.'
)),
Field(
'filter_by_os',
'boolean',
default=True,
label=T('OS'),
comment=
T('Match exploits to Operating System, known vulnerabilities or ports'
)),
Field('filter_by_vuln',
'boolean',
default=True,
label=T('Vulnerabilities')),
Field('filter_by_ports', 'boolean', default=True, label=T('Ports')),
Field('dry_run',
'boolean',
default=False,
label=T('Dry run'),
comment=T('Prepare for execution but do nothing')),
Field('payload',
'string',
default='auto',
label=T('Payload method'),
requires=IS_IN_SET(['auto', 'reverse', 'bind'])),
Field('payload_type',
'string',
default='meterpreter',
label=T('Paylod type'),
requires=IS_IN_SET(['meterpreter', 'shell'])),
Field('payload_ports',
'string',
default='4000-5000',
label=T('Payload ports'),
requires=IS_NOT_EMPTY(),
comment=T('Port range for reverse/connect payloads')),
Field('evasion_tcp',
'integer',
default=0,
label=T('TCP Evasion Level'),
requires=IS_INT_IN_RANGE(0, 4)),
Field('evasion_app',
'integer',
default=0,
label=T('Application Evasion'),
requires=IS_INT_IN_RANGE(0, 4)),
Field(
'modules',
'list:string',
label=T('Specifc Module(s)'),
requires=IS_EMPTY_OR(IS_IN_SET(module_list, multiple=True)),
comment=
T('A whitelist of modules to execute, by default all that match are tried'
)),
table_name='msfpro_exploit',
_class="form-horizontal")
if form.process().accepted:
args = {
'workspace': msf_settings['workspace'],
'username': msf_settings['user'],
'DS_WHITELIST_HOSTS': form.vars.targets,
'DS_BLACKLIST_HOSTS': form.vars.blacklist_hosts,
'DS_WHITELIST_PORTS': form.vars.ports,
'DS_BLACKLIST_PORTS': form.vars.blacklist_ports,
'DS_MinimumRank': form.vars.min_rank,
'DS_EXPLOIT_SPEED': form.vars.exploit_speed,
'DS_EXPLOIT_TIMEOUT': form.vars.exploit_timeout,
'DS_LimitSessions': form.vars.limit_sessions,
'DS_IgnoreFragileDevices': form.vars.ignore_fragile,
'DS_FilterByOS': form.vars.filter_by_os,
'DS_MATCH_VULNS': form.vars.filter_by_vuln,
'DS_MATCH_PORTS': form.vars.filter_by_ports,
'DS_OnlyMatch': form.vars.dry_run,
'DS_PAYLOAD_METHOD': form.vars.payload,
'DS_PAYLOAD_TYPE': form.vars.payload_type,
'DS_PAYLOAD_PORTS': form.vars.payload_ports,
'DS_EVASION_LEVEL_TCP': form.vars.evasion_tcp,
'DS_EVASION_LEVEL_APP': form.vars.evasion_app,
#'DS_ModuleFilter': form.vars.filter_by_os,
}
task = msf.start_exploit(args)
msfurl = os.path.join(msf_settings['url'], 'workspaces',
msf_settings['workspace_num'], 'tasks',
task['task_id'])
redirect(msfurl)
elif form.errors:
response.flash = "Error in form"
return dict(form=form, alert=alert)
@auth.requires_login()
def api_settings():
"""Settings Metasploit API"""
msf_settings = msf_get_config(session)
response.title = "%s :: Metasploit API Settings" % (settings.title)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError, error:
return dict(error=str(error), alert=True, form=None)
error=None
alert=False
msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
try:
workspaces = [w for w in msf.pro_workspaces().keys()]
users = [u for u in msf.pro_users().get('users').keys()]
except MSFProAPIError, e:
error = str(e)
alert = True
workspaces = []
users = []
form=SQLFORM.factory(
Field('workspace', 'string', default=msf_settings['workspace'], label=T('Workspace Name'), requires=IS_IN_SET(workspaces)),
Field('workspace_num', 'string', default=msf_settings['ws_num'], label=T('Workspace Number')),
Field('user', 'string', default=msf_settings['user'], label=T('MSF User'), requires=IS_IN_SET(users)),
Field('url', 'string', default=msf_settings['url'], label=T('MSF URL')),
Field('key', 'string', default=msf_settings['key'], label=T('API Key')),
def import_scan():
"""
Upload and import Nessus Scan file
"""
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
from skaldship.general import check_datadir
import time
import os
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nessus Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
nessus_config = nessus_get_config()
# {'ignored_vulnids': [19506, 11219, 34277],
# 'servers': {'server_1': {'password': '******',
# 'url': 'https://localhost:8834/',
# 'user': '******'}}}
nessusreports = [[0, None]]
import NessusAPI
from skaldship.nessus.ness6api import Nessus6API
servers = nessus_config.get('servers', {})
for k, v in servers.iteritems():
if v.get('version') == 6:
# nessus >= 6
try:
nessus = Nessus6API(
url=v.get('url'),
username=v.get('username'),
password=v.get('password'),
access_key=v.get('access_key'),
secret_key=v.get('secret_key'),
verify=v.get('verify_ssl'),
proxies=v.get('proxies')
)
except Exception, e:
logger.error("Error communicating with %s: %s" % (k, str(e)))
try:
for report in nessus.get_scans():
ts = time.ctime(float(report.get('last_modification_date', 0)))
nessusreports.append([
"%s:%s" % (k, report.get('id')),
"%s: %s - %s (%s)" % (k, report.get('name'), ts, report.get('status'))
])
except Exception, e:
logger.error("Error making scan list: %s" % (str(e)))
def import_report_xml():
"""
Upload/import Metasploit XML export file
"""
import time
import os
from skaldship.general import check_datadir
msf_settings = msf_get_config(session)
response.title = "%s :: Import Metasploit Pro Report XML" % (
settings.title)
filedir = os.path.join(request.folder, 'data', 'scanfiles')
fields = []
alert = False
error = None
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
fields.append(
Field('f_filename',
'upload',
uploadfolder=filedir,
label=T('Metasploit XML File')))
# check to see if we have a Metasploit Pro instance configured and talking
# if so pull a list of the workspaces and present them
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
except ImportError as error:
msf = None
if msf:
try:
msf_reports_res = msf.report_list(
workspace=msf_settings['workspace'])
except MSFProAPIError as error:
msf_reports_res = None
if msf_reports_res:
from datetime import datetime
msf_reports = []
for rpt in list(msf_reports_res.keys()):
report_name = "Generated: %s" % (datetime.strftime(
datetime.fromtimestamp(msf_reports_res[rpt]['created_at']),
"%m-%d-%y %H:%M:%S"))
msf_reports.append([rpt, report_name])
fields.append(
Field('f_msf_report',
type='string',
label=T('MSF Pro Report'),
requires=IS_EMPTY_OR(IS_IN_SET(msf_reports, zero=None))))
fields.append(
Field('f_engineer',
type='integer',
label=T('Engineer'),
default=auth.user.id,
requires=IS_IN_SET(userlist)))
fields.append(
Field('f_asset_group',
type='string',
label=T('Asset Group for new Hosts'),
default="Metasploit Import",
requires=IS_NOT_EMPTY()))
fields.append(
Field('f_include_list', type='text', label=T('Hosts to Only Include')))
fields.append(
Field('f_ignore_list', type='text', label=T('Hosts to Ignore')))
fields.append(
Field('f_update_hosts',
type='boolean',
default=True,
label=T('Update Existing Hosts')))
fields.append(
Field('f_taskit',
type='boolean',
default=auth.user.f_scheduler_tasks,
label=T('Run in background task')))
form = SQLFORM.factory(*fields, table_name='metasploit_xml')
if form.errors:
response.flash = 'Error in form'
elif form.accepts(request.vars, session):
# build the hosts only/exclude list
ip_exclude = []
data = form.vars.get('f_ignore_list')
if data:
ip_exclude = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_include = []
data = form.vars.get('f_include_list')
if data:
ip_include = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if form.vars.f_msf_report:
try:
msf_report = msf.report_download(rptid=form.vars.f_msf_report)
except MSFProAPIError as error:
error = "Unable to download report from Metasploit Pro: %s" % (
str(error))
return dict(form=form, alert=True, error=error)
check_datadir(request.folder)
filename = os.path.join(
filedir, "msfpro-%s-%s.xml" %
(msf_settings['workspace'], int(time.time())))
fout = open(filename, "w")
fout.write(msf_report['data'])
fout.close()
del (msf_report)
else:
filename = os.path.join(filedir, form.vars.f_filename)
if form.vars.f_taskit:
task = scheduler.queue_task(
scanner_import,
pvars=dict(
scanner='metasploit',
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
),
group_name=settings.scheduler_group_name,
sync_output=5,
timeout=settings.scheduler_timeout)
if task.id:
redirect(URL('tasks', 'status', args=task.id))
else:
response.flash = "Error submitting job: %s" % (task.errors)
else:
from skaldship.metasploit.pro import process_report_xml
logger.info("Starting Metasploit XML Import")
result = process_report_xml(
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
)
response.flash = result
redirect(URL('default', 'index'))
return dict(form=form, alert=alert, error=error)
def send_accounts():
"""Builds a list of username:passwords and sends it to Metasploit"""
msf_settings = msf_get_config(session)
response.title = "%s :: Send Kvasir Passwords to Metasploit Pro" % (
settings.title)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(error=str(error), alert=True, form=None)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
except MSFProAPIError as error:
return dict(error=str(error), alert=True, form=None)
form = SQLFORM.factory(
Field('userpass', 'boolean', default=True,
label=T('User/Pass Combos')),
Field('pwdump', 'boolean', default=True, label=T('PWDUMP Hashes')),
)
if form.accepts(request, session):
pass
elif form.errors:
reseponse.flash = "Error in your form"
return dict(form=form, alert=False, error=None)
else:
return dict(form=form, alert=False, error=None)
"""
First build a list of username:passwords from the t_accounts database and
make a temporary file, then start_import_creds the file
Second build a pwdump list for LM/NT hashes, make a temporary file, then
start_import_creds that file!
Requires MSFPRO and Kvasir be on the same workstation.
"""
tasks = {}
import tempfile
if form.vars.userpass:
# build username:password file
rows = db(db.t_accounts.f_compromised == True).select(
db.t_accounts.f_username,
db.t_accounts.f_password,
cache=(cache.ram, 60))
if rows is not None:
tmpfile = tempfile.NamedTemporaryFile(delete=False)
fname = tmpfile.name
for row in rows:
tmpfile.write("%s %s\n" % (row.f_username, row.f_password))
tmpfile.close()
opts = {
'workspace': msf_workspace,
'DS_FTYPE': 'userpass',
'DS_IMPORT_PATH': fname,
'DS_NAME': 'Kvasir import %s' % (fname),
'DS_DESC': 'Kvasir import',
'DS_REMOVE_FILE': True,
}
task = msf.pro_start_import_creds(opts)
redirect(URL('task_log', args=task.get('id')))
else:
response.flash = "No user:pass combos to import"
if form.vars.pwdump:
# build pwdump file
rows = db(db.t_accounts.f_hash1_type == "LM").select(
db.t_accounts.f_username,
db.t_accounts.f_uid,
db.t_accounts.f_hash1,
db.t_accounts.f_hash2,
cache=(cache.ram, 60))
if rows is not None:
tmpfile = tempfile.NamedTemporaryFile(delete=False)
fname = tmpfile.name
for row in rows:
tmpfile.write(
"%s:%s:%s:%s:::\n" %
(row.f_username, row.f_uid, row.f_hash1, row.f_hash2))
tmpfile.close()
opts = {
'workspace': msf_workspace,
'DS_FTYPE': 'pwdump',
'DS_IMPORT_PATH': fname,
'DS_NAME': 'Kvasir pwdump import %s' % (fname),
'DS_DESC': 'Kvasir pwdump import',
'DS_REMOVE_FILE': True,
}
task = msf.pro_start_import_creds(opts)
redirect(URL('task_log', args=task.get('id')))
else:
response.flash = "No pwdump hashes to import"
return dict(form=form, alert=False, error=None)
def process_scanfile(
filename=None,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
"""
Process a Nessus XML or CSV Report file. There are two types of CSV output, the first
is very basic and is generated by a single Nessus instance. The second comes from the
centralized manager. I forget what it's called but it packs more data. If you have a
standalone scanner, always export/save as .nessus.
:param filename: A local filename to process
:param asset_group: Asset group to assign hosts to
:param engineer: Engineer record number to assign hosts to
:param msf_workspace: If set a Metasploit workspace to send the scanfile to via the API
:param ip_ignore_list: List of IP addresses to ignore
:param ip_include_list: List of IP addresses to ONLY import (skip all others)
:param update_hosts: Boolean to update/append to hosts, otherwise hosts are skipped
:returns msg: A string status message
"""
from skaldship.cpe import lookup_cpe
nessus_config = nessus_get_config()
db = current.globalenv['db']
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nessus scan file %s" % filename)
fIN = open(filename, "rb")
# check to see if file is a CSV file, if so set nessus_csv to True
line = fIN.readline()
fIN.seek(0)
if line.startswith('Plugin'):
import csv
csv.field_size_limit(sys.maxsize) # field size must be increased
nessus_iterator = csv.DictReader(fIN)
nessus_csv_type = 'Standalone'
log(" [*] CSV file is from Standalone scanner")
elif line.startswith('"Plugin"'):
import csv
csv.field_size_limit(sys.maxsize) # field size must be increased
nessus_iterator = csv.DictReader(fIN)
nessus_csv_type = 'SecurityCenter'
log(" [*] CSV file is from SecurityCenter")
else:
nessus_csv_type = False
try:
nessus_xml = etree.parse(filename)
log(" [*] XML file identified")
except etree.ParseError as e:
msg = " [!] Invalid Nessus scan file (%s): %s " % (filename, e)
log(msg, logging.ERROR)
return msg
root = nessus_xml.getroot()
nessus_iterator = root.findall("Report/ReportHost")
nessus_hosts = NessusHosts(engineer, asset_group, ip_include_list,
ip_ignore_list, update_hosts)
nessus_vulns = NessusVulns()
services = Services()
svcs = db.t_services
def _plugin_parse(host_id, vuln_id, vulndata, vulnextradata):
# Parse the plugin data. This is where CSV and XML diverge.
port = vulnextradata['port']
proto = vulnextradata['proto']
svcname = vulnextradata['svcname']
plugin_output = vulnextradata['plugin_output']
pluginID = vulnextradata['pluginID']
# check to see if we are to ignore this plugin ID or not.
if int(pluginID) in nessus_config.get('ignored_plugins'):
return
svc_fields = {
'f_proto': proto,
'f_number': port,
'f_name': svcname,
'f_hosts_id': host_id
}
svc_rec = services.get_record(**svc_fields)
# Nessus only guesses the services (and appends a ? at the end)
splited = svc_fields['f_name'].split("?")
if svc_rec is not None:
if splited[0] != svc_rec.f_name and svc_rec.f_name not in splited[
0]:
svc_fields['f_name'] = "%s | %s" % (svc_rec.f_name, splited[0])
svc_id = svcs.update_or_insert(_key=svc_rec.id, **svc_fields)
else:
svc_fields['f_name'] = splited[0]
svc_rec = services.get_record(create_or_update=True, **svc_fields)
# create t_service_vulns entry for this pluginID
svc_vuln = {
'f_services_id': svc_rec.id,
'f_vulndata_id': vuln_id,
'f_proof': plugin_output
}
# you may be a vulnerability if...
if vulnextradata['exploit_available'] == 'true':
# if you have exploits available you may be an extra special vulnerability
svc_vuln['f_status'] = 'vulnerable-exploited'
elif svcname == 'general':
# if general service then you may not be a vulnerability
svc_vuln['f_status'] = 'general'
elif vulndata['f_severity'] == 0:
# if there is no severity then you may not be a vulnerability
svc_vuln['f_status'] = 'general'
else:
# you're a vulnerability
svc_vuln['f_status'] = 'vulnerable'
db.t_service_vulns.update_or_insert(**svc_vuln)
######################################################################################################
## Let the parsing of Nessus Plugin Output commence!
##
## Many Plugins provide useful data in plugin_output. We'll go through the list here and extract
## out the good bits and add them to Kvasir's database. Some Plugins will not be added as vulnerabilities
## because they're truly informational. This list will change if somebody keeps it up to date.
##
## TODO: This should be moved into a separate function so we can also process csv data
## TODO: Add t_service_info key/value records (standardize on Nexpose-like keys?)
##
######################################################################################################
d = {}
nessus_vulns.stats['added'] += 1
#### SNMP
if pluginID == '10264':
# snmp community strings
for snmp in re.findall(' - (.*)', plugin_output):
db.t_snmp.update_or_insert(f_hosts_id=host_id,
f_community=snmp)
db.commit()
#### SMB/NetBIOS
if pluginID in ['10860', '10399']:
# SMB Use Host SID (10860) or Domain SID (10399) to enumerate users
for user in re.findall(' - (.*)', plugin_output):
username = user[0:user.find('(') - 1]
try:
gid = re.findall('\(id (\d+)', user)[0]
except:
gid = '0'
# Windows users, local groups, and global groups
d['f_username'] = username
d['f_gid'] = gid
d['f_services_id'] = svc_rec.id
d['f_source'] = '10860'
db.t_accounts.update_or_insert(**d)
db.commit()
if pluginID == '17651':
# Microsoft Windows SMB : Obtains the Password Policy
d['f_hosts_id'] = host_id
try:
d['f_lockout_duration'] = re.findall(
'Locked account time \(s\): (\d+)', plugin_output)[0]
d['f_lockout_limit'] = re.findall(
'Number of invalid logon before locked out \(s\): (\d+)',
plugin_output)[0]
except IndexError:
d['f_lockout_duration'] = 1800
d['f_lockout_limit'] = 0
db.t_netbios.update_or_insert(**d)
db.commit()
if pluginID == '10395':
# Microsoft Windows SMB Shares Enumeration
d['f_hosts_id'] = host_id
d['f_shares'] = re.findall(' - (.*)', plugin_output)
db.t_netbios.update_or_insert(**d)
if pluginID == '10150':
# Windows NetBIOS / SMB Remote Host Information Disclosure
try:
d['f_hosts_id'] = host_id
d['f_domain'] = re.findall('(\w+).*= Workgroup / Domain name',
plugin_output)[0]
db.t_netbios.update_or_insert(**d)
except IndexError:
pass
#### FTP
if pluginID == '10092':
# FTP Server Detection
RE_10092 = re.compile('The remote FTP banner is :\n\n(.*)',
re.DOTALL)
try:
d['f_banner'] = RE_10092.findall(plugin_output)[0]
svc_rec.update(**d)
db.commit()
db(db.t_service_info)
db.t_service_info.update_or_insert(f_services_id=svc_rec.id,
f_name='ftp.banner',
f_text=d['f_banner'])
db.commit()
except Exception as e:
log("Error parsing FTP banner (id 10092): %s" % str(e),
logging.ERROR)
#### SSH
if pluginID == '10267':
# SSH Server Type and Version Information
try:
ssh_banner, ssh_supported_auth = re.findall(
'SSH version : (.*)\nSSH supported authentication : (.*)',
plugin_output)[0]
d['f_banner'] = ssh_banner
svc_rec.update_record(**d)
db.commit()
db.t_service_info.update_or_insert(f_services_id=svc_rec.id,
f_name='ssh.banner',
f_text=d['f_banner'])
db.t_service_info.update_or_insert(f_services_id=svc_rec.id,
f_name='ssh.authentication',
f_text=ssh_supported_auth)
db.commit()
except Exception as e:
log("Error parsing SSH banner (id 10267): %s" % str(e),
logging.ERROR)
if pluginID == '10881':
# SSH Protocol Versions Supported
try:
ssh_versions = re.findall(' - (.*)', plugin_output)
db.t_service_info.update_or_insert(
f_services_id=svc_rec.id,
f_name='ssh.versions',
f_text=', '.join(ssh_versions))
db.commit()
except Exception as e:
log("Error parsing SSH versions (id 10881): %s" % str(e),
logging.ERROR)
try:
fingerprint = re.findall('SSHv2 host key fingerprint : (.*)',
plugin_output)
db.t_service_info.update_or_insert(f_services_id=svc_rec.id,
f_name='ssh.fingerprint',
f_text=fingerprint[0])
db.commit()
except Exception as e:
log("Error parsing SSH fingerprint (id 10881): %s" % str(e),
logging.ERROR)
### Telnet
if pluginID in ['10281', '42263']:
# Telnet banner
try:
snip_start = plugin_output.find(
'snip ------------------------------\n')
snip_end = plugin_output.rfind(
'snip ------------------------------\n')
if snip_start > 0 and snip_end > snip_start:
d['f_banner'] = plugin_output[snip_start + 36:snip_end -
36]
svc_rec.update(**d)
db.commit()
else:
log(
"Error finding Telnet banner: (st: %s, end: %s, banner: %s)"
% (snip_start, snip_end, plugin_output), logging.ERROR)
except Exception as e:
log("Error parsing Telnet banner: %s" % str(e), logging.ERROR)
### HTTP Server Info
if pluginID == '10107':
# HTTP Server Type and Version
RE_10107 = re.compile('The remote web server type is :\n\n(.*)',
re.DOTALL)
try:
d['f_banner'] = RE_10107.findall(plugin_output)[0]
svc_rec.update(**d)
db.commit()
db.t_service_info.update_or_insert(f_services_id=svc_rec.id,
f_name='http.banner.server',
f_text=d['f_banner'])
db.commit()
except Exception as e:
log("Error parsing HTTP banner (id 10107): %s" % str(e),
logging.ERROR)
### Operating Systems and CPE
if pluginID == '45590':
# Common Platform Enumeration (CPE)
for cpe_os in re.findall('(cpe:/o:.*)', plugin_output):
os_id = lookup_cpe(cpe_os.replace('cpe:/o:', '').rstrip(' '))
if os_id:
db.t_host_os_refs.update_or_insert(
f_certainty='0.90', # just a stab
f_family='Unknown', # not given in Nessus
f_class=hostdata.get('system-type'),
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
for host in nessus_iterator:
if not nessus_csv_type:
(host_id, hostdata,
hostextradata) = nessus_hosts.parse(host.find('HostProperties'))
else:
(host_id, hostdata, hostextradata) = nessus_hosts.parse(host)
if not host_id:
# no host_id returned, it was either skipped or errored out
continue
if not nessus_csv_type:
# Parse the XML <ReportItem> sections where plugins, ports and output are all in
for rpt_item in host.iterfind('ReportItem'):
(vuln_id, vulndata, extradata) = nessus_vulns.parse(rpt_item)
if not vuln_id:
# no vulnerability id
continue
_plugin_parse(host_id, vuln_id, vulndata, extradata)
else:
(vuln_id, vulndata, extradata) = nessus_vulns.parse(host)
_plugin_parse(host_id, vuln_id, vulndata, extradata)
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'),
apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception as error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error),
logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception as error:
log(
" [!] Error loading scan data to send to Metasploit: %s" %
str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
task = msf_api.pro_import_data(
msf_settings.get('workspace'),
"".join(scan_data),
{
#'preserve_hosts': form.vars.preserve_hosts,
'blacklist_hosts': "\n".join(ip_ignore_list)
},
)
msf_workspace_num = session.msf_workspace_num or 'unknown'
msfurl = os.path.join(msf_settings.get('url'), 'workspaces',
msf_workspace_num, 'tasks', task['task_id'])
log(" [*] Added file to MSF Pro: %s" % msfurl)
# any new Nessus vulns need to be checked against exploits table and connected
log(" [*] Connecting exploits to vulns and performing do_host_status")
connect_exploits()
do_host_status(asset_group=asset_group)
msg = (' [*] Import complete: hosts: %s added, %s updated, %s skipped '
'- %s vulns processed, %s added' %
(nessus_hosts.stats['added'], nessus_hosts.stats['updated'],
nessus_hosts.stats['skipped'], nessus_vulns.stats['processed'],
nessus_vulns.stats['added']))
log(msg)
return msg
@auth.requires_login()
def api_settings():
"""Settings Metasploit API"""
msf_settings = msf_get_config(session)
response.title = "%s :: Metasploit API Settings" % (settings.title)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError, error:
return dict(error=str(error), alert=True, form=None)
error=None
alert=False
msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
try:
workspaces = [w for w in msf.pro_workspaces().keys()]
users = [u for u in msf.pro_users().get('users').keys()]
except MSFProAPIError, e:
error = str(e)
alert = True
workspaces = []
users = []
form=SQLFORM.factory(
Field('workspace', 'string', default=msf_settings['workspace'], label=T('Workspace Name'), requires=IS_IN_SET(workspaces)),
Field('workspace_num', 'string', default=msf_settings['ws_num'], label=T('Workspace Number')),
Field('user', 'string', default=msf_settings['user'], label=T('MSF User'), requires=IS_IN_SET(users)),
Field('url', 'string', default=msf_settings['url'], label=T('MSF URL')),
Field('key', 'string', default=msf_settings['key'], label=T('API Key')),
# Parse the XML <ReportItem> sections where plugins, ports and output are all in
for rpt_item in host.iterfind('ReportItem'):
(vuln_id, vulndata, extradata) = nessus_vulns.parse(rpt_item)
if not vuln_id:
# no vulnerability id
continue
_plugin_parse(host_id, vuln_id, vulndata, extradata)
else:
(vuln_id, vulndata, extradata) = nessus_vulns.parse(host)
_plugin_parse(host_id, vuln_id, vulndata, extradata)
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'),
apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception, error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error),
logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception, error:
log(
" [!] Error loading scan data to send to Metasploit: %s" %
str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
def send_scanxml():
"""Sends scan XML output file to MSF Pro for importing"""
import os
response.title = "%s :: Send Scan XML Data to Metasploit" % (
settings.title)
msf_settings = msf_get_config(session)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(error=str(error), alert=True, form=None)
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
except MSFProAPIError as error:
return dict(error=str(error), alert=True, form=None)
filedir = os.path.join(request.folder, 'data', 'scanfiles')
try:
scanfiles = os.listdir(filedir)
except OSError:
scanfiles = []
file_select = []
count = 0
for fn in scanfiles:
file_select.append([count, fn])
count += 1
form = SQLFORM.factory(
Field('fname',
'string',
requires=IS_IN_SET(file_select),
label=T('Scan File')),
Field(
'blacklist',
'text',
label=T('Blacklisted hosts'),
comment=
T('Targets to blacklist can be IP Addresses, ranged lists or subnets. One per line.'
)),
Field('preserve_hosts',
'boolean',
default=False,
label=T('Preserve existing hosts')),
)
if form.accepts(request, session):
fname = file_select[int(form.vars.fname)][1]
fname = os.path.join(filedir, fname)
try:
scan_data = open(fname, "r+").readlines()
except Exception as error:
return dict(form=form, error=str(error), alert=True)
task = msf.pro_import_data(
msf_workspace,
"".join(scan_data),
{
'preserve_hosts': form.vars.preserve_hosts,
'blacklist_hosts': "\n".join(form.vars.blacklist)
},
)
"""
# documented in API but not valid yet @9/6/13
#validate = msf.pro_validate_import_file(fname)
task = msf.pro_start_import({
'workspace': msf_workspace,
'username': msf_settings['user'],
'DS_PATH': fname,
'DS_PRESERVE_HOSTS': form.vars.preserve_hosts,
'DS_BLACKLIST_HOSTS': "\n".join(form.vars.blacklist),
'DS_REMOVE_FILE': False,
'DS_ImportTags': True,
})
"""
msfurl = os.path.join(msf_settings['url'], 'workspaces',
msf_settings['workspace_num'], 'tasks',
task['task_id'])
redirect(msfurl)
elif form.errors:
response.flash = "Errors in your form"
return dict(form=form, alert=False, error=None)
def import_xml_scan():
"""
Upload/import Nmap XML Scan file via scheduler task
"""
import time
from skaldship.general import check_datadir
from skaldship.metasploit import msf_get_config
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
filedir = os.path.join(request.folder,'data','scanfiles')
check_datadir(request.folder)
response.title = "%s :: Import Nmap XML Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append( [ user.id, user.username ] )
fields.append(Field('f_filename', 'upload', uploadfolder=filedir, label=T('Nmap XML File')))
fields.append(Field('f_engineer', type='integer', label=T('Engineer'), default=auth.user.id, requires=IS_IN_SET(userlist)))
fields.append(Field('f_asset_group', type='string', label=T('Asset Group'), requires=IS_NOT_EMPTY()))
# If Metasploit available, pull a list of the workspaces and present them
if working_msf_api:
msf_workspaces = []
msf_workspaces.append( "None" )
for w in msf_api.pro_workspaces().keys():
msf_workspaces.append(w)
fields.append(Field('f_msf_workspace', type='string', label=T('MSF Pro Workspace'), requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None))))
fields.append(Field('f_addnoports', type='boolean', label=T('Add Hosts w/o Ports'), default=False))
fields.append(Field('f_include_list', type='text', label=T('Hosts to Only Include')))
fields.append(Field('f_ignore_list', type='text', label=T('Hosts to Ignore')))
fields.append(Field('f_update_hosts', type='boolean', label=T('Update Host Information'), default=False))
fields.append(Field('f_taskit', type='boolean', default=auth.user.f_scheduler_tasks, label=T('Run in background task')))
form = SQLFORM.factory(*fields, table_name='nmap_xml')
if form.errors:
response.flash = 'Error in form'
elif form.accepts(request.vars, session):
# process a nmap file
filename = os.path.join(filedir, form.vars.f_filename)
# build the hosts only/exclude list
ip_exclude = []
data = form.vars.get('f_ignore_list')
if data:
ip_exclude = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_include = []
data = form.vars.get('f_include_list')
if data:
ip_include = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if form.vars.f_msf_workspace:
msf_workspace = form.vars.f_msf_workspace
if msf_workspace == "None":
msf_workspace = None
else:
msf_workspace = None
msf_settings = {'workspace': msf_workspace, 'url': msf_settings['url'], 'key': msf_settings['key']}
if form.vars.f_taskit:
task = scheduler.queue_task(
scanner_import,
pvars=dict(
scanner='nmap',
filename=filename,
addnoports=form.vars.f_addnoports,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
),
group_name=settings.scheduler_group_name,
sync_output=5,
timeout=settings.scheduler_timeout
)
if task.id:
redirect(URL('tasks', 'status', args=task.id))
else:
response.flash = "Error submitting job: %s" % (task.errors)
else:
from skaldship.nmap import process_xml
print("Starting Nmap XML Import")
process_xml(
filename=filename,
addnoports=form.vars.f_addnoports,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
)
response.flash = "Nmap XML upload complete"
redirect(URL('default', 'index'))
return dict(form=form)
def list_lootfiles():
"""
Lists local loot files for import processing into Kvasir. This does not
use the Metasploit API and depends upon a directory being local to the
web2py server instance. The API is used to check if pro is installed
and sets the loot_dir to Linux or Windows path
"""
import os
import re
response.title = "%s :: Metasploit Loots" % (settings.title)
msf_settings = msf_get_config(session)
dbsvcs = db.t_services
loot_dir = request.args(0)
if not loot_dir:
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
if msf.pro_about():
if platform in ["linux", "linux2"]:
loot_dir = "/opt/metasploit_pro/apps/pro/loot"
else:
loot_dir = "C:\\Metasploit\\apps\\pro\\loot"
except ImportError as error:
pass
if not loot_dir:
from sys import platform
if platform in ["linux", "linux2", "darwin", "freebsd"]:
loot_dir = os.path.join(os.environ.get('HOME'), '.msf4/loot')
elif platform in ["win32", "cygwin"]:
loot_dir = '$FINDYOUR/msf4/loot/path'
try:
os.chdir(loot_dir)
loot_files = os.listdir(loot_dir)
except OSError:
loot_files = []
loot_file_details = []
for loot in loot_files:
try:
(timestamp, workspace, ipaddr, filetype,
extension) = re.split('_', loot)
except ValueError:
logging.warn("Invalid loot file: %s" % (loot))
continue
# TODO: service_list = get_services(ipaddr)
host_rec = get_host_record(ipaddr)
services = []
for service in db(dbsvcs.f_hosts_id == host_rec).select(
dbsvcs.id, dbsvcs.f_proto, dbsvcs.f_number,
cache=(cache.ram, 120)):
services.append(
[service.id,
"%s/%s" % (service.f_proto, service.f_number)])
loot_file_details.append([workspace, ipaddr, services, filetype])
form_lootdir = SQLFORM.factory(
Field('lootdir',
'string',
default=loot_dir,
requires=IS_NOT_EMPTY(),
label=T('Metasploit Loot Directory')), )
return dict(form_lootdir=form_lootdir, loot_file_details=loot_file_details)
def process_xml(
filename=None,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nexpose XML Scan file
from skaldship.cpe import lookup_cpe
from skaldship.hosts import get_host_record
from gluon.validators import IS_IPADDRESS
import os
db = current.globalenv['db']
session = current.globalenv['session']
parser = html.parser.HTMLParser()
user_id = db.auth_user(engineer)
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nexpose scan file %s" % filename)
try:
nexpose_xml = etree.parse(filename)
except etree.ParseError as e:
msg = " [!] Invalid Nexpose XML file (%s): %s " % (filename, e)
log(msg, logging.ERROR)
return msg
root = nexpose_xml.getroot()
existing_vulnids = db(db.t_vulndata()).select(
db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
log(" [*] Found %d vulnerabilities in the database already." %
len(existing_vulnids))
# start with the vulnerability details
vulns_added = 0
vulns_skipped = 0
vulns = root.findall("VulnerabilityDefinitions/vulnerability")
log(" [*] Parsing %d vulnerabilities" % len(vulns))
for vuln in vulns:
# nexpose identifiers are always lower case in kvasir. UPPER CASE IS FOR SHOUTING!!!
vulnid = vuln.attrib['id'].lower()
if vulnid in existing_vulnids:
#log(" [-] Skipping %s - It's in the db already" % vulnid)
vulns_skipped += 1
else:
# add the vulnerability to t_vulndata - any duplicates are errored out
(vulnfields, references) = vuln_parse(vuln, fromapi=False)
try:
vulnid = db.t_vulndata.update_or_insert(**vulnfields)
if not vulnid:
vulnid = db(db.t_vulndata.f_vulnid ==
vulnfields['f_vulnid']).select().first().id
vulns_added += 1
db.commit()
except Exception as e:
log(
" [!] Error inserting %s to vulndata: %s" %
(vulnfields['f_vulnid'], e), logging.ERROR)
vulnid = None
db.commit()
continue
# add the references
if vulnid is not None:
for reference in references:
# check to see if reference exists first
ref_id = db(db.t_vuln_refs.f_text == reference[1])
if ref_id.count() == 0:
# add because it doesn't
ref_id = db.t_vuln_refs.insert(f_source=reference[0],
f_text=reference[1])
db.commit()
else:
# pick the first reference as the ID
ref_id = ref_id.select()[0].id
# make many-to-many relationship with t_vuln_data
res = db.t_vuln_references.insert(f_vuln_ref_id=ref_id,
f_vulndata_id=vulnid)
db.commit()
log(" [*] %d Vulnerabilities added, %d skipped" %
(vulns_added, vulns_skipped))
# re-make the existing_vulnids dict() since we've updated the system
existing_vulnids = db(db.t_vulndata()).select(
db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the nodes now
nodes = root.findall("nodes/node")
log(" [-] Parsing %d nodes" % len(nodes))
hoststats = {'added': 0, 'skipped': 0, 'updated': 0, 'errored': 0}
hosts = [] # array of host_id fields
for node in nodes:
log(" [-] Node %s status is: %s" %
(node.attrib['address'], node.attrib['status']))
#sys.stderr.write(msg)
if node.attrib['status'] != "alive":
hoststats['skipped'] += 1
continue
if node.attrib['address'] in ip_exclude:
log(" [-] Node is in exclude list... skipping")
hoststats['skipped'] += 1
continue
nodefields = {}
if len(ip_only) > 0 and node.attrib['address'] not in ip_only:
log(" [-] Node is not in the only list... skipping")
hoststats['skipped'] += 1
continue
# we'll just take the last hostname in the names list since it'll usually be the full dns name
names = node.findall("names/name")
for name in names:
nodefields['f_hostname'] = name.text
ip = node.attrib['address']
if IS_IPADDRESS()(ip):
nodefields['f_ipaddr'] = ip
else:
log(" [!] Invalid IP Address: %s" % ip, logging.ERROR)
nodefields['f_engineer'] = user_id
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
if 'hardware-address' in node.attrib:
nodefields['f_macaddr'] = node.attrib['hardware-address']
if node.find('names/name') is not None:
# XXX: for now just take the first hostname
nodefields['f_hostname'] = node.find('names/name').text
# check to see if IP exists in DB already
query = (db.t_hosts.f_ipaddr == ip)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
log(" [-] Adding IP: %s" % ip)
elif update_hosts:
db.commit()
db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(nodefields['f_ipaddr'])
host_id = host_id.id
hoststats['updated'] += 1
log(" [-] Updating IP: %s" % ip)
else:
hoststats['skipped'] += 1
db.commit()
log(" [-] Skipped IP: %s" % ip)
continue
hosts.append(host_id)
# tests that aren't specific to any port we wrap up into a meta service
# called "INFO"
tests = node.findall("tests/test")
if len(tests) > 0:
svc_id = db.t_services.update_or_insert(f_proto="info",
f_number="0",
f_status="info",
f_hosts_id=host_id)
db.commit()
for test in tests:
d = {}
vulnid = test.get('id').lower()
# we may have valid username.
if "cifs-acct-" in vulnid:
username = test.get('key')
if username is not None:
d['f_services_id'] = svc_id
d['f_username'] = username
d['f_active'] = True
d['f_source'] = vulnid
query = (db.t_accounts.f_services_id == d['f_services_id']) &\
(db.t_accounts.f_username == d['f_username'])
db.t_accounts.update_or_insert(query, **d)
db.commit()
if test.attrib['status'] == 'vulnerable-exploited' or \
test.attrib['status'] == 'potential' or \
test.attrib['status'] == 'exception-vulnerable-exploited' or \
test.attrib['status'] == 'exception-vulnerable-version' or \
test.attrib['status'] == 'exception-vulnerable-potential' or \
test.attrib['status'] == 'vulnerable-version':
if vulnid in existing_vulnids:
vuln_id = existing_vulnids[vulnid]['id']
else:
continue
if vulnid == 'cifs-nt-0001':
# Windows users, local groups, and global groups
infotext = nx_xml_to_html(
StringIO(etree.tostring(test, xml_declaration=False)))
try:
unames = re.search(
"Found user\(s\): (?P<unames>.+?) </li>",
infotext).group('unames')
except AttributeError as e:
# regex not found
continue
for uname in unames.split():
# add account
d['f_username'] = uname
d['f_services_id'] = svc_id
d['f_source'] = 'cifs-nt-0001'
db.t_accounts.update_or_insert(**d)
db.commit()
test_str = etree.tostring(test,
xml_declaration=False,
encoding=str)
test_str = test_str.encode('ascii', 'xmlcharrefreplace')
proof = nx_xml_to_html(StringIO(test_str))
proof = html_to_markmin(proof)
if vulnid == 'cifs-insecure-acct-lockout-limit':
d['f_hosts_id'] = host_id
try:
d['f_lockout_limit'] = re.search(
"contains: (?P<l>\d+)", proof).group('l')
except AttributeError:
d['f_lockout_limit'] = 0
query = (db.t_netbios.f_hosts_id == host_id)
db.t_netbios.update_or_insert(query, **d)
db.commit()
# Check for CIFS uid/pw
if "cifs-" in vulnid:
try:
uid = re.search("uid\[(?P<u>.*?)\]", proof).group('u')
pw = re.search("pw\[(?P<p>.*?)\]", proof).group('p')
realm = re.search("realm\[(?P<r>.*?)\]",
proof).group('r')
d = {
'f_services_id': svc_id,
'f_username': uid,
'f_password': pw,
'f_description': realm,
'f_active': True,
'f_compromised': True,
'f_source': vulnid
}
query = (db.t_accounts.f_services_id
== svc_id) & (db.t_accounts.f_username == uid)
db.t_accounts.update_or_insert(query, **d)
db.commit()
except AttributeError:
db.commit()
except Exception as e:
log("Error inserting account (%s): %s" % (uid, e),
logging.ERROR)
db.commit()
# solaris-kcms-readfile shadow file
if vulnid.lower() == "rpc-solaris-kcms-readfile":
# funky chicken stuff, if they mess with this output then we've got to
# change this around as well. thems the breaks, maynard!
shadow = parser.unescape(proof)
for line in shadow.split("<br />")[1:-1]:
user, pw, uid = line.split(':')[0:3]
d['f_services_id'] = svc_id
d['f_username'] = user
d['f_hash1'] = pw
d['f_hash1_type'] = "crypt"
d['f_uid'] = uid
d['f_source'] = "shadow"
d['f_active'] = True
d['f_source'] = "rpc-solaris-kcms-readfile"
query = (db.t_accounts.f_services_id == svc_id) & (
db.t_accounts.f_username == user)
db.t_accounts.update_or_insert(query, **d)
db.commit()
db.t_service_vulns.update_or_insert(
f_services_id=svc_id,
f_status=test.attrib['status'],
f_proof=proof,
f_vulndata_id=vuln_id)
if "cisco-default-http-account" in vulnid.lower():
d['f_services_id'] = svc_id
d['f_username'] = vulnid.split('-')[4]
d['f_password'] = vulnid.split('-')[6]
d['f_source'] = "cisco-default-http-account"
query = (db.t_accounts.f_services_id == svc_id) & (
db.t_accounts.f_username == d['f_username'])
db.t_accounts.update_or_insert(query, **d)
db.commit()
# add services (ports) and resulting vulndata
for endpoint in node.findall("endpoints/endpoint"):
f_proto = endpoint.attrib['protocol']
f_number = endpoint.attrib['port']
f_status = endpoint.attrib['status']
query = (db.t_services.f_hosts_id == host_id) \
& (db.t_services.f_proto == f_proto) \
& (db.t_services.f_number == f_number)
svc_id = db.t_services.update_or_insert(query,
f_proto=f_proto,
f_number=f_number,
f_status=f_status,
f_hosts_id=host_id)
if not svc_id:
svc_id = db(query).select().first().id
for service in endpoint.findall("services/service"):
d = {}
if 'name' in service.attrib:
db.t_services[svc_id] = dict(f_name=service.attrib['name'])
for test in service.findall("tests/test"):
vulnid = test.get('id').lower()
if test.attrib['status'] == 'vulnerable-exploited' or \
test.attrib['status'] == 'potential' or \
test.attrib['status'] == 'exception-vulnerable-exploited' or \
test.attrib['status'] == 'exception-vulnerable-version' or \
test.attrib['status'] == 'exception-vulnerable-potential' or \
test.attrib['status'] == 'vulnerable-version':
if vulnid in existing_vulnids:
vuln_id = existing_vulnids[vulnid]['id']
else:
log(
" [!] Unknown vulnid, Skipping! (id: %s)" %
vulnid, logging.ERROR)
continue
test_str = etree.tostring(test,
xml_declaration=False,
encoding=str)
test_str = test_str.encode('ascii',
'xmlcharrefreplace')
proof = nx_xml_to_html(StringIO(test_str))
proof = html_to_markmin(proof)
# Check for SNMP strings
if "snmp-read-" in vulnid:
snmpstring = re.search("pw\[(?P<pw>.*?)\]",
proof).group('pw')
db.t_snmp.update_or_insert(f_hosts_id=host_id,
f_community=snmpstring,
f_access="READ",
f_version="v1")
db.commit()
if "snmp-write" in vulnid:
snmpstring = re.search("pw\[(?P<pw>.*?)\]",
proof).group('pw')
db.t_snmp.update_or_insert(f_hosts_id=host_id,
f_community=snmpstring,
f_access="WRITE",
f_version="v1")
db.commit()
# TODO: account names
# Dell DRAC root/calvin
if vulnid == "http-drac-default-login":
d['f_services_id'] = svc_id
d['f_username'] = '******'
d['f_password'] = '******'
d['f_active'] = True
d['f_compromised'] = True
d['f_source'] = vulnid
query = (db.t_accounts.f_services_id == svc_id) & (
db.t_accounts.f_username == 'root')
db.t_accounts.update_or_insert(query, **d)
db.commit()
# Check for uid/pw
if "ftp-iis-" in vulnid or \
"telnet-" in vulnid or \
"cifs-" in vulnid or \
"tds-" in vulnid or \
"oracle-" in vulnid or \
"-default-" in vulnid or \
"ftp-generic-" in vulnid:
try:
uid = re.search("uid\[(?P<u>.*?)\]",
proof).group('u')
pw = re.search("pw\[(?P<p>.*?)\]",
proof).group('p')
realm = re.search("realm\[(?P<r>.*?)\]",
proof).group('r')
d['f_services_id'] = svc_id
d['f_username'] = uid
d['f_password'] = pw
d['f_description'] = realm
d['f_active'] = True
d['f_compromised'] = True
d['f_source'] = vulnid
query = (db.t_accounts.f_services_id
== svc_id) & (db.t_accounts.f_username
== uid)
db.t_accounts.update_or_insert(query, **d)
db.commit()
except AttributeError:
db.commit()
except Exception as e:
log(
"Error inserting account (%s): %s" %
(uid, e), logging.ERROR)
db.commit()
# cisco default http login accounts
if "cisco-default-http-account" in vulnid.lower():
d['f_services_id'] = svc_id
d['f_username'] = vulnid.split('-')[4]
d['f_password'] = vulnid.split('-')[6]
d['f_source'] = "cisco-default-http-account"
query = (db.t_accounts.f_services_id == svc_id) \
& (db.t_accounts.f_username == d['f_username'])
db.t_accounts.update_or_insert(query, **d)
db.commit()
db.t_service_vulns.update_or_insert(
f_services_id=svc_id,
f_status=test.attrib['status'],
f_proof=proof,
f_vulndata_id=vuln_id)
db.commit()
for config in service.findall("configuration/config"):
db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=config.attrib['name'],
f_text=config.text)
db.commit()
if re.match('\w+.banner$', config.attrib['name']):
db.t_services[svc_id] = dict(f_banner=config.text)
db.commit()
if config.attrib['name'] == 'mac-address':
# update the mac address of the host
db.t_hosts[host_id] = dict(f_macaddr=config.text)
db.commit()
if "advertised-name" in config.attrib['name']:
# netbios computer name
d = config.text.split(" ")[0]
if "Computer Name" in config.text:
data = {'f_netbios_name': d}
# if hostname isn't defined then lowercase netbios name and put it in
if db.t_hosts[host_id].f_hostname is None:
data['f_hostname'] = d.lower()
db(db.t_hosts.id == host_id).update(**data)
db.commit()
elif "Domain Name" in config.text:
query = (db.t_netbios.f_hosts_id == host_id)
db.t_netbios.update_or_insert(query,
f_hosts_id=host_id,
f_domain=d)
db.commit()
for os_rec in node.findall('fingerprints/os'):
"""
<os certainty="1.00" device-class="Workstation" vendor="Microsoft" family="Windows" product="Windows 2000 Professional" version="SP4" arch="x86"/>
if using SCAP output the os line looks like:
<os certainty="0.66" device-class="General" vendor="Microsoft" family="Windows" product="Windows XP" arch="x86" cpe="cpe:/o:microsoft:windows_xp::sp3"/>
"""
if 'cpe' in os_rec.attrib:
# we have a cpe entry from xml! hooray!
cpe_name = os_rec.attrib['cpe'].replace('cpe:/o:', '')
os_id = lookup_cpe(cpe_name)
else:
# no cpe attribute in xml, go through our messy lookup
os_id = guess_cpe_os(os_rec)
if os_id is not None:
db.t_host_os_refs.update_or_insert(
f_certainty=os_rec.attrib['certainty'],
f_family=os_rec.get('family', 'Unknown'),
f_class=os_rec.get('device-class', 'Other'),
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
log(
" [!] os_rec could not be parsed: %s" %
etree.tostring(os_rec), logging.ERROR)
db.commit()
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'),
apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception as error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error),
logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception as error:
log(
" [!] Error loading scan data to send to Metasploit: %s" %
str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
task = msf_api.pro_import_data(
msf_settings.get('workspace'),
"".join(scan_data),
{
#'preserve_hosts': form.vars.preserve_hosts,
'blacklist_hosts': "\n".join(ip_ignore_list)
},
)
msf_workspace_num = session.msf_workspace_num or 'unknown'
msfurl = os.path.join(msf_settings.get('url'), 'workspaces',
msf_workspace_num, 'tasks', task['task_id'])
log(" [*] Added file to MSF Pro: %s" % msfurl)
# any new nexpose vulns need to be checked against exploits table and connected
log(" [*] Connecting exploits to vulns and performing do_host_status")
connect_exploits()
do_host_status(asset_group=asset_group)
msg = " [*] Import complete: hosts: %s added, %s skipped, %s errors - vulns: %s added, %s skipped" % (
hoststats['added'], hoststats['skipped'], hoststats['errored'],
vulns_added, vulns_skipped)
log(msg)
return msg
def import_scan():
"""
Upload and import Nessus Scan file
"""
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
from skaldship.general import check_datadir
import time
import os
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nessus Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
nessus_config = nessus_get_config()
# {'ignored_vulnids': [19506, 11219, 34277],
# 'servers': {'server_1': {'password': '******',
# 'url': 'https://localhost:8834/',
# 'user': '******'}}}
nessusreports = [[0, None]]
import NessusAPI
from skaldship.nessus.ness6api import Nessus6API
servers = nessus_config.get('servers', {})
for k, v in servers.iteritems():
if v.get('version') == 6:
# nessus >= 6
try:
nessus = Nessus6API(url=v.get('url'),
username=v.get('username'),
password=v.get('password'),
access_key=v.get('access_key'),
secret_key=v.get('secret_key'),
verify=v.get('verify_ssl'),
proxies=v.get('proxies'))
except Exception, e:
logger.error("Error communicating with %s: %s" % (k, str(e)))
try:
for report in nessus.get_scans():
ts = time.ctime(
float(report.get('last_modification_date', 0)))
nessusreports.append([
"%s:%s" % (k, report.get('id')),
"%s: %s - %s (%s)" %
(k, report.get('name'), ts, report.get('status'))
])
except Exception, e:
logger.error("Error making scan list: %s" % (str(e)))
def bruteforce():
"""
Launches a Metasploit Pro Bruteforce based upon a list of host records
"""
response.title = "%s :: Metasploit Pro Bruteforce" % (settings.title)
msf_settings = msf_get_config(session)
try:
from MetasploitProAPI import MetasploitProAPI, MSFProAPIError
except ImportError as error:
return dict(alert=True, error=str(error), form=None)
host_records = request.vars.host_records
if host_records:
def host_to_ip(host_rec):
if isinstance(host_rec, (int, str)):
host_rec = get_host_record(host_rec)
if not host_rec:
return None
return host_rec.get('f_ipaddr')
target_ips = '\n'.join(
[host_to_ip(x) for x in host_records.split('|')])
else:
target_ips = ''
loot_list = [] # list of loot IDs and IPs
alert = False
msf = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
try:
msf.login()
except MSFProAPIError as error:
return dict(alert=True, error=str(error), form=None)
form = SQLFORM.factory(
Field(
'targets',
'text',
default=target_ips,
label=T('Targets'),
requires=IS_NOT_EMPTY(),
comment=
T('Targets to scan can be IP Addresses, ranged lists or subnets. One per line.'
),
),
Field(
'blacklist',
'text',
label=T('Blacklisted hosts'),
comment=
T('Targets to blacklist can be IP Addresses, ranged lists or subnets. One per line.'
)),
Field(
'stop_on_success',
'boolean',
default=True,
label=T('Stop on success'),
comment=T('Stop scanning a host after first user login success')),
Field('verbose', 'boolean', default=False, label=T('Verbose')),
Field('include_known',
'boolean',
default=True,
label=T('Include known'),
comment=T('Include known credentials from the Workspace')),
Field('dry_run',
'boolean',
default=False,
label=T('Dry run'),
comment=T('Prepare for execution but do nothing')),
Field(
'scope',
'string',
default='normal',
label=T('Scope'),
requires=IS_IN_SET([
'quick', 'defaults', 'normal', 'deep', 'known', 'imported',
'50k'
]),
),
Field('speed',
'string',
default=3,
label=T('Speed'),
requires=IS_IN_SET([[0, 'Glacial'], [1, 'Slow'], [2, 'Stealthy'],
[3, 'Normal'], [4, 'Fast'], [5, 'Turbo']])),
Field(
'services',
'list:string',
label=T('Services'),
requires=IS_EMPTY_OR(
IS_IN_SET([
'Telnet',
'SSH',
'SMB',
'VNC',
'SNMP',
'Postgres',
'MySQL',
'MSSQL',
'Oracle',
'DB2',
'FTP',
'HTTP',
'HTTPS',
'EXEC',
'LOGIN',
'SHELL',
],
multiple=(1, 17))),
comment=T('List of services to bruteforce, multiples permitted')),
Field(
'addl_creds',
'text',
label=T('Additional credentals'),
comment=
T('List additional credentials to test. One per line with space between username and password.'
)),
Field('getsession',
'boolean',
default=True,
label=T('Execute session'),
comment=T(
'On successful access pop a shell or meterpreter session')),
Field('payload',
'string',
default='auto',
label=T('Payload method'),
requires=IS_IN_SET(['auto', 'reverse', 'bind'])),
Field('payload_type',
'string',
default='meterpreter',
label=T('Paylod type'),
requires=IS_IN_SET(['meterpreter', 'shell'])),
Field(
'payload_ports',
'string',
default='4000-5000',
label=T('Payload ports'),
requires=IS_NOT_EMPTY(),
comment=T('Port range for reverse/connect payloads'),
),
Field('smb_domains',
'string',
label=T('SMB Domains'),
comment=T('List of SMB domains, separated by spaces, to use')),
Field(
'preverse_domains',
'boolean',
default=True,
label=T('Preserve Domains'),
comment=T('Use previously identified SMB Domains with usernames')),
Field('mssql_windows_auth',
'boolean',
default=False,
label=T('MSSQL Windows Auth'),
comment=T(
'MSSQL attempts should use NTLM instead of Standard mode')),
Field('skip_blank_pw', 'boolean', default=False, label=T('Blanks')),
Field('skip_machine_names',
'boolean',
default=False,
label=T('Machine names')),
Field('skip_builtin_windows',
'boolean',
default=False,
label=T('Built-in Windows')),
Field('skip_builtin_unix',
'boolean',
default=False,
label=T('Built-in UNIX')),
Field('recombine_creds',
'boolean',
default=False,
label=T('Recombine credentials')),
Field('max_guess_per_svc',
'integer',
default=0,
label=T('Per service'),
requires=IS_INT_IN_RANGE(0, 65535)),
Field('max_guess_per_user',
'integer',
default=0,
label=T('Per user'),
requires=IS_INT_IN_RANGE(0, 65535)),
Field('max_guess_overall',
'integer',
default=0,
label=T('Overall'),
requires=IS_INT_IN_RANGE(0, 65535)),
Field('max_time_per_svc',
'integer',
default=0,
label=T('Per service'),
requires=IS_INT_IN_RANGE(0, 1440),
comment=T('Maximum time to bruteforce per service (in minutes')),
Field('max_time',
'integer',
default=0,
label=T('Overall'),
requires=IS_INT_IN_RANGE(0, 65535),
comment=T('Maximum time to run brute force (in minutes)')),
table_name='msfpro_bruteforce',
_class="form-horizontal")
if form.process().accepted:
args = {
'workspace': msf_settings['msf_workspace'],
'username': msf_settings['user'],
'DS_WHITELIST_HOSTS': form.vars.targets,
'DS_BLACKLIST_HOSTS': form.vars.blacklist,
'DS_STOP_ON_SUCCESS': form.vars.stop_on_success,
'DS_VERBOSE': form.vars.verbose,
'DS_INCLUDE_KNOWN': form.vars.include_known,
'DS_DRY_RUN': form.vars.dry_run,
'DS_BRUTEFORCE_SCOPE': form.vars.scope,
'DS_BRUTEFORCE_SPEED': form.vars.speed,
'DS_BRUTEFORCE_SERVICES': " ".join(form.vars.services),
'DS_BRUTEFORCE_GETSESSION': form.vars.getsession,
'DS_QUICKMODE_CREDS': form.vars.addl_creds,
'DS_PAYLOAD_METHOD': form.vars.payload,
'DS_PAYLOAD_TYPE': form.vars.payload_type,
'DS_PAYLOAD_PORTS': form.vars.payload_ports,
'DS_SMB_DOMAINS': form.vars.smb_domains,
'DS_PRESERVE_DOMAINS': form.vars.preverse_domains,
'DS_MAXGUESSESPERSERVICE': form.vars.max_guess_per_svc,
'DS_MAXGUESSESPERUSER': form.vars.max_guess_per_user,
'DS_MAXGUESSESOVERALL': form.vars.max_guess_overall,
'DS_MAXMINUTESPERSERVICE': form.vars.max_time_per_svc,
'DS_MAXMINUTESOVERALL': form.vars.max_time,
'DS_BRUTEFORCE_SKIP_BLANK_PASSWORDS': form.vars.skip_blank_pw,
'DS_BRUTEFORCE_SKIP_MACHINE_NAMES': form.vars.skip_machine_names,
'DS_BRUTEFORCE_SKIP_BUILTIN_WINDOWS_ACCOUNTS':
form.vars.skip_builtin_windows,
'DS_BRUTEFORCE_SKIP_BLANK_BUILTIN_UNIX_ACCOUNTS':
form.vars.skip_builtin_unix,
'DS_BRUTEFORCE_RECOMBINE_CREDS': form.vars.recombine_creds,
'DS_MSSQL_WINDOWS_AUTH': form.vars.mssql_windows_auth
}
task = msf.start_bruteforce(args)
msfurl = os.path.join(msf_settings['url'], 'workspaces',
msf_settings['workspace_num'], 'tasks',
task['task_id'])
redirect(msfurl)
elif form.errors:
response.flash = "Error in form"
return dict(form=form, alert=alert, error=False)
(options, params) = optparser.parse_args()
rows = db(db.auth_user.username == options.engineer)
if rows.count() != 1:
exit("An error was encountered when selecting a user. Please try with a valid user name.")
msf_settings = msf_get_config(session)
msf_workspaces = [None]
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
if working_msf_api:
for w in msf_api.pro_workspaces().keys():
msf_workspaces.append(w)
try:
msf_workspace = msf_workspaces[int(options.msfidx)]
except IndexError:
exit("An invalid workspace index has been provided. Aborting.")
msf_settings = {'workspace': msf_workspace, 'url': msf_settings['url'], 'key': msf_settings['key']}
rows = db(db.auth_user.username == options.engineer)
if rows.count() != 1:
exit(
"An error was encountered when selecting a user. Please try with a valid user name."
)
msf_settings = msf_get_config(session)
msf_workspaces = [None]
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
if working_msf_api:
for w in msf_api.pro_workspaces().keys():
msf_workspaces.append(w)
try:
msf_workspace = msf_workspaces[int(options.msfidx)]
except IndexError:
exit("An invalid workspace index has been provided. Aborting.")
msf_settings = {
'workspace': msf_workspace,
f_family=os_rec.get('family', 'Unknown'),
f_class=os_rec.get('device-class', 'Other'),
f_hosts_id=host_id,
f_os_id=os_id
)
db.commit()
else:
log(" [!] os_rec could not be parsed: %s" % etree.tostring(os_rec), logging.ERROR)
db.commit()
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'), apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception, error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error), logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception, error:
log(" [!] Error loading scan data to send to Metasploit: %s" % str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
task = msf_api.pro_import_data(
msf_settings.get('workspace'),
"".join(scan_data),
def import_scan():
"""
Upload and import Nessus Scan file
"""
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
from skaldship.general import check_datadir
import time
import os
filedir = os.path.join(request.folder, 'data', 'scanfiles')
response.title = "%s :: Import Nessus Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
nessus_config = nessus_get_config()
# {'ignored_vulnids': [19506, 11219, 34277],
# 'servers': {'server_1': {'password': '******',
# 'url': 'https://localhost:8834/',
# 'user': '******'}}}
nessusreports = [[0, None]]
import NessusAPI
from skaldship.nessus.ness6api import Nessus6API
servers = nessus_config.get('servers', {})
for k, v in servers.items():
if v.get('version') == 6:
# nessus >= 6
try:
nessus = Nessus6API(url=v.get('url'),
username=v.get('username'),
password=v.get('password'),
access_key=v.get('access_key'),
secret_key=v.get('secret_key'),
verify=v.get('verify_ssl'),
proxies=v.get('proxies'))
except Exception as e:
logger.error("Error communicating with %s: %s" % (k, str(e)))
try:
for report in nessus.get_scans():
ts = time.ctime(
float(report.get('last_modification_date', 0)))
nessusreports.append([
"%s:%s" % (k, report.get('id')),
"%s: %s - %s (%s)" %
(k, report.get('name'), ts, report.get('status'))
])
except Exception as e:
logger.error("Error making scan list: %s" % (str(e)))
else:
# nessus <= 5
try:
# check to see if NessusAPI is working
nessus = NessusAPI.NessusConnection(v.get('user'),
v.get('password'),
url=v.get('url'))
reports = nessus.list_reports()
for report in reports:
ts = time.ctime(float(report.timestamp))
nessusreports.append([
"%s:%s" % (k, report.name),
"%s: %s - %s (%s)" %
(k, report.readablename, ts, report.status)
])
except Exception as e:
logger.error("Error communicating with %s: %s" % (k, str(e)))
if len(nessusreports) > 1:
fields.append(
Field('f_nessus_report',
type='integer',
label=T('Nessus Report'),
requires=IS_IN_SET(nessusreports, zero=None)))
fields.append(
Field('f_filename',
'upload',
uploadfolder=filedir,
label=T('Nessus Scan File')))
fields.append(
Field('f_engineer',
type='integer',
label=T('Engineer'),
default=auth.user.id,
requires=IS_IN_SET(userlist)))
fields.append(
Field('f_asset_group',
type='string',
label=T('Asset Group'),
requires=IS_NOT_EMPTY()))
# check to see if we have a Metasploit Pro instance configured and talking
# if so pull a list of the workspaces and present them
if working_msf_api:
msf_workspaces = ["None"]
for w in list(msf_api.pro_workspaces().keys()):
msf_workspaces.append(w)
fields.append(
Field('f_msf_workspace',
type='string',
label=T('MSF Pro Workspace'),
requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None))))
fields.append(
Field('f_include_list', type='text', label=T('Hosts to Only Include')))
fields.append(
Field('f_ignore_list', type='text', label=T('Hosts to Ignore')))
fields.append(
Field('f_update_hosts',
type='boolean',
label=T('Update Host Information'),
default=False))
fields.append(
Field('f_taskit',
type='boolean',
default=auth.user.f_scheduler_tasks,
label=T('Run in background task')))
form = SQLFORM.factory(*fields, table_name='nessus_scan')
# form processing
if form.errors:
response.flash = 'Error in form'
elif form.accepts(request.vars, session):
# if no Nessus servers configured or valid, set report_name to 0
if nessusreports == [[0, None]]:
report_name = '0'
else:
if form.vars.f_nessus_report != "0":
# If a nessus report is selected, try to parse it to set report_name
try:
(server,
report_name) = form.vars.f_nessus_report.split(':')
except ValueError as e:
msg = "Invalid report name sent: %s" % (
form.vars.f_nessus_report)
response.flash = msg
logging.error(msg)
return dict(form=form)
else:
# set report_name to 0 if no f_nessus_report sent
report_name = '0'
if report_name != '0':
# download a report from a Nessus server
n_server = nessus_config.get('servers').get(server)
filename = os.path.join(
filedir, "nessus-%s-%s.xml" %
(form.vars.f_asset_group, int(time.time())))
check_datadir(request.folder)
if n_server.get('version') == 6:
# nessus version => 6
try:
nessus = Nessus6API(url=n_server.get('url'),
username=n_server.get('username'),
password=n_server.get('password'),
access_key=n_server.get('access_key'),
secret_key=n_server.get('secret_key'),
verify=n_server.get('verify_ssl'),
proxies=n_server.get('proxies'))
nessus_report = nessus.report_download(report_name)
fout = open(filename, "w")
fout.write(nessus_report)
fout.close()
except Exception as e:
msg = ("Error download Nessus report: %s" % (e))
logger.error(msg)
response.flash = msg
return dict(form=form)
else:
# nessus version <= 5
try:
# build a new nessus connection with the configured server details and download the report
n_server = nessus_config.get('servers').get(server)
nessus = NessusAPI.NessusConnection(
n_server.get('user'),
n_server.get('password'),
url=n_server.get('url'))
fout = open(filename, "w")
nessus.download_report(report_name, fout)
fout.close()
except Exception as e:
msg = ("Error download Nessus report: %s" % (e))
logger.error(msg)
response.flash = msg
return dict(form=form)
else:
filename = os.path.join(filedir, form.vars.f_filename)
# build the hosts only/exclude list
ip_exclude = []
data = form.vars.get('f_ignore_list')
if data:
ip_exclude = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_include = []
data = form.vars.get('f_include_list')
if data:
ip_include = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if form.vars.f_msf_workspace:
msf_workspace = form.vars.f_msf_workspace
if msf_workspace == "None":
msf_workspace = None
else:
msf_workspace = None
msf_settings = {
'workspace': msf_workspace,
'url': msf_settings['url'],
'key': msf_settings['key']
}
if form.vars.f_taskit:
task = scheduler.queue_task(
scanner_import,
pvars=dict(
scanner='nessus',
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
),
group_name=settings.scheduler_group_name,
sync_output=5,
timeout=settings.scheduler_timeout)
if task.id:
redirect(URL('tasks', 'status', args=task.id))
else:
response.flash = "Error submitting job: %s" % (task.errors)
else:
from skaldship.nessus.processor import process_scanfile
logger.info("Starting Nessus Report Import")
response.flash = process_scanfile(
filename=filename,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
)
redirect(URL('default', 'index'))
return dict(form=form)
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_workspace=False,
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Qualys XML Scan file
import os, time, re, html.parser
from io import StringIO
from MetasploitProAPI import MetasploitProAPI
from skaldship.hosts import html_to_markmin, get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
parser = html.parser.HTMLParser()
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS name: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile(
'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
RE_IPV4 = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$')
if msf_workspace:
msf = MetasploitProAPI(host=user_id.f_msf_pro_url,
apikey=user_id.f_msf_pro_key)
if msf.login():
logger.info(" [-] Authenticated to Metasploit PRO")
else:
logger.error(
" [!] Unable to login to Metasploit PRO, check your API key")
msf = None
else:
logger.warn(" [-] No Metasploit workspace provided!")
msf = None
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
print((" [*] Processing Qualys scan file %s" % (filename)))
try:
nmap_xml = etree.parse(filename)
except etree.ParseError as e:
print((" [!] Invalid XML file (%s): %s " % (filename, e)))
return
root = nmap_xml.getroot()
db = current.globalenv['db']
cache = current.globalenv['cache']
existing_vulnids = db(db.t_vulndata()).select(
db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
#print(" [*] Found %d vulnerabilities in the database already." % (len(existing_vulnids)))
# check for any CPE OS data
if db(db.t_cpe_os).count() > 0:
have_cpe = True
else:
have_cpe = False
user_id = db.auth_user(engineer) or auth.user.id
# parse the hosts, where all the goodies are
nodes = root.findall('IP')
print((" [-] Parsing %d hosts" % (len(nodes))))
hoststats = {}
hoststats['added'] = 0
hoststats['skipped'] = 0
hoststats['updated'] = 0
hoststats['errored'] = 0
hosts = [] # array of host_id fields
vulns_added = 0
vulns_skipped = 0
for node in nodes:
nodefields = {}
ipaddr = node.get('value')
nodefields['f_ipaddr'] = ipaddr
nodefields['f_hostname'] = node.get('hostname')
nodefields['f_netbios_name'] = node.findtext('NETBIOS_HOSTNAME')
# nodefields['f_macaddr'] = address.get('addr')
"""
status = node.find('status').get('state')
print(" [-] Host %s status is: %s" % (ipaddr, status))
if status != "up":
hoststats['skipped'] += 1
continue
"""
if ipaddr in ip_exclude:
print(" [-] Host is in exclude list... skipping")
hoststats['skipped'] += 1
continue
if len(ip_only) > 0 and ipaddr not in ip_only:
print(" [-] Host is not in the only list... skipping")
hoststats['skipped'] += 1
continue
ports = node.findall('INFOS')
if len(ports) < 1 and not addnoports:
print(
" [-] No ports open and not asked to add those kind... skipping"
)
hoststats['skipped'] += 1
continue
nodefields['f_engineer'] = user_id
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
# check to see if IPv4/IPv6 exists in DB already
if 'f_ipaddr' in nodefields:
host_rec = db(db.t_hosts.f_ipaddr ==
nodefields['f_ipaddr']).select().first()
else:
logging.warn("No IP Address found in record. Skipping")
continue
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
print((" [-] Adding %s" % (ipaddr)))
elif host_rec is not None and update_hosts:
db.commit()
host_id = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(ipaddr)
host_id = host_id.id
hoststats['updated'] += 1
print((" [-] Updating %s" % (ipaddr)))
else:
hoststats['skipped'] += 1
db.commit()
print((" [-] Skipped %s" % (ipaddr)))
continue
hosts.append(host_id)
# :
for hostscripts in node.findall('hostscript/script'):
svc_id = db.t_services.update_or_insert(f_proto='info',
f_number=0,
f_status='open',
f_hosts_id=host_id)
db.commit()
for script in hostscripts:
script_id = script.get('id')
output = script.get('output')
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=script_id, f_text=output)
db.commit()
# add ports and resulting vulndata
for port in node.findall("ports/port"):
f_proto = port.get('protocol')
f_number = port.get('portid')
f_status = port.find('state').get('state')
port_svc = port.find('service')
if port_svc:
f_name = port_svc.get('name')
f_product = port_svc.get('product')
svc_fp = port_svc.get('servicefp')
else:
f_name = None
f_product = None
svc_fp = None
print(
(" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name)))
svc_id = db.t_services.update_or_insert(f_proto=f_proto,
f_number=f_number,
f_status=f_status,
f_hosts_id=host_id,
f_name=f_name)
if f_product:
version = port.find('service').get('version', None)
if version:
f_product += " (%s)" % (version)
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=f_name, f_text=f_product)
db.commit()
if svc_fp:
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id, f_name=svc_fp, f_text=svc_fp)
db.commit()
# Process <script> service entries
for script in port.findall('service/script'):
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=script.get('id'),
f_text=script.get('output'))
db.commit()
# Process <cpe> service entries
for port_cpe in port.findall('service/cpe'):
cpe_id = port_cpe.text.replace('cpe:/', '')
if cpe_id[0] == "a":
# process CPE Applications
print((" [-] Found Application CPE data: %s" % (cpe_id)))
svc_info = db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name='CPE ID',
f_text="cpe:/%s" % (cpe_id))
db.commit()
elif cpe_id[0] == "o":
# process CPE Operating System
os_id = lookup_cpe(cpe_id[2:])
if os_id is not None:
db.t_host_os_refs.insert(f_certainty='0.9',
f_family='Unknown',
f_class='Other',
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
# So no CPE or existing OS data, lets split up the CPE data and make our own
print(" [!] No os_id found, this is odd !!!")
for config in port.findall("configuration/config"):
cfg_id = db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=config.attrib['name'],
f_text=config.text)
db.commit()
if re.match('\w+.banner$', config.attrib['name']):
db.t_services[svc_id] = dict(f_banner=config.text)
db.commit()
if config.attrib['name'] == 'mac-address':
# update the mac address of the host
db.t_hosts[host_id] = dict(f_macaddr=config.text)
db.commit()
if "advertised-name" in config.attrib['name']:
# netbios computer name
d = config.text.split(" ")[0]
if "Computer Name" in config.text:
data = {}
data['f_netbios_name'] = d
# if hostname isn't defined then lowercase netbios name and put it in
if db.t_hosts[host_id].f_hostname is None:
data['f_hostname'] = d.lower()
db(db.t_hosts.id == host_id).update(**data)
elif "Domain Name" in config.text:
db(db.t_netbios.f_hosts_id == host_id).update(
f_domain=d) or db.t_netbios.insert(
f_hosts_id=host_id, f_domain=d)
db.commit()
for script in port.findall("script"):
# process <script> results. This data contains both info
# and vulnerability data. For now we'll take a list of
# known nmap vuln checks from private/nmap_vulns.csv and
# use that to separate between service_info and vulndata.
pass
if msf is not None:
# send the downloaded nexpose file to MSF for importing
try:
res = msf.pro_import_file(
msf_workspace,
filename,
{
'DS_REMOVE_FILE': False,
'tag': asset_group,
},
)
print((" [*] Added file to MSF Pro: %s" % (res)))
except MSFAPIError as e:
logging.error("MSFAPI Error: %s" % (e))
pass
# any new nexpose vulns need to be checked against exploits table and connected
print(" [*] Connecting exploits to vulns and performing do_host_status")
do_host_status(asset_group=asset_group)
print((
" [*] Import complete: hosts: %s added, %s skipped, %s errors - vulns: %s added, %s skipped"
% (hoststats['added'], hoststats['skipped'], hoststats['errored'],
vulns_added, vulns_skipped)))
def import_xml_scan():
"""
Upload/import Nmap XML Scan file via scheduler task
"""
import time
from skaldship.general import check_datadir
from skaldship.metasploit import msf_get_config
msf_settings = msf_get_config(session)
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings['url'],
apikey=msf_settings['key'])
working_msf_api = msf_api.login()
except:
working_msf_api = False
filedir = os.path.join(request.folder, 'data', 'scanfiles')
check_datadir(request.folder)
response.title = "%s :: Import Nmap XML Scan Results" % (settings.title)
fields = []
# buld the dropdown user list
users = db(db.auth_user).select()
userlist = []
for user in users:
userlist.append([user.id, user.username])
fields.append(
Field('f_filename',
'upload',
uploadfolder=filedir,
label=T('Nmap XML File')))
fields.append(
Field('f_engineer',
type='integer',
label=T('Engineer'),
default=auth.user.id,
requires=IS_IN_SET(userlist)))
fields.append(
Field('f_asset_group',
type='string',
label=T('Asset Group'),
requires=IS_NOT_EMPTY()))
# If Metasploit available, pull a list of the workspaces and present them
if working_msf_api:
msf_workspaces = []
msf_workspaces.append("None")
for w in list(msf_api.pro_workspaces().keys()):
msf_workspaces.append(w)
fields.append(
Field('f_msf_workspace',
type='string',
label=T('MSF Pro Workspace'),
requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None))))
fields.append(
Field('f_addnoports',
type='boolean',
label=T('Add Hosts w/o Ports'),
default=False))
fields.append(
Field('f_include_list', type='text', label=T('Hosts to Only Include')))
fields.append(
Field('f_ignore_list', type='text', label=T('Hosts to Ignore')))
fields.append(
Field('f_update_hosts',
type='boolean',
label=T('Update Host Information'),
default=False))
fields.append(
Field('f_taskit',
type='boolean',
default=auth.user.f_scheduler_tasks,
label=T('Run in background task')))
form = SQLFORM.factory(*fields, table_name='nmap_xml')
if form.errors:
response.flash = 'Error in form'
elif form.accepts(request.vars, session):
# process a nmap file
filename = os.path.join(filedir, form.vars.f_filename)
# build the hosts only/exclude list
ip_exclude = []
data = form.vars.get('f_ignore_list')
if data:
ip_exclude = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_include = []
data = form.vars.get('f_include_list')
if data:
ip_include = data.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
if form.vars.f_msf_workspace:
msf_workspace = form.vars.f_msf_workspace
if msf_workspace == "None":
msf_workspace = None
else:
msf_workspace = None
msf_settings = {
'workspace': msf_workspace,
'url': msf_settings['url'],
'key': msf_settings['key']
}
if form.vars.f_taskit:
task = scheduler.queue_task(
scanner_import,
pvars=dict(
scanner='nmap',
filename=filename,
addnoports=form.vars.f_addnoports,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
),
group_name=settings.scheduler_group_name,
sync_output=5,
timeout=settings.scheduler_timeout)
if task.id:
redirect(URL('tasks', 'status', args=task.id))
else:
response.flash = "Error submitting job: %s" % (task.errors)
else:
from skaldship.nmap import process_xml
print("Starting Nmap XML Import")
process_xml(
filename=filename,
addnoports=form.vars.f_addnoports,
asset_group=form.vars.f_asset_group,
engineer=form.vars.f_engineer,
msf_settings=msf_settings,
ip_ignore_list=ip_exclude,
ip_include_list=ip_include,
update_hosts=form.vars.f_update_hosts,
)
response.flash = "Nmap XML upload complete"
redirect(URL('default', 'index'))
return dict(form=form)
def process_xml(
filename=None,
addnoports=False,
asset_group=None,
engineer=None,
msf_workspace=False,
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Qualys XML Scan file
import os, time, re, HTMLParser
from StringIO import StringIO
from MetasploitProAPI import MetasploitProAPI
from skaldship.hosts import html_to_markmin, get_host_record, do_host_status
from skaldship.cpe import lookup_cpe
parser = HTMLParser.HTMLParser()
# output regexes
RE_NETBIOS_NAME = re.compile('NetBIOS name: (?P<d>.*),')
RE_NETBIOS_MAC = re.compile('NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))')
RE_IPV4 = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$')
if msf_workspace:
msf = MetasploitProAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key)
if msf.login():
logger.info(" [-] Authenticated to Metasploit PRO")
else:
logger.error(" [!] Unable to login to Metasploit PRO, check your API key")
msf = None
else:
logger.warn(" [-] No Metasploit workspace provided!")
msf = None
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
print(" [*] Processing Qualys scan file %s" % (filename))
try:
nmap_xml = etree.parse(filename)
except etree.ParseError, e:
print(" [!] Invalid XML file (%s): %s " % (filename, e))
return
