def process_mass_password(pw_file=None, pw_type=None, message=None, proto=None, portnum=None, add_hosts=False, user_id=1): """ Process a medusa/hydra mass password run """ import fileinput db = current.globalenv['db'] cache = current.globalenv['cache'] added = 0 updated = 0 new_hosts = 0 ip_dict = {} if pw_file is not None: try: fIN = fileinput.input(files=pw_file) except IOError, e: log("Error opening %s: %s" % (pw_file, e), logging.ERROR) return "Error opening %s: %s" % (pw_file, e)
def process_medusa(line): """ Process a medusa line and return a dictionary :param line: A line from a medusa output file :returns dict: A dictionary based upon the content { 'ip' : ip address 'port': port info - can be port # or module name 'user': username, 'pass': password, 'hash': ntlm hash if smbnt hash used 'msg' : status message } >>> process_medusa("# Medusa v.2.0a_rc1 (2012-12-06 10:44:10)") {} >>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.23 User: spauser Password: password [SUCCESS]") {'ip': '10.89.172.23', 'error': False, 'user': '******', 'pass': '******', 'msg': '[SUCCESS]', 'type': 'cleartext', 'port': '[smbnt]'} >>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.24 User: spauser Password: AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0::: [SUCCESS]") {'ip': '10.89.172.23', 'port': 'smbnt', 'user': '******', 'password': '', 'hash': 'AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0', 'msg': '[SUCCESS]'} """ retval = {} try: data = line.split() except Exception, e: log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR) return retval
def process_password_file(pw_file=None, pw_data=None, file_type=None, source=None): """ Process a password file and return a dictionary fit for t_accounts :param pw_file: Filename to process :param pw_data: List of password lines instead of processing a file :param file_type: 'PWDUMP', 'MSCa$h Dump', 'UNIX Passwd', 'UNIX Shadow', 'Medusa', 'Hydra', 'Username:Password', 'Usernames', 'AccountDB', 'Metasploit Creds CSV' :param source: Source to add to f_source field """ import fileinput from skaldship.passwords.medusa import process_medusa from skaldship.passwords.hydra import process_hydra from skaldship.passwords.metasploit import process_msfcsv accounts = {} if pw_file is not None: try: pw_data = [] for line in fileinput.input(files=pw_file): pw_data.append(line) except IOError, e: log("Error opening %s: %s" % (pw_file, e), logging.ERROR) return accounts
def scan_template(): from lxml import etree from StringIO import StringIO response.title = "%s :: Nexpose Scan Templates" % (settings.title) formupload = SQLFORM.factory( Field('f_filename', 'upload', uploadfolder=os.path.join(request.folder, 'data', 'misc'), label=T('Import Nexpose Scan Template')), _formname='uploader') if formupload.accepts(request.vars, formname='uploader'): najax = NXAJAX(session.najaxsession) template_class = ScanTemplates() filename = os.path.join(request.folder,'data','misc',formupload.vars.f_filename) template_xml = etree.parse(filename, etree.XMLParser()) imported = template_class.importtemplate(etree.tostring(template_xml), najax) response.flash = imported templates = ScanTemplates.listscantemps(True, najax) parse_templates = DIV(TAG(templates).elements('templateid')) return dict(form="", form2=formupload, html=parse_templates) najax = NXAJAX(session.najaxsession) najax.host = auth.user.f_nexpose_host najax.port = auth.user.f_nexpose_port if najax.login(user_id=auth.user.f_nexpose_user, password=auth.user.f_nexpose_pw): log("Logged in to Nexpose API. Session cached.") session.najaxsession = najax.getsession() template_class = ScanTemplates() templates = template_class.listscantemps(True, najax) response.flash = "Loaded %s scan templates" % (templates.count('<templateid>')) parse_templates = DIV(TAG(templates).elements('templateid')) return dict(form="", form2=formupload, html=parse_templates) else: response.flash = "Unable to login to Nexpose" return dict(form=formlogin, form2="", html="")
def run_scan( blacklist=None, target_list=None, scan_options=None, ): ''' Executes nmap scan ''' from zenmapCore_Kvasir.NmapCommand import NmapCommand from zenmapCore_Kvasir.NmapOptions import NmapOptions from time import sleep if scan_options[0] is not 'nmap': if 'nmap' in settings: scan_options.insert(0, settings.nmap) else: scan_options.insert(0, 'nmap') if target_list: data = [] for ip in target_list: data.append(ip.strip(' \t\n\r')) target_list = data if blacklist: data = [] for ip in blacklist: data.append(ip.strip(' \t\n\r')) blacklist = [','.join(map(str, data))] blacklist.insert(0, "--exclude") ops = NmapOptions() try: ops.parse(scan_options + target_list + blacklist) except Exception as e: log("[!] %s" % e) cmd = NmapCommand(ops.render_string()) log(" [*] Starting Nmap Scan: %s" % (cmd.command)) cmd.run_scan() try: cmd.scan_state() except Exception as e: log("[!] %s" % e) full_output = "" while cmd.scan_state(): sleep(5) result = cmd.get_output() start = len(full_output) - len(result) output = result[start:] full_output = "%s%s" % (full_output, output) log(output) log(" [*] Nmap Scan Complete") filename = cmd.get_xml_output_filename() return filename
def get_or_create_record(argument, **defaults): """ Returns a t_host record based on the argument. If argument is an ipv4/ipv6 address it looks it up. If it's an integer it returns it. If none exist and argument is an ipv4/ipv6 address it creates a new record using the defaults provided. :param argument: ip address or db.t_hosts.id :param defaults: dictionary of db.t_hosts fields, validated before inserting :returns: Row with id >>> get_or_create_record('2.2.2.2') <Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}> >>> get_or_create_record('9.9.9.9', f_engineer=9999) None >>> get_or_create_record(1) <Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}> >>> get_or_create_record(9999) None """ if argument is None: return None from gluon.validators import IS_IPADDRESS db = current.globalenv['db'] auth = current.globalenv['auth'] record = get_host_record(argument) if not record: fields = {} for k in list(defaults.keys()): if k in db.t_hosts.fields: fields[k] = defaults[k] # set defaults for assetgroup/engineer if not set if 'f_asset_group' not in fields: fields['f_asset_group'] = 'undefined' if 'f_engineer' not in fields: fields['f_engineer'] = auth.user_id or 1 if IS_IPADDRESS()(argument)[1] == None: fields['f_ipaddr'] = argument else: # invalid ip address, clear the fields fields = None if fields: host_rec = db.t_hosts.validate_and_insert(**fields) if host_rec.errors: log("Error creating host record: %s" % host_rec.errors, logging.ERROR) else: db.commit() record = db.t_hosts(host_rec.get('id')) return record
def get_or_create_record(argument, **defaults): """ Returns a t_host record based on the argument. If argument is an ipv4/ipv6 address it looks it up. If it's an integer it returns it. If none exist and argument is an ipv4/ipv6 address it creates a new record using the defaults provided. :param argument: ip address or db.t_hosts.id :param defaults: dictionary of db.t_hosts fields, validated before inserting :returns: Row with id >>> get_or_create_record('2.2.2.2') <Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}> >>> get_or_create_record('9.9.9.9', f_engineer=9999) None >>> get_or_create_record(1) <Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}> >>> get_or_create_record(9999) None """ if argument is None: return None from gluon.validators import IS_IPADDRESS db = current.globalenv["db"] auth = current.globalenv["auth"] record = get_host_record(argument) if not record: fields = {} for k in defaults.keys(): if k in db.t_hosts.fields: fields[k] = defaults[k] # set defaults for assetgroup/engineer if not set if "f_asset_group" not in fields: fields["f_asset_group"] = "undefined" if "f_engineer" not in fields: fields["f_engineer"] = auth.user_id or 1 if IS_IPADDRESS()(argument)[1] == None: fields["f_ipaddr"] = argument else: # invalid ip address, clear the fields fields = None if fields: host_rec = db.t_hosts.validate_and_insert(**fields) if host_rec.errors: log("Error creating host record: %s" % host_rec.errors, logging.ERROR) else: db.commit() record = db.t_hosts(host_rec.get("id")) return record
def process_report( filename=None, host_list=[], query=None, ip_ignore_list=None, ip_include_list=None, engineer=1, asset_group="ShodanHQ Import", ): """ Processes a ShodanHQ XML Report adding records to the db """ settings = current.globalenv['settings'] #try: # from shodan import WebAPI # from shodan.api import WebAPIError # webapi = WebAPI(settings.shodanhq_apikey) #except ImportError: # webapi = None sd = ShodanData() sd.engineer = engineer sd.asset_group = asset_group # build the hosts only/exclude list if ip_ignore_list: sd.ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals if ip_include_list: sd.ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals hosts = [] if filename: log(" [*] Processing ShodanHQ report file: %s" % (filename)) try: from lxml import etree except ImportError: try: import xml.etree.cElementTree as etree except ImportError: try: import xml.etree.ElementTree as etree except: raise Exception("Unable to find valid ElementTree module.") try: xml = etree.parse(filename) except etree.ParseError, e: raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e)) return root = xml.getroot() hosts = root.findall("host")
def run_scan(blacklist=None, target_list=None, scan_options=None): """ Executes nmap scan """ from zenmapCore_Kvasir.NmapCommand import NmapCommand from zenmapCore_Kvasir.NmapOptions import NmapOptions from time import sleep if scan_options[0] is not "nmap": if "nmap" in settings: scan_options.insert(0, settings.nmap) else: scan_options.insert(0, "nmap") if target_list: data = [] for ip in target_list: data.append(ip.strip(" \t\n\r")) target_list = data if blacklist: data = [] for ip in blacklist: data.append(ip.strip(" \t\n\r")) blacklist = [",".join(map(str, data))] blacklist.insert(0, "--exclude") ops = NmapOptions() try: ops.parse(scan_options + target_list + blacklist) except Exception as e: log("[!] %s" % e) cmd = NmapCommand(ops.render_string()) log(" [*] Starting Nmap Scan: %s" % (cmd.command)) cmd.run_scan() try: cmd.scan_state() except Exception as e: log("[!] %s" % e) full_output = "" while cmd.scan_state(): sleep(5) result = cmd.get_output() start = len(full_output) - len(result) output = result[start:] full_output = "%s%s" % (full_output, output) log(output) log(" [*] Nmap Scan Complete") filename = cmd.get_xml_output_filename() return filename
def grab_screenshot(url=None, outfile=None, phantomjs="/usr/bin/phantomjs"): """ Capture a PNG image of a URL using phantomjs @args: url: Website URL to retrieve outfile: Output filename, will overwrite but not remove failures phantomjs: Full path to phantomjs binary @output: [True/False, png image data] """ import os db = current.globalenv['db'] if not outfile: raise Exception("No output filename provided") try: os.stat(phantomjs) except OSError: phantomjs = "/usr/local/bin/phantomjs" try: os.stat(phantomjs) except OSError: logging.error("Unable to locate phantomjs binary") return [False, None] # encode the url to make sure it passes cleanly to phantomjs url = urllib.quote(url, safe='/:') folder = current.globalenv['request'].folder from sys import platform if platform in ["linux", "linux2"]: timeout = ["/usr/bin/timeout", "-k", "2", "5"] elif platform in ["darwin", "freebsd"]: timeout = [os.path.join(folder, 'private/timeout3'), "-t" "5"] else: timeout = [] phantom = timeout + [ phantomjs, "--ignore-ssl-errors=true", "%s/modules/skaldship/valkyries/webimaging.js" % (folder), url, outfile ] log("calling: %s" % str(phantom), logging.DEBUG) call(phantom) try: f = file(outfile) imgdata = f.read() f.close() result = True except: result = False imgdata = None return [result, imgdata]
def process_report_xml( filename=None, ip_ignore_list=None, ip_include_list=None, engineer=1, asset_group="Metasploit Import", update_hosts=True, ): """ Processes a Metasploit XML Export for the following data and adds to the db: - Hosts and services - Credentials Generate the XML report by using db_export -t xml filename.xml or through WebUI TODO: Auto-exploits successful exploit attempts if matching CVE/VulnDB entry found """ from gluon.validators import IS_IPADDRESS from skaldship.passwords.utils import lookup_hash from skaldship.hosts import get_host_record, get_or_create_record from skaldship.services import Services services = Services() db = current.globalenv['db'] #cache = current.globalenv['cache'] try: from lxml import etree except ImportError: try: import xml.etree.cElementTree as etree except ImportError: try: import xml.etree.ElementTree as etree except: raise Exception("Unable to find valid ElementTree module.") # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals log(" [*] Processing Metasploit Pro report file: %s" % (filename)) try: xml = etree.parse(filename) except etree.ParseError, e: raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
def process_exploits(filename=None): """ Process Nexpose exploits.xml file into the database """ log("Processing Nexpose exploits file: %s ..." % filename) try: exploits = etree.parse(filename) except etree.ParseError, e: raise Exception("Error processing file: %s" % e)
def grab_screenshot(url=None, outfile=None, phantomjs="/usr/bin/phantomjs"): """ Capture a PNG image of a URL using phantomjs @args: url: Website URL to retrieve outfile: Output filename, will overwrite but not remove failures phantomjs: Full path to phantomjs binary @output: [True/False, png image data] """ import os db = current.globalenv['db'] if not outfile: raise Exception("No output filename provided") try: os.stat(phantomjs) except OSError: phantomjs = "/usr/local/bin/phantomjs" try: os.stat(phantomjs) except OSError: logging.error("Unable to locate phantomjs binary") return [False, None] # encode the url to make sure it passes cleanly to phantomjs url = urllib.quote(url, safe='/:') folder = current.globalenv['request'].folder from sys import platform if platform in ["linux", "linux2"]: timeout = ["/usr/bin/timeout", "-k", "2", "5"] elif platform in ["darwin", "freebsd"]: timeout = [os.path.join(folder, 'private/timeout3'), "-t" "5"] else: timeout = [] phantom = timeout + [phantomjs, "--ignore-ssl-errors=true", "%s/modules/skaldship/valkyries/webimaging.js" % (folder), url, outfile] log("calling: %s" % str(phantom), logging.DEBUG) call(phantom) try: f = file(outfile) imgdata = f.read() f.close() result = True except: result = False imgdata = None return [result, imgdata]
def process_hydra(line): """ Process a hydra line and return a dictionary: { 'ip' : ip address 'port': port info - can be port # or module name 'user': username, 'pass': password, 'hash': ntlm hash if smbnt hash used 'msg' : status message } """ # line: [22][ssh] host: 1.1.1.1 login: username password: pw1234 retval = {} try: data = line.split() except Exception as e: log("Error processing hydra line: %s -- %s" % (e, line), logging.ERROR) return retval if data[1] == "host:": # these fields are always there.. sometimes password is not retval['port'] = data[0][1:data[0].find("]")] retval['ip'] = data[2] retval['user'] = data[4] if "password:"******"[smb]" in data and "Error:" in data: if len(retval['pass']) == 68 and retval['pass'][65:68] == ":::": # we have an ntlm hash cap'n retval['hash'] = ":".join(retval['pass'].split(':')[:2]) retval['pass'] = lookup_hash(retval['hash']) retval['type'] = 'smb' else: retval['type'] = 'cleartext' retval['hash'] = None retval['error'] = False return retval
def add_or_update(hostfields, update=False): """ Add a host and return the record. If update is True and host already exists then the record is updated and returned """ if not isinstance(hostfields, dict()): log(" [!] Hostfields is not a dictionary", logging.ERROR) return None host_rec = db(db.t_hosts.f_ipaddr == hostfields.get('f_ipaddr')) if not host_rec: try: host_id = db.t_hosts.insert(**hostfields) db.commit() except Exception as e: log("Error adding host: %s" % strerror(e)) return None host_rec = db.t_hosts[host_id] log(" [*] Added host: %s" % host_title_maker(host_rec)) else: if update: host_rec.update(**hostfields) log(" [*] Updated host: %s" % host_title_maker(host_rec)) return host_rec
def vuln_time_convert(vtime=''): """Converts Nexpose timetsamp (YYYYMMDDTHHMMSSUUU) into python datetime""" if not vtime: tval = datetime(1970, 1, 1) else: if isinstance(vtime, str): if vtime[8] == "T": tstr = "%%Y%%m%%dT%%H%%M%%S%s" % vtime[15:] tval = time.strptime(vtime, tstr) else: log("Unknown datetime value: %s" % vtime, logging.ERROR) else: log("Invalid datetime value provided: %s" % vtime, logging.ERROR) tval = datetime(1970, 1, 1) return datetime.fromtimestamp(time.mktime(tval))
def process_xml( filename=None, asset_group=None, engineer=None, msf_settings={}, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): """ Process a Nessus XML Report file Args: filename: A local filename to process asset_group: Asset group to assign hosts to engineer: Engineer record number to assign hosts to msf_workspace: If set a Metasploit workspace to send the scanfile to via the API ip_ignore_list: List of IP addresses to ignore ip_include_list: List of IP addresses to ONLY import (skip all others) update_hosts: Boolean to update/append to hosts, otherwise hosts are skipped Returns: msg: A string status message """ from skaldship.cpe import lookup_cpe db = current.globalenv['db'] cache = current.globalenv['cache'] settings = current.globalenv['settings'] # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals log(" [*] Processing Nessus scan file %s" % filename) try: nessus_xml = etree.parse(filename) except etree.ParseError, e: msg = " [!] Invalid Nessus scan file (%s): %s " % (filename, e) log(msg, logging.ERROR) return msg
def process_cracked_file(pw_file=None, file_type=None, message=""): """ Process a file of cracked passwords and update the cleartext with the new results. """ import fileinput db = current.globalenv['db'] cache = current.globalenv['cache'] if pw_file is not None: try: fIN = fileinput.input(files=pw_file) except IOError, e: log("Error opening %s: %s" % (pw_file, e), logging.ERROR) return "Error opening %s: %s" % (pw_file, e)
def process_file(filename=None, asset_group=None, engineer=None): # Upload and process hping Scan file from skaldship.hosts import get_host_record, do_host_status, add_or_update from gluon.validators import IS_IPADDRESS log(" [*] Processing hping scan file %s" % filename) hoststats = 0 nodefields = {'f_engineer': engineer, 'f_asset_group': asset_group, 'f_confirmed': False} svc_db = db.t_services host_ip = None ICMP_type = '' answer_ip = '' with open(filename) as f: for line in f: if "IP: " in line: host_ip = line.split()[1] if IS_IPADDRESS()(host_ip)[1] == None: nodefields['f_ipaddr'] = host_ip db.t_hosts.update_or_insert(**nodefields) db.commit() hoststats += 1 else: log(" [!] ERROR: Not a valid IP Address (%s)" % host_ip, logging.ERROR) if "[*] " in line: ICMP_type = line.split()[1] if "ip=" in line: ip = line.split('=')[2] answer_ip = ip.split()[0] if "transmitted" in line: packets = line.split() if packets[0] == packets[3]: if answer_ip != host_ip: response = "No" else: response = "Yes" else: response = "No" get_id = get_host_record(host_ip) svc_db.update_or_insert( f_hosts_id=get_id.id, f_proto='ICMP', f_number='0', f_status=response, f_name=ICMP_type ) db.commit()
def process_loot_files(loot_list=[]): """ Processes locally stored (to web2py) MSF password loot files into the account database. Args: loot_list: an array of [filename, settings.password_file_types, port, host_id] Returns: An array of [filename, result text] """ from skaldship.passwords.utils import process_password_file, insert_or_update_acct #import os db = current.globalenv['db'] data = [] for loot in loot_list: if isinstance(loot, []): (filename, file_type, port) = loot else: log("Invalid loot sent: %s" % loot, logging.ERROR) continue try: (proto, number) = port.split('/') except AttributeError as e: log("Invalid port sent: %s", port, logging.ERROR) try: pw_data = open(filename, "rb").readlines().split('\n') except IOError as e: log("Error opening %s: %s" % (filename, e), logging.ERROR) accounts = process_password_file( pw_data=pw_data, file_type=file_type, source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == number) query &= (db.t_services.f_proto == proto) query &= (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto=proto, f_number=number, f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added loot accounts for host: %s" % host.f_ipaddr) data.append({loot['host']: resp_text})
def process_pwdump_loot(loot_list=[], msf=None): """ Takes an array of loot records in loot_list, downloads the pwdump file and adds the users. """ from skaldship.passwords.utils import process_password_file, insert_or_update_acct db = current.globalenv['db'] #cache = current.globalenv['cache'] data = [] for loot_id in loot_list: loot = msf.loot_download(loot_id) if loot['ltype'] not in ['host.windows.pwdump', 'windows.hashes']: log("Loot is not a pwdump, it is a %s" % loot['ltype'], logging.ERROR) continue else: # process the pwdump file pw_data = loot['data'].split('\n') accounts = process_password_file( pw_data=pw_data, file_type='PWDUMP', source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == '0') & ( db.t_services.f_proto == 'info') & (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto="info", f_number="0", f_status="info", f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added pwdump records for host: %s" % host.f_ipaddr) data.append({loot['host']: resp_text}) return data
def clean_html(htmldata): """Cleans up the HTML using lxml.html clean_html for now.""" try: from lxml.html.clean import clean_html except ImportError: log("You don't have lxml installed", logging.ERROR) return htmldata if htmldata is None: return htmldata newdata = clean_html(htmldata) newdata = newdata.replace('\n', ' ') newdata = newdata.replace('<div>', '') newdata = newdata.replace('</div>', '') #newdata = re.compile('\s*\n\s*').sub('\n', newdata) return newdata
def process_password_file(pw_file=None, pw_data=None, file_type=None, source=None): """ Process a password file and return a dictionary fit for t_accounts file_type values: ('PWDUMP', 'MSCa$h Dump', 'UNIX Passwd', 'UNIX Shadow', 'Medusa', 'Hydra', 'Username:Password', 'AccountDB') """ import fileinput accounts = {} if pw_file is not None: try: pw_data = [] for line in fileinput.input(files=pw_file): pw_data.append(line) except IOError, e: log("Error opening %s: %s" % (pw_file, e), logging.ERROR) return accounts
def _update_or_insert(self, **fields): """ Our own update_or_insert routine :param fields: Matching db.t_services fields (f_proto, f_number, etc) :returns: t_services record id """ if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']: log("No protocol, number or hosts_id sent", logging.ERROR) return None svc_id = self.svc_db.update_or_insert(**fields) if not svc_id: # update_or_insert will not return an id if a record is updated. record = self._get_record(**fields) if record: svc_id = record.id return svc_id
def process_hydra(line): """ Process a hydra line and return a dictionary: { 'ip' : ip address 'port': port info - can be port # or module name 'user': username, 'pass': password, 'hash': ntlm hash if smbnt hash used 'msg' : status message } """ # line: [22][ssh] host: 1.1.1.1 login: username password: pw1234 retval = {} try: data = line.split() except Exception, e: log("Error processing hydra line: %s -- %s" % (e, line), logging.ERROR) return retval
def clean_html(htmldata): """Cleans up the HTML using lxml.html clean_html for now.""" import re try: from lxml.html.clean import clean_html except ImportError: log("You don't have lxml installed", logging.ERROR) return htmldata if htmldata is None: return htmldata newdata = clean_html(htmldata) newdata = newdata.replace('\n', ' ') newdata = newdata.replace('<div>', '') newdata = newdata.replace('</div>', '') newdata = newdata.replace('\t', '') # tabs? not needed, no never newdata = re.sub(' +', ' ', newdata) return newdata
def process_cracked_file(pw_file=None, file_type=None, message=""): """ Process a file of cracked passwords and update the cleartext with the new results. :param pw_file: Filename to process :param file_type: String of a file type :param message: Message string to add to f_message field """ import fileinput db = current.globalenv['db'] if pw_file is not None: try: fIN = fileinput.input(files=pw_file) except IOError, e: log("Error opening %s: %s" % (pw_file, e), logging.ERROR) return "Error opening %s: %s" % (pw_file, e)
def process_medusa(line): """ Process a medusa line and return a dictionary: { 'ip' : ip address 'port': port info - can be port # or module name 'user': username, 'pass': password, 'hash': ntlm hash if smbnt hash used 'msg' : status message } """ retval = {} try: data = line.split() except Exception, e: log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR) return retval
def scan_template(): from lxml import etree from StringIO import StringIO response.title = "%s :: Nexpose Scan Templates" % (settings.title) formupload = SQLFORM.factory(Field( 'f_filename', 'upload', uploadfolder=os.path.join(request.folder, 'data', 'misc'), label=T('Import Nexpose Scan Template')), _formname='uploader') if formupload.accepts(request.vars, formname='uploader'): najax = NXAJAX(session.najaxsession) template_class = ScanTemplates() filename = os.path.join(request.folder, 'data', 'misc', formupload.vars.f_filename) template_xml = etree.parse(filename, etree.XMLParser()) imported = template_class.importtemplate(etree.tostring(template_xml), najax) response.flash = imported templates = ScanTemplates.listscantemps(True, najax) parse_templates = DIV(TAG(templates).elements('templateid')) return dict(form="", form2=formupload, html=parse_templates) nexpose_config = nexpose_get_config() najax = NXAJAX(session.najaxsession) najax.host = nexpose_config['host'] najax.port = nexpose_config['port'] if najax.login(user_id=nexpose_config['user'], password=nexpose_config['password']): log("Logged in to Nexpose API. Session cached.") session.najaxsession = najax.getsession() template_class = ScanTemplates() templates = template_class.listscantemps(True, najax) response.flash = "Loaded %s scan templates" % ( templates.count('<templateid>')) parse_templates = DIV(TAG(templates).elements('templateid')) return dict(form="", form2=formupload, html=parse_templates) else: response.flash = "Unable to login to Nexpose" return dict(form=formlogin, form2="", html="")
def run_scanner( scanner=None, asset_group=None, engineer=None, target_list=None, blacklist=None, scan_options=None, addnoports=False, update_hosts=False, **kwargs ): ''' Schedule handler to process nmap scan ''' from skaldship.log import log if not isinstance(scanner, str): return False scanner = scanner.upper() logger.info(" [*] Processing Nmap scan ") if scanner == 'NMAP': from skaldship.nmap import run_scan nmap_xml_file = run_scan( blacklist=blacklist, target_list=target_list, scan_options=scan_options, ) if nmap_xml_file: from skaldship.nmap import process_xml log("Processing nmap xml file: %s" % (nmap_xml_file)) process_xml( filename=nmap_xml_file, addnoports=addnoports, asset_group=asset_group, engineer=engineer, msf_settings={}, ip_ignore_list=None, ip_include_list=None, update_hosts=update_hosts, )
def scan_template(): from lxml import etree from StringIO import StringIO response.title = "%s :: Nexpose Scan Templates" % (settings.title) formupload = SQLFORM.factory( Field( "f_filename", "upload", uploadfolder=os.path.join(request.folder, "data", "misc"), label=T("Import Nexpose Scan Template"), ), _formname="uploader", ) if formupload.accepts(request.vars, formname="uploader"): najax = NXAJAX(session.najaxsession) template_class = ScanTemplates() filename = os.path.join(request.folder, "data", "misc", formupload.vars.f_filename) template_xml = etree.parse(filename, etree.XMLParser()) imported = template_class.importtemplate(etree.tostring(template_xml), najax) response.flash = imported templates = ScanTemplates.listscantemps(True, najax) parse_templates = DIV(TAG(templates).elements("templateid")) return dict(form="", form2=formupload, html=parse_templates) nexpose_config = nexpose_get_config() najax = NXAJAX(session.najaxsession) najax.host = nexpose_config["host"] najax.port = nexpose_config["port"] if najax.login(user_id=nexpose_config["user"], password=nexpose_config["password"]): log("Logged in to Nexpose API. Session cached.") session.najaxsession = najax.getsession() template_class = ScanTemplates() templates = template_class.listscantemps(True, najax) response.flash = "Loaded %s scan templates" % (templates.count("<templateid>")) parse_templates = DIV(TAG(templates).elements("templateid")) return dict(form="", form2=formupload, html=parse_templates) else: response.flash = "Unable to login to Nexpose" return dict(form=formlogin, form2="", html="")
def _get_record(self, **fields): """ Obtain a record from specified fields. Requires f_proto, f_number and f_hosts_id :param fields: Matching db.t_services fields (f_proto, f_number, etc) :returns: t_services record """ if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']: log("No protocol, number or hosts_id sent", logging.ERROR) return None query = (self.svc_db.f_proto == fields['f_proto']) &\ (self.svc_db.f_number == fields['f_number']) &\ (self.svc_db.f_hosts_id == fields['f_hosts_id']) record = self.db(query).select().first() if record and record.id not in self.services: self.services[record.id] = record return record
def process_xml( filename=None, asset_group=None, engineer=None, msf_settings={}, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process Nexpose XML Scan file from skaldship.cpe import lookup_cpe from skaldship.hosts import get_host_record from gluon.validators import IS_IPADDRESS import os db = current.globalenv['db'] session = current.globalenv['session'] parser = HTMLParser.HTMLParser() user_id = db.auth_user(engineer) # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals log(" [*] Processing Nexpose scan file %s" % filename) try: nexpose_xml = etree.parse(filename) except etree.ParseError, e: msg = " [!] Invalid Nexpose XML file (%s): %s " % (filename, e) log(msg, logging.ERROR) return msg
def process_pwdump_loot(loot_list=[], msf=None): """ Takes an array of loot records in loot_list, downloads the pwdump file and adds the users. """ from skaldship.passwords.utils import process_password_file, insert_or_update_acct db = current.globalenv['db'] #cache = current.globalenv['cache'] data = [] for loot_id in loot_list: loot = msf.loot_download(loot_id) if loot['ltype'] not in ['host.windows.pwdump', 'windows.hashes']: log("Loot is not a pwdump, it is a %s" % loot['ltype'], logging.ERROR) continue else: # process the pwdump file pw_data = loot['data'].split('\n') accounts = process_password_file( pw_data=pw_data, file_type='PWDUMP', source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == '0') & (db.t_services.f_proto == 'info') & (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto="info", f_number="0", f_status="info", f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added pwdump records for host: %s" % host.f_ipaddr) data.append({loot['host']: resp_text}) return data
def process_loot_files(loot_list=[]): """ Processes locally stored (to web2py) MSF password loot files into the account database. Args: loot_list: an array of [filename, settings.password_file_types, port, host_id] Returns: An array of [filename, result text] """ from skaldship.passwords.utils import process_password_file, insert_or_update_acct #import os db = current.globalenv['db'] data = [] for loot in loot_list: if isinstance(loot, []): (filename, file_type, port) = loot else: log("Invalid loot sent: %s" % loot, logging.ERROR) continue try: (proto, number) = port.split('/') except AttributeError, e: log("Invalid port sent: %s", port, logging.ERROR) try: pw_data = open(filename, "rb").readlines().split('\n') except IOError, e: log("Error opening %s: %s" % (filename, e), logging.ERROR)
def import_all_vulndata(overwrite=False, nexpose_server={}): """ Uses the NexposeAPI and imports each and every vulnerability to Kvasir. Can take a looooong time. Args: overwrite: Whether or not to overwrite an existing t_vulndata record Returns: msg: A string message of status. """ from NexposeAPI import VulnData db = current.globalenv['db'] vuln_class = VulnData() vuln_class.host = nexpose_server.get('host', 'localhost') vuln_class.port = nexpose_server.get('port', '3780') if vuln_class.login(user_id=nexpose_server.get('user'), password=nexpose_server.get('pw')): log(" [*] Populating list of Nexpose vulnerability ID summaries") try: vuln_class.populate_summary() except Exception, e: log(" [!] Error populating summaries: %s" % str(e), logging.ERROR) return False try: vulnxml = etree.parse(StringIO(vuln_class.vulnxml)) except Exception, e: log(" [!] Error parsing summary XML: %s" % str(e), logging.ERROR) return False
def process_exploits(filename=None): """ Process Nexpose exploits.xml file into the database """ log("Processing Nexpose exploits file: %s ..." % filename) try: exploits = etree.parse(filename) except etree.ParseError as e: raise Exception("Error processing file: %s" % e) except IOError as e: raise Exception("Error opening file: %s" % e) r = exploits.getroot() counter = 0 from .exploits import add_exploit, connect_exploits for exploit in r.findall('exploit'): #"adobe-unspec-bof-cve-2010-1297","13787","0day Exploit for Adobe Flash and Reader PoC (from the wild)","Description","1","Expert" f_name = exploit.findtext('name') f_title = exploit.findtext('id') f_description = str(exploit.findtext('description')).encode( 'iso-8859-1').decode('cp1252') f_description = f_description.replace("\\'", "'").replace('\\x', "0x") f_source = exploit.findtext('source') f_level = exploit.findtext( 'rank') or 'Unknown' # exploiter experience level estimate f_rank = exploit.findtext( 'exploitrank') or 'Unknown' # rank of the exploit # exploit records can have multiple Nexpose vulnerabilitiy identifiers f_vulnid = [] for nex_id in exploit.findall("vulnerabilities/vulnerability"): f_vulnid.append(nex_id.get('id').lower()) res = add_exploit( cve=None, vuln_ids=f_vulnid, f_name=f_name, f_title=f_title, f_description=f_description, f_source=f_source, f_level=f_level, f_rank=f_rank, ) if res > 0: counter += 1 else: log("Error importing exploit: %s" % f_name, logging.ERROR) connect_exploits() log("%d exploits added/updated" % counter) return True
def db_vuln_refs(self, vuln_id=None, vulndata={}, extradata={}): """ Add or update vulnerability references such as CPE, MSF Bulletins, OSVDB, Bugtraq, etc. Args: vuln_id: The db.t_vulndata reference id vulndadta: A dictionary of vulnerability data from t_vulndata extradata: A dictionary of extra vulndata Returns: None """ if not vulndata: log(" [!] No vulndata sent!", logging.ERROR) return if not extradata: log(" [!] No extradata sent!", logging.ERROR) return if not vuln_id: log(" [!] No vulnerability record id sent!", logging.ERROR) return ref_types = self.ref_types ref_types.extend(self.single_refs) # ugh this needs to be more pythonic. it's 1:30am and I'm tired for refname in ref_types: if refname in extradata: for reftext in extradata[refname]: if reftext: # add the vuln_ref ref_id = self.db.t_vuln_refs.update_or_insert( f_text=reftext, f_source=refname.upper(), ) if not ref_id: ref_id = self.db(self.db.t_vuln_refs.f_text == reftext).select( cache=(self.cache.ram, 180) ).first().id # link vuln_ref to vulndata self.db.t_vuln_references.update_or_insert( f_vulndata_id=vuln_id, f_vuln_ref_id=ref_id ) return
def db_vuln_refs(self, vuln_id=None, vulndata={}, extradata={}): """ Add or update vulnerability references such as CPE, MSF Bulletins, OSVDB, Bugtraq, etc. Args: vuln_id: The db.t_vulndata reference id vulndadta: A dictionary of vulnerability data from t_vulndata extradata: A dictionary of extra vulndata Returns: None """ if not vulndata: log(" [!] No vulndata sent!", logging.ERROR) return if not extradata: log(" [!] No extradata sent!", logging.ERROR) return if not vuln_id: log(" [!] No vulnerability record id sent!", logging.ERROR) return ref_types = self.ref_types ref_types.extend(self.single_refs) # ugh this needs to be more pythonic. it's 1:30am and I'm tired for refname in ref_types: if refname in extradata: for reftext in extradata[refname]: if reftext: # add the vuln_ref ref_id = self.db.t_vuln_refs.update_or_insert( f_text=reftext, f_source=refname.upper(), ) if not ref_id: ref_id = self.db( self.db.t_vuln_refs.f_text == reftext).select( cache=(self.cache.ram, 180)).first().id # link vuln_ref to vulndata self.db.t_vuln_references.update_or_insert( f_vulndata_id=vuln_id, f_vuln_ref_id=ref_id) return
def add_or_update(hostfields, update=False): """ Add a host and return the record. If update is True and host already exists then the record is updated and returned """ if not isinstance(hostfields, dict()): log(" [!] Hostfields is not a dictionary", logging.ERROR) return None host_rec = db(db.t_hosts.f_ipaddr == hostfields.get('f_ipaddr')) if not host_rec: try: host_id = db.t_hosts.insert(**hostfields) db.commit() except Exception, e: log("Error adding host: %s" % strerror(e)) return None host_rec = db.t_hosts[host_id] log(" [*] Added host: %s" % host_title_maker(host_rec))
def import_vulnid(): """ Downloads the detailed vulnerability data from Nexpose based on a vuln id passed to it """ form = SQLFORM.factory( Field('nexid', 'string', label=T('Nexpose ID')), Field('nexid_list', 'text', label=T('Nexpose ID List'))) response.title = "%s :: Import Nexpose VulnID" % settings.title nexpose_config = nexpose_get_config() if form.process().accepted: from NexposeAPI import VulnData from skaldship.nexpose import vuln_parse nxvulns = VulnData() nxvulns.host = nexpose_config['host'] nxvulns.port = nexpose_config['port'] nexpose_ids = [] if form.vars.nexid: nexpose_ids.extend([form.vars.nexid]) if form.vars.nexid_list: nexpose_ids.extend(form.vars.nexid_list.split('\r\n')) res = nxvulns.login(user_id=nexpose_config['user'], password=nexpose_config['password']) if res: stats = {'added': 0, 'invalid': 0} for nexid in nexpose_ids: vulndetails = nxvulns.detail(nexid) if vulndetails is not None: (vulnfields, references) = vuln_parse( vulndetails.find('Vulnerability'), fromapi=True) else: stats['invalid'] += 1 continue # add the vulnerability to t_vulndata query = (db.t_vulndata.f_vulnid == nexid) vulnid = db.t_vulndata.update_or_insert(query, **vulnfields) if not vulnid: row = db(query).select().first() if row: vulnid = row.id else: log(" [!] Could not find %s in database.." % nexid, logging.WARN) stats['invalid'] += 1 continue db.commit() # add the references if vulnid is not None and references: for reference in references: # check to see if reference exists first query = (db.t_vuln_refs.f_source == reference[0]) & ( db.t_vuln_refs.f_text == reference[1]) ref_id = db.t_vuln_refs.update_or_insert( query, f_source=reference[0], f_text=reference[1]) if not ref_id: ref_id = db(query).select().first().id # make many-to-many relationship with t_vuln_data db.t_vuln_references.update_or_insert( f_vuln_ref_id=ref_id, f_vulndata_id=vulnid) db.commit() from skaldship.exploits import connect_exploits connect_exploits() log(" [-] Added Nexpose vulnerability: %s" % nexid) stats['added'] += 1 response.flash = "%s added, %s skipped" % (stats['added'], stats['invalid']) return dict(form=form) else: response.flash = "Unable to login to Nexpose" elif form.errors: response.flash = "Error in form" return dict(form=form)
asset_group=form.vars.f_asset_group, engineer=form.vars.f_engineer, msf_settings=msf_settings, ip_ignore_list=ip_exclude, ip_include_list=ip_include, update_hosts=form.vars.f_update_hosts, ), group_name=settings.scheduler_group_name, sync_output=5, timeout=settings.scheduler_timeout) if task.id: redirect(URL('tasks', 'status', args=task.id)) else: response.flash = "Error submitting job: %s" % (task.errors) else: from skaldship.nexpose import process_xml log("Starting Nexpose XML Import") process_xml( filename=filename, asset_group=form.vars.f_asset_group, engineer=form.vars.f_engineer, msf_settings=msf_settings, ip_ignore_list=ip_exclude, ip_include_list=ip_include, update_hosts=form.vars.f_update_hosts, ) response.flash = "Nexpose XML upload complete" redirect(URL('default', 'index')) return dict(form=form)