def import_report(): """ Import a MSF Pro XML Report. TODO: FINISH HIM! """ msf_settings = msf_get_config(session) if msf_settings['workspace'] is None: redirect(URL('api_settings')) msf = MetasploitAPI(host=msf_settings['url'], apikey=msf_settings['key']) if not msf.login(): response.flash = "Error logging into Metasploit, check your settings" redirect(URL('api_settings')) form = SQLFORM.factory( Field('whitelist', 'text', label=T('Whitelist hosts/nets')), Field('blacklist', 'text', label=T('Blacklist hosts/nets')), ) if form.accepts(request, sesssion): # build the configuration hash rpt_data = {} rpt_data['DS_REPORT_TYPE'] = 'XML' rpt_data['DS_WHITELIST_HOSTS'] = form.vars.whitelist rpt_data['DS_BLACKLIST_HOSTS'] = form.vars.blacklist rpt_data['Workdspace'] = msf_settings['workspace'] # send the report request and get the task id rpt_taskid = msf.pro_start_report(rpt_data)
def import_report(): """ Import a MSF Pro XML Report. TODO: FINISH HIM! """ workspace = session.msf_workspace if workspace is None: redirect(URL('api_settings')) msf = MetasploitAPI(host=auth.user.f_msf_pro_url, apikey=auth.user.f_msf_pro_key) if not msf.login(): response.flash = "Error logging into Metasploit, check your settings" redirect(URL('api_settings')) form = SQLFORM.factory( Field('whitelist', 'text', label=T('Whitelist hosts/nets')), Field('blacklist', 'text', label=T('Blacklist hosts/nets')), ) if form.accepts(request, sesssion): # build the configuration hash rpt_data = {} rpt_data['DS_REPORT_TYPE'] = 'XML' rpt_data['DS_WHITELIST_HOSTS'] = form.vars.whitelist rpt_data['DS_BLACKLIST_HOSTS'] = form.vars.blacklist rpt_data['Workdspace'] = session.msf_workspace # send the report request and get the task id rpt_taskid = msf.pro_start_report(rpt_data)
def process_xml( filename=None, asset_group=None, engineer=None, msf_workspace=False, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process Nexpose XML Scan file from skaldship.cpe import lookup_cpe from skaldship.general import get_host_record db = current.globalenv['db'] cache = current.globalenv['cache'] parser = HTMLParser.HTMLParser() localdb = current.globalenv['db'] user_id = localdb.auth_user(engineer) if msf_workspace: msf = MetasploitAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key) if msf.login(): print(" [-] Authenticated to Metasploit PRO") #sys.stderr.write(" [-] Authenticated to Metasploit PRO\n") else: logger.error(" [!] Unable to login to Metasploit PRO, check your API key") #sys.stderr.write(" [!] Unable to login to Metasploit PRO, check your API key\n") msf = None else: print(" [-] No Metasploit workspace provided!") #sys.stderr.write( " [-] No Metasploit workspace provided!\n") msf = None # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals print(" [*] Processing Nexpose scan file %s" % (filename)) #sys.stderr.write(msg) try: nexpose_xml = etree.parse(filename) except etree.ParseError, e: msg = " [!] Invalid Nexpose XML file (%s): %s " % (filename, e) logger.error(msg) return msg
def import_scan(): """ Upload and import Nexpose Scan file """ msf_settings = msf_get_config(session) try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings["url"], apikey=msf_settings["key"]) working_msf_api = msf_api.login() except: working_msf_api = False from skaldship.general import check_datadir import time import os filedir = os.path.join(request.folder, "data", "scanfiles") response.title = "%s :: Import Nessus Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append([user.id, user.username]) nessus_config = nessus_get_config(session) # {'ignored_vulnids': [19506, 11219, 34277], # 'servers': {'server_1': {'password': '******', # 'url': 'https://localhost:8834/', # 'user': '******'}}} nessusreports = [[0, None]] import NessusAPI # if auth.user.f_nessus_host is not None: servers = nessus_config.get("servers", {}) for k, v in servers.iteritems(): try: # check to see if NessusAPI is working nessus = NessusAPI.NessusConnection(v.get("user"), v.get("password"), url=v.get("url")) reports = nessus.list_reports() for report in reports: ts = time.ctime(float(report.timestamp)) nessusreports.append( ["%s:%s" % (k, report.name), "%s: %s - %s (%s)" % (k, report.readablename, ts, report.status)] ) except Exception, e: logger.error("Error communicating with %s: %s" % (k, str(e)))
def import_xml_scan(): """ Upload/import Nexpose XML Scan file via scheduler task """ from NexposeAPI import NexposeAPI, Sites, Report from skaldship.general import check_datadir from skaldship.metasploit import msf_get_config import time import os msf_settings = msf_get_config(session) try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings['url'], apikey=msf_settings['key']) working_msf_api = msf_api.login() except: working_msf_api = False filedir = os.path.join(request.folder, 'data', 'scanfiles') response.title = "%s :: Import Nexpose XML Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append([user.id, user.username]) # check to see if nexpose is configured/active and get site listing nexpose_config = nexpose_get_config() nxsitelist = [] if nexpose_config['host'] is not None and nexpose_config[ 'user'] is not None: # see if the host is open/active first if nexpose_config['host'] is not None: sites = Sites() sites.host = nexpose_config['host'] sites.port = nexpose_config['port'] try: if sites.login(user_id=nexpose_config['user'], password=nexpose_config['password']): sites = sites.listings() nxsitelist.append([0, None]) for k, v in sites.iteritems(): nxsitelist.append([int(k), sites[k]['name']]) except Exception, e: pass
def import_scan(): """ Upload and import Nexpose Scan file """ msf_settings = msf_get_config(session) try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings['url'], apikey=msf_settings['key']) working_msf_api = msf_api.login() except: working_msf_api = False from skaldship.general import check_datadir import time import os filedir = os.path.join(request.folder, 'data', 'scanfiles') response.title = "%s :: Import Nessus Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append([user.id, user.username]) nessus_config = nessus_get_config(session) # {'ignored_vulnids': [19506, 11219, 34277], # 'servers': {'server_1': {'password': '******', # 'url': 'https://localhost:8834/', # 'user': '******'}}} nessusreports = [[0, None]] import NessusAPI #if auth.user.f_nessus_host is not None: servers = nessus_config.get('servers', {}) for k, v in servers.iteritems(): try: # check to see if NessusAPI is working nessus = NessusAPI.NessusConnection(v.get('user'), v.get('password'), url=v.get('url')) reports = nessus.list_reports() for report in reports: ts = time.ctime(float(report.timestamp)) nessusreports.append(["%s:%s" % (k, report.name), "%s: %s - %s (%s)" % (k, report.readablename, ts, report.status)]) except Exception, e: logger.error("Error communicating with %s (u:%s/p:%s): %s" % (k, v.get('user'), v.get('password'), str(e)))
def import_xml_scan(): """ Upload/import Nexpose XML Scan file via scheduler task """ from NexposeAPI import NexposeAPI, Sites, Report from skaldship.general import check_datadir from skaldship.metasploit import msf_get_config import time import os msf_settings = msf_get_config(session) try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings['url'], apikey=msf_settings['key']) working_msf_api = msf_api.login() except: working_msf_api = False filedir = os.path.join(request.folder, 'data', 'scanfiles') response.title = "%s :: Import Nexpose XML Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append( [ user.id, user.username ] ) # check to see if nexpose is configured/active and get site listing nexpose_config = nexpose_get_config() nxsitelist = [] if nexpose_config['host'] is not None and nexpose_config['user'] is not None: # see if the host is open/active first if nexpose_config['host'] is not None: sites = Sites() sites.host = nexpose_config['host'] sites.port = nexpose_config['port'] try: if sites.login(user_id=nexpose_config['user'], password=nexpose_config['password']): sites = sites.listings() nxsitelist.append( [ 0, None ] ) for k,v in sites.iteritems(): nxsitelist.append( [int(k), sites[k]['name']] ) except Exception, e: pass
def import_xml_scan(): """ Upload/import Nexpose XML Scan file via scheduler task """ try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=auth.user.f_msf_pro_url, apikey=auth.user.f_msf_pro_key) working_msf_api = msf_api.login() except: working_msf_api = False from skaldship.general import check_datadir import time import os filedir = os.path.join(request.folder, 'data', 'scanfiles') response.title = "%s :: Import Nessus Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append([user.id, user.username]) nessusreports = [[0, None]] if auth.user.f_nessus_host is not None: try: # check to see if NessusAPI is working import NessusAPI nessus = NessusAPI.NessusConnection(auth.user.f_nessus_user, auth.user.f_nessus_pw, url=auth.user.f_nessus_host) reports = nessus.list_reports() for report in reports: ts = time.ctime(float(report.timestamp)) nessusreports.append([report.name, "%s - %s (%s)" % (report.readablename, ts, report.status)]) except Exception, e: logger.error("Error communicating with Nessus: %s" % str(e)) if len(nessusreports) > 1: fields.append(Field('f_nessus_report', type='integer', label=T('Nessus Report'), requires=IS_IN_SET(nessusreports, zero=None)))
def process_xml( filename=None, addnoports=False, asset_group=None, engineer=None, msf_workspace=False, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process nMap XML Scan file import re from MetasploitAPI import MetasploitAPI from skaldship.general import get_host_record, do_host_status from skaldship.cpe import lookup_cpe from zenmapCore_Kvasir.NmapParser import NmapParser # output regexes RE_NETBIOS_NAME = re.compile('NetBIOS computer name: (?P<d>.*),') RE_NETBIOS_WORKGROUP = re.compile('Workgroup: (?P<d>.*),') RE_NETBIOS_MAC = re.compile('NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))') # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals log(" [*] Processing nMap scan file %s" % (filename)) nmap_parsed = NmapParser() nmap_parsed.parse_file(filename) #existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid') user_id = db.auth_user(engineer) or auth.user.id # parse the hosts, where all the goodies are log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts))) hoststats = {} hoststats['added'] = 0 hoststats['skipped'] = 0 hoststats['updated'] = 0 hoststats['errored'] = 0 hosts = [] # array of host_id fields svc_db = db.t_services for node in nmap_parsed.hosts: nodefields = {} if node.ipv6: ipaddr = node.ipv6 nodefields['f_ipv4'] = ipaddr elif node.ip.get('type') == 'ipv4': ipaddr = node.ip.get('addr') nodefields['f_ipv4'] = ipaddr else: log(" [!] No IPv4/IPv6 address, skipping") continue nodefields['f_macaddr'] = node.mac status = node.state log(" [-] Host %s status is: %s" % (ipaddr, status)) if status != "up": hoststats['skipped'] += 1 continue if ipaddr in ip_exclude: log(" [-] Host is in exclude list... skipping") hoststats['skipped'] += 1 continue if len(ip_only) > 0 and ipaddr not in ip_only: log(" [-] Host is not in the only list... skipping") hoststats['skipped'] += 1 continue if not node.ports and not addnoports: log(" [-] No ports open and not asked to add those kind... skipping") hoststats['skipped'] += 1 continue # we'lll just take the last hostname in the names list since it'll usually be the full dns name for name in node.hostnames: nodefields['f_hostname'] = name nodefields['f_engineer'] = user_id nodefields['f_asset_group'] = asset_group nodefields['f_confirmed'] = False # check to see if IPv4/IPv6 exists in DB already if 'f_ipv4' in nodefields: host_rec = db(db.t_hosts.f_ipv4 == nodefields['f_ipv4']).select().first() elif 'f_ipv6' in nodefields: host_rec = db(db.t_hosts.f_ipv6 == nodefields['f_ipv6']).select().first() else: log("No IP Address found in record. Skipping", logging.ERROR) continue if host_rec is None: host_id = db.t_hosts.insert(**nodefields) db.commit() hoststats['added'] += 1 log(" [-] Adding %s" % (ipaddr)) elif host_rec is not None and update_hosts: db.commit() if 'f_ipv4' in nodefields: host_id = db(db.t_hosts.f_ipv4 == nodefields['f_ipv4']).update(**nodefields) else: host_id = db(db.t_hosts.f_ipv6 == nodefields['f_ipv6']).update(**nodefields) db.commit() host_id = get_host_record(ipaddr) host_id = host_id.id hoststats['updated'] += 1 log(" [-] Updating %s" % (ipaddr)) else: hoststats['skipped'] += 1 db.commit() log(" [-] Skipped %s" % (ipaddr)) continue hosts.append(host_id) # process non-port <hostscript> entries. Add to info/0: for hostscripts in node.hostscripts: query = (svc_db.f_proto == 'info') & (svc_db.f_number == 0) & (svc_db.f_hosts_id == host_id) svc_id = db.t_services.update_or_insert(query, f_proto='info', f_number=0, f_status='open', f_hosts_id=host_id) if not svc_id: svc_rec = db(query).select(cache=(cache.ram, 180)).first() if svc_rec: svc_id = svc_rec.id else: log(" [!] Service record wasn't created", logging.ERROR) continue db.commit() for script in hostscripts: script_id = script.id output = script.output db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=script_id, f_text=output) db.commit() if script_id == 'nbstat': # pull out NetBIOS info from nbstat output result = RE_NETBIOS_MAC.search(output) if 'd' in result.groupdict(): host_rec.update(f_macaddr=result.group('d')) db.commit() result = RE_NETBIOS_NAME.search(output) if 'd' in result.groupdict(): host_rec.update(f_netbios_name=result.group('d')) db.commit() result = RE_NETBIOS_WORKGROUP.search(output) if 'd' in result.groupdict(): db(db.t_netbios.update_or_insert(f_hosts_id=host_id, f_domain=result.group('d'))) db.commit() # add ports and resulting vulndata for port in node.ports: f_proto = port.get('protocol') f_number = port.get('portid') f_status = port.get('port_state') f_name = port.get('service_name') f_product = port.get('service_product') log(" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name)) svc_id = db.t_services.update_or_insert(f_proto=f_proto, f_number=f_number, f_status=f_status, f_hosts_id=host_id, f_name=f_name) if f_product: version = port.get('service_version') if version: f_product += " (%s)" % (version) db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=f_name, f_text=f_product) db.commit() # Process <script> service entries for script in port.get('scripts'): db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=script.get('id'), f_text=script.get('output')) db.commit() # Process <cpe> service entries for port_cpe in port.get('service_cpe'): cpe_id = port_cpe.text.lstrip('cpe:/') if cpe_id[0] == "a": # process CPE Applications #log(" [-] Found Application CPE data: %s" % (cpe_id)) db.t_service_info.update_or_insert(f_services_id=svc_id, f_name='cpe.app', f_text="cpe:/%s" % (cpe_id)) db.commit() elif cpe_id[0] == "o": # process CPE Operating System os_id = lookup_cpe(cpe_id[2:]) if os_id is not None: db.t_host_os_refs.insert(f_certainty='0.9', f_family='Unknown', f_class='Other', f_hosts_id=host_id, f_os_id=os_id) db.commit() else: # So no CPE or existing OS data, lets split up the CPE data and make our own log(" [!] No os_id found, this is odd !!!") if msf_workspace: msf = MetasploitAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key) if msf.login(): try: res = msf.pro_import_file( msf_workspace, filename, { 'DS_REMOVE_FILE': False, 'tag': asset_group, }, ) log(" [*] Added file to MSF Pro: %s" % (res)) except MetasploitAPI.MSFAPIError, e: logging.error("MSFAPI Error: %s" % (e)) pass else: log(" [!] Unable to login to Metasploit PRO, check your API key", logging.ERROR) msf = None
(vuln_id, vulndata, extradata) = nessus_vulns.parse(rpt_item) if not vuln_id: # no vulnerability id continue _plugin_parse(host_id, vuln_id, vulndata, extradata) else: (vuln_id, vulndata, extradata) = nessus_vulns.parse(host) _plugin_parse(host_id, vuln_id, vulndata, extradata) if msf_settings.get('workspace'): try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings.get('url'), apikey=msf_settings.get('key')) working_msf_api = msf_api.login() except Exception, error: log(" [!] Unable to authenticate to MSF API: %s" % str(error), logging.ERROR) working_msf_api = False try: scan_data = open(filename, "r+").readlines() except Exception, error: log( " [!] Error loading scan data to send to Metasploit: %s" % str(error), logging.ERROR) scan_data = None if scan_data and working_msf_api: task = msf_api.pro_import_data(
def process_xml( filename=None, addnoports=False, asset_group=None, engineer=None, msf_settings={}, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process Nmap XML Scan file import re import os from skaldship.general import get_host_record, do_host_status from skaldship.cpe import lookup_cpe from zenmapCore_Kvasir.NmapParser import NmapParser # output regexes RE_NETBIOS_NAME = re.compile("NetBIOS computer name: (?P<d>.*),") RE_NETBIOS_WORKGROUP = re.compile("Workgroup: (?P<d>.*),") RE_NETBIOS_MAC = re.compile("NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))") # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split("\r\n") # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split("\r\n") # TODO: check for ip subnet/range and break it out to individuals log(" [*] Processing Nmap scan file %s" % (filename)) nmap_parsed = NmapParser() nmap_parsed.parse_file(filename) # existing_vulnids = db(db.t_vulndata()).select(db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid') # parse the hosts, where all the goodies are log(" [-] Parsing %d hosts" % (len(nmap_parsed.hosts))) hoststats = {} hoststats["added"] = 0 hoststats["skipped"] = 0 hoststats["updated"] = 0 hoststats["errored"] = 0 hosts = [] # array of host_id fields svc_db = db.t_services for node in nmap_parsed.hosts: nodefields = {} if node.ipv6: ipaddr = node.ipv6 nodefields["f_ipv4"] = ipaddr elif node.ip.get("type") == "ipv4": ipaddr = node.ip.get("addr") nodefields["f_ipv4"] = ipaddr else: log(" [!] No IPv4/IPv6 address, skipping") continue try: nodefields["f_macaddr"] = node.mac["addr"] except TypeError: nodefields["f_macaddr"] = None status = node.state log(" [-] Host %s status is: %s" % (ipaddr, status)) if status != "up": hoststats["skipped"] += 1 continue if ipaddr in ip_exclude: log(" [-] Host is in exclude list... skipping") hoststats["skipped"] += 1 continue if len(ip_only) > 0 and ipaddr not in ip_only: log(" [-] Host is not in the only list... skipping") hoststats["skipped"] += 1 continue if not node.ports and not addnoports: log(" [-] No ports open and not asked to add those kind... skipping") hoststats["skipped"] += 1 continue # we'lll just take the last hostname in the names list since it'll usually be the full dns name for name in node.hostnames: nodefields["f_hostname"] = name["hostname"] nodefields["f_engineer"] = engineer nodefields["f_asset_group"] = asset_group nodefields["f_confirmed"] = False # see if host exists, if so update. if not, insert! query = (db.t_hosts.f_ipv4 == ipaddr) | (db.t_hosts.f_ipv6 == ipaddr) host_rec = db(query).select().first() if host_rec is None: host_id = db.t_hosts.insert(**nodefields) db.commit() hoststats["added"] += 1 log(" [-] Adding %s" % (ipaddr)) elif host_rec is not None and update_hosts: db.commit() if "f_ipv4" in nodefields: host_id = db(db.t_hosts.f_ipv4 == nodefields["f_ipv4"]).update(**nodefields) else: host_id = db(db.t_hosts.f_ipv6 == nodefields["f_ipv6"]).update(**nodefields) db.commit() host_id = get_host_record(ipaddr) host_id = host_id.id hoststats["updated"] += 1 log(" [-] Updating %s" % (ipaddr)) else: hoststats["skipped"] += 1 db.commit() log(" [-] Skipped %s" % (ipaddr)) continue hosts.append(host_id) # process non-port <hostscript> entries. Add to info/0: for hostscripts in node.hostscripts: query = (svc_db.f_proto == "info") & (svc_db.f_number == 0) & (svc_db.f_hosts_id == host_id) svc_id = db.t_services.update_or_insert( query, f_proto="info", f_number=0, f_status="open", f_hosts_id=host_id ) if not svc_id: svc_rec = db(query).select(cache=(cache.ram, 180)).first() if svc_rec: svc_id = svc_rec.id else: log(" [!] Service record wasn't created", logging.ERROR) continue db.commit() for script in hostscripts: script_id = script.id output = script.output db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=script_id, f_text=output) db.commit() if script_id == "nbstat": # pull out NetBIOS info from nbstat output result = RE_NETBIOS_MAC.search(output) if "d" in result.groupdict(): host_rec.update(f_macaddr=result.group("d")) db.commit() result = RE_NETBIOS_NAME.search(output) if "d" in result.groupdict(): host_rec.update(f_netbios_name=result.group("d")) db.commit() result = RE_NETBIOS_WORKGROUP.search(output) if "d" in result.groupdict(): db(db.t_netbios.update_or_insert(f_hosts_id=host_id, f_domain=result.group("d"))) db.commit() # add ports and resulting vulndata for port in node.ports: f_proto = port.get("protocol") f_number = port.get("portid") f_status = port.get("port_state") f_name = port.get("service_name") f_product = port.get("service_product") log(" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name)) svc_id = db.t_services.update_or_insert( f_proto=f_proto, f_number=f_number, f_status=f_status, f_hosts_id=host_id, f_name=f_name ) if f_product: version = port.get("service_version") if version: f_product += " (%s)" % (version) db.t_service_info.update_or_insert(f_services_id=svc_id, f_name=f_name, f_text=f_product) db.commit() # Process <script> service entries for script in port.get("scripts"): db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=script.get("id"), f_text=script.get("output") ) db.commit() # Process <cpe> service entries port_cpe = port.get("service_cpe") if port_cpe: cpe_id = port_cpe.lstrip("cpe:/") if cpe_id.startswith("a"): # process CPE Applications # log(" [-] Found Application CPE data: %s" % (cpe_id)) db.t_service_info.update_or_insert( f_services_id=svc_id, f_name="cpe.app", f_text="cpe:/%s" % (cpe_id) ) db.commit() elif cpe_id.startswith("o"): # process CPE Operating System os_id = lookup_cpe(cpe_id[2:]) if os_id is not None: db.t_host_os_refs.insert( f_certainty="0.9", f_family="Unknown", f_class="Other", f_hosts_id=host_id, f_os_id=os_id ) db.commit() else: # So no CPE or existing OS data, lets split up the CPE data and make our own log(" [!] No os_id found, this is odd !!!") if msf_settings.get("workspace"): try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings.get("url"), apikey=msf_settings.get("key")) working_msf_api = msf_api.login() except Exception, error: log(" [!] Unable to authenticate to MSF API: %s" % str(error), logging.ERROR) working_msf_api = False try: scan_data = open(filename, "r+").readlines() except Exception, error: log(" [!] Error loading scan data to send to Metasploit: %s" % str(error), logging.ERROR) scan_data = None
def import_xml_scan(): """ Upload/import Nmap XML Scan file via scheduler task """ import time from skaldship.general import check_datadir try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=auth.user.f_msf_pro_url, apikey=auth.user.f_msf_pro_key) working_msf_api = msf_api.login() except: working_msf_api = False filedir = os.path.join(request.folder,'data','scanfiles') check_datadir(request.folder) response.title = "%s :: Import Nmap XML Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append( [ user.id, user.username ] ) fields.append(Field('f_filename', 'upload', uploadfolder=filedir, label=T('Nmap XML File'))) fields.append(Field('f_engineer', type='integer', label=T('Engineer'), default=auth.user.id, requires=IS_IN_SET(userlist))) fields.append(Field('f_asset_group', type='string', label=T('Asset Group'), requires=IS_NOT_EMPTY())) # If Metasploit available, pull a list of the workspaces and present them if working_msf_api: msf_workspaces = [] msf_workspaces.append( "None" ) for w in msf_api.pro_workspaces().keys(): msf_workspaces.append(w) fields.append(Field('f_msf_workspace', type='string', label=T('MSF Pro Workspace'), requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None)))) fields.append(Field('f_addnoports', type='boolean', label=T('Add Hosts w/o Ports'), default=False)) fields.append(Field('f_include_list', type='text', label=T('Hosts to Only Include'))) fields.append(Field('f_ignore_list', type='text', label=T('Hosts to Ignore'))) fields.append(Field('f_update_hosts', type='boolean', label=T('Update Host Information'), default=False)) fields.append(Field('f_taskit', type='boolean', default=auth.user.f_scheduler_tasks, label=T('Run in background task'))) form = SQLFORM.factory(*fields, table_name='nmap_xml') if form.errors: response.flash = 'Error in form' elif form.accepts(request.vars, session): # process a nmap file filename = form.vars.f_filename filename = os.path.join(filedir, form.vars.f_filename) # build the hosts only/exclude list ip_exclude = [] data = form.vars.get('f_ignore_list') if data: ip_exclude = data.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_include = [] data = form.vars.get('f_include_list') if data: ip_include = data.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals if form.vars.f_msf_workspace: msf_workspace = form.vars.f_msf_workspace if msf_workspace == "None": msf_workspace = None else: msf_workspace = None msf_settings = {'workspace': msf_workspace, 'url': auth.user.f_msf_pro_url, 'key': auth.user.f_msf_pro_key} if form.vars.f_taskit: task = scheduler.queue_task( scanner_import, pvars=dict( scanner='nmap', filename=filename, addnoports=form.vars.f_addnoports, asset_group=form.vars.f_asset_group, engineer=form.vars.f_engineer, msf_settings=msf_settings, ip_ignore_list=ip_exclude, ip_include_list=ip_include, update_hosts=form.vars.f_update_hosts, ), group_name=settings.scheduler_group_name, sync_output=5, timeout=3600 # 1 hour ) if task.id: redirect(URL('tasks', 'status', args=task.id)) else: response.flash = "Error submitting job: %s" % (task.errors) else: from skaldship.nmap import process_xml print("Starting Nmap XML Import") process_xml( filename=filename, addnoports=form.vars.f_addnoports, asset_group=form.vars.f_asset_group, engineer=form.vars.f_engineer, msf_settings=msf_settings, ip_ignore_list=ip_exclude, ip_include_list=ip_include, update_hosts=form.vars.f_update_hosts, ) response.flash = "Nmap XML upload complete" redirect(URL('default', 'index')) return dict(form=form)
if host_records: def host_to_ip(host_rec): if isinstance(host_rec, (int, str)): host_rec = get_host_record(host_rec) if not host_rec: return None return host_rec.get('f_ipv4') or host_rec.get('f_ipv6') target_ips = '\n'.join([host_to_ip(x) for x in host_records.split('|')]) else: target_ips = '' loot_list = [] # list of loot IDs and IPs alert = False msf = MetasploitAPI(host=msf_settings['url'], apikey=msf_settings['key']) try: msf.login() except MSFAPIError, error: return dict(alert=True, error=str(error), form=None) form = SQLFORM.factory( Field('targets', 'text', default=target_ips, label=T('Targets'), requires=IS_NOT_EMPTY(), comment=T('Targets to scan can be IP Addresses, ranged lists or subnets. One per line.'), ), Field('blacklist', 'text', label=T('Blacklisted hosts'), comment=T('Targets to blacklist can be IP Addresses, ranged lists or subnets. One per line.') ), Field('stop_on_success', 'boolean', default=True, label=T('Stop on success'), comment=T('Stop scanning a host after first user login success') ), Field('verbose', 'boolean', default=False, label=T('Verbose')), Field('include_known', 'boolean', default=True, label=T('Include known'),
def process_xml( filename=None, addnoports=False, asset_group=None, engineer=None, msf_workspace=False, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process Qualys XML Scan file import os, time, re, HTMLParser from StringIO import StringIO from MetasploitAPI import MetasploitAPI from skaldship.hosts import html_to_markmin, get_host_record, do_host_status from skaldship.cpe import lookup_cpe parser = HTMLParser.HTMLParser() # output regexes RE_NETBIOS_NAME = re.compile('NetBIOS name: (?P<d>.*),') RE_NETBIOS_MAC = re.compile( 'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))') RE_IPV4 = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$') if msf_workspace: msf = MetasploitAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key) if msf.login(): logger.info(" [-] Authenticated to Metasploit PRO") else: logger.error( " [!] Unable to login to Metasploit PRO, check your API key") msf = None else: logger.warn(" [-] No Metasploit workspace provided!") msf = None try: from lxml import etree except ImportError: try: import xml.etree.cElementTree as etree except ImportError: try: import xml.etree.ElementTree as etree except: raise Exception("Unable to find valid ElementTree module.") # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals print(" [*] Processing Qualys scan file %s" % (filename)) try: nmap_xml = etree.parse(filename) except etree.ParseError, e: print(" [!] Invalid XML file (%s): %s " % (filename, e)) return
def import_xml_scan(): """ Upload/import Nmap XML Scan file via scheduler task """ import time from skaldship.general import check_datadir from skaldship.metasploit import msf_get_config msf_settings = msf_get_config(session) try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings['url'], apikey=msf_settings['key']) working_msf_api = msf_api.login() except: working_msf_api = False filedir = os.path.join(request.folder, 'data', 'scanfiles') check_datadir(request.folder) response.title = "%s :: Import Nmap XML Scan Results" % (settings.title) fields = [] # buld the dropdown user list users = db(db.auth_user).select() userlist = [] for user in users: userlist.append([user.id, user.username]) fields.append( Field('f_filename', 'upload', uploadfolder=filedir, label=T('Nmap XML File'))) fields.append( Field('f_engineer', type='integer', label=T('Engineer'), default=auth.user.id, requires=IS_IN_SET(userlist))) fields.append( Field('f_asset_group', type='string', label=T('Asset Group'), requires=IS_NOT_EMPTY())) # If Metasploit available, pull a list of the workspaces and present them if working_msf_api: msf_workspaces = [] msf_workspaces.append("None") for w in msf_api.pro_workspaces().keys(): msf_workspaces.append(w) fields.append( Field('f_msf_workspace', type='string', label=T('MSF Pro Workspace'), requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None)))) fields.append( Field('f_addnoports', type='boolean', label=T('Add Hosts w/o Ports'), default=False)) fields.append( Field('f_include_list', type='text', label=T('Hosts to Only Include'))) fields.append( Field('f_ignore_list', type='text', label=T('Hosts to Ignore'))) fields.append( Field('f_update_hosts', type='boolean', label=T('Update Host Information'), default=False)) fields.append( Field('f_taskit', type='boolean', default=auth.user.f_scheduler_tasks, label=T('Run in background task'))) form = SQLFORM.factory(*fields, table_name='nmap_xml') if form.errors: response.flash = 'Error in form' elif form.accepts(request.vars, session): # process a nmap file filename = form.vars.f_filename filename = os.path.join(filedir, form.vars.f_filename) # build the hosts only/exclude list ip_exclude = [] data = form.vars.get('f_ignore_list') if data: ip_exclude = data.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_include = [] data = form.vars.get('f_include_list') if data: ip_include = data.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals if form.vars.f_msf_workspace: msf_workspace = form.vars.f_msf_workspace if msf_workspace == "None": msf_workspace = None else: msf_workspace = None msf_settings = { 'workspace': msf_workspace, 'url': msf_settings['url'], 'key': msf_settings['key'] } if form.vars.f_taskit: task = scheduler.queue_task( scanner_import, pvars=dict( scanner='nmap', filename=filename, addnoports=form.vars.f_addnoports, asset_group=form.vars.f_asset_group, engineer=form.vars.f_engineer, msf_settings=msf_settings, ip_ignore_list=ip_exclude, ip_include_list=ip_include, update_hosts=form.vars.f_update_hosts, ), group_name=settings.scheduler_group_name, sync_output=5, timeout=settings.scheduler_timeout) if task.id: redirect(URL('tasks', 'status', args=task.id)) else: response.flash = "Error submitting job: %s" % (task.errors) else: from skaldship.nmap import process_xml print("Starting Nmap XML Import") process_xml( filename=filename, addnoports=form.vars.f_addnoports, asset_group=form.vars.f_asset_group, engineer=form.vars.f_engineer, msf_settings=msf_settings, ip_ignore_list=ip_exclude, ip_include_list=ip_include, update_hosts=form.vars.f_update_hosts, ) response.flash = "Nmap XML upload complete" redirect(URL('default', 'index')) return dict(form=form)
def process_xml( filename=None, addnoports=False, asset_group=None, engineer=None, msf_workspace=False, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process Qualys XML Scan file import os, time, re, HTMLParser from StringIO import StringIO from MetasploitAPI import MetasploitAPI from skaldship.hosts import html_to_markmin, get_host_record, do_host_status from skaldship.cpe import lookup_cpe parser = HTMLParser.HTMLParser() # output regexes RE_NETBIOS_NAME = re.compile('NetBIOS name: (?P<d>.*),') RE_NETBIOS_MAC = re.compile('NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))') RE_IPV4 = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$') if msf_workspace: msf = MetasploitAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key) if msf.login(): logger.info(" [-] Authenticated to Metasploit PRO") else: logger.error(" [!] Unable to login to Metasploit PRO, check your API key") msf = None else: logger.warn(" [-] No Metasploit workspace provided!") msf = None try: from lxml import etree except ImportError: try: import xml.etree.cElementTree as etree except ImportError: try: import xml.etree.ElementTree as etree except: raise Exception("Unable to find valid ElementTree module.") # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals print(" [*] Processing Qualys scan file %s" % (filename)) try: nmap_xml = etree.parse(filename) except etree.ParseError, e: print(" [!] Invalid XML file (%s): %s " % (filename, e)) return
if host_records: def host_to_ip(host_rec): if isinstance(host_rec, (int, str)): host_rec = get_host_record(host_rec) if not host_rec: return None return host_rec.get('f_ipv4') or host_rec.get('f_ipv6') target_ips = '\n'.join([host_to_ip(x) for x in host_records.split('|')]) else: target_ips = '' loot_list = [] # list of loot IDs and IPs alert = False msf = MetasploitAPI(host=auth.user.f_msf_pro_url, apikey=auth.user.f_msf_pro_key) try: msf.login() except MSFAPIError, error: return dict(alert=True, error=str(error), form=None) form = SQLFORM.factory( Field('targets', 'text', default=target_ips, label=T('Targets'), requires=IS_NOT_EMPTY(), comment=T('Targets to scan can be IP Addresses, ranged lists or subnets. One per line.'), ), Field('blacklist', 'text', label=T('Blacklisted hosts'), comment=T('Targets to blacklist can be IP Addresses, ranged lists or subnets. One per line.') ), Field('stop_on_success', 'boolean', default=True, label=T('Stop on success'), comment=T('Stop scanning a host after first user login success') ), Field('verbose', 'boolean', default=False, label=T('Verbose')), Field('include_known', 'boolean', default=True, label=T('Include known'),
if os_id is not None: db.t_host_os_refs.insert( f_certainty="0.9", f_family="Unknown", f_class="Other", f_hosts_id=host_id, f_os_id=os_id ) db.commit() else: # So no CPE or existing OS data, lets split up the CPE data and make our own log(" [!] No os_id found, this is odd !!!") if msf_settings.get("workspace"): try: # check to see if we have a Metasploit RPC instance configured and talking from MetasploitAPI import MetasploitAPI msf_api = MetasploitAPI(host=msf_settings.get("url"), apikey=msf_settings.get("key")) working_msf_api = msf_api.login() except Exception, error: log(" [!] Unable to authenticate to MSF API: %s" % str(error), logging.ERROR) working_msf_api = False try: scan_data = open(filename, "r+").readlines() except Exception, error: log(" [!] Error loading scan data to send to Metasploit: %s" % str(error), logging.ERROR) scan_data = None if scan_data and working_msf_api: task = msf_api.pro_import_data( msf_settings.get("workspace"), "".join(scan_data), {
nxsitelist.append( [int(k), sites[k]['name']] ) except Exception, e: pass if nxsitelist: fields.append(Field('f_nexpose_site', type='integer', label=T('Nexpose Site'), requires=IS_IN_SET(nxsitelist, zero=None))) fields.append(Field('f_filename', 'upload', uploadfolder=filedir, label=T('Nexpose XML File'))) fields.append(Field('f_engineer', type='integer', label=T('Engineer'), default=auth.user.id, requires=IS_IN_SET(userlist))) fields.append(Field('f_asset_group', type='string', label=T('Asset Group'), requires=IS_NOT_EMPTY())) # check to see if we have a Metasploit Pro instance configured and talking # if so pull a list of the workspaces and present them msf = MetasploitAPI(host=auth.user.f_msf_pro_url, apikey=auth.user.f_msf_pro_key) try: res = msf.login() except: res = False if res: msf_workspaces = [] msf_workspaces.append( "None" ) for w in msf.pro_workspaces().keys(): msf_workspaces.append(w) fields.append(Field('f_msf_workspace', type='string', label=T('MSF Pro Workspace'), requires=IS_EMPTY_OR(IS_IN_SET(msf_workspaces, zero=None)))) fields.append(Field('f_include_list', type='text', label=T('Hosts to Only Include'))) fields.append(Field('f_ignore_list', type='text', label=T('Hosts to Ignore'))) fields.append(Field('f_update_hosts', type='boolean', label=T('Update Host Information'), default=False)) fields.append(Field('f_taskit', type='boolean', default=auth.user.f_scheduler_tasks, label=T('Run in background task'))) form = SQLFORM.factory(*fields, table_name='nexpose_xml')