def process_screenshot_loot(loot_list=[], msf=None): """ Takes an array of loot records in loot_list, downloads the screenshot and adds it to the database """ db = current.globalenv['db'] #cache = current.globalenv['cache'] loot_count = 0 for loot_id in loot_list: loot = msf.loot_download(loot_id) ip = loot_list[loot_id] if loot['ltype'] != 'host.windows.screenshot': logging.error(" [!] %s/%s is not a screenshot, it is a %s" % (ip, loot['name'], loot['ltype'])) else: record = get_host_record(ip) if not record: logging.error(" [!] Cannot find record for %s" % ip) continue db.t_evidence.update_or_insert( f_hosts_id=record.id, f_filename="%s-msfpro-%s.png" % (ip, loot['name']), f_evidence="%s-msfpro-%s.png" % (ip, loot['name']), f_data=loot['data'], f_type='Screenshot', f_text='From MetasploitPRO' ) db.commit() loot_count += 1 return loot_count
def add(): if request.args(0): record = get_host_record(request.args(0)) db.t_netbios.f_hosts_id.default = record.id response.title = "%s :: Add NetBIOS Data" % (settings.title) form=crud.create(db.t_netbios, next='edit/[id]', message="NetBIOS data added") return dict(form=form)
def add(): if request.args(0) is not None: record = get_host_record(request.args(0)) db.t_evidence.f_hosts_id.default = record.id else: record = None if request.extension == 'load': buttons=[] else: buttons=['submit'] if record: form=SQLFORM(db.t_evidence, buttons=buttons, upload=URL('download'), fields=['f_type', 'f_other_type', 'f_text', 'f_evidence'], _action=URL('add', args=[ record.id ]), _id="evidence_add_form") else: form=SQLFORM(db.t_evidence, buttons=buttons, upload=URL('download'), fields=['f_hosts_id', 'f_type', 'f_other_type', 'f_text', 'f_evidence'], _action=URL('add'), _id="evidence_add_form") if request.vars.f_evidence is not None: form.vars.f_filename = request.vars.f_evidence.filename if form.accepts(request.vars, session): response.flash = "Evidence added" response.headers['web2py-component-command'] = 'evidencetable.fnReloadAjax();' return "" elif form.errors: response.flash = "Error in form submission" return TABLE(*[TR(k, v) for k, v in form.errors.items()]) db.t_evidence.f_hosts_id.default = None response.title = "%s :: Add Evidence" % (settings.title) return dict(form=form)
def process_screenshot_loot(loot_list=[], msf=None): """ Takes an array of loot records in loot_list, downloads the screenshot and adds it to the database """ db = current.globalenv['db'] #cache = current.globalenv['cache'] loot_count = 0 for loot_id in loot_list: loot = msf.loot_download(loot_id) ip = loot_list[loot_id] if loot['ltype'] != 'host.windows.screenshot': logging.error(" [!] %s/%s is not a screenshot, it is a %s" % (ip, loot['name'], loot['ltype'])) else: record = get_host_record(ip) if not record: logging.error(" [!] Cannot find record for %s" % ip) continue db.t_evidence.update_or_insert( f_hosts_id=record.id, f_filename="%s-msfpro-%s.png" % (ip, loot['name']), f_evidence="%s-msfpro-%s.png" % (ip, loot['name']), f_data=loot['data'], f_type='Screenshot', f_text='From MetasploitPRO') db.commit() loot_count += 1 return loot_count
def update_db(f_type=None, record=None, data=None, filename=None, ipaddr=None): """Adds or updates an existing record id""" if record is None: # inserting a new record into the database if ipaddr is None: print "ERROR: No IPv4 address provided" return False host_id = get_host_record(ipaddr) if not host_id: print "ERROR: %s is not a host in the database" % (ipaddr) return False try: db.t_evidence.insert( f_hosts_id = host_id.id, f_filename = filename, f_data = data, f_type = f_type ) except Exception, e: print "ERROR inserting record:", e db.commit() return False
def mass_assign(): """ Upload a CSV file that mass-assigns OS records to Hosts. If a CPE record is provided, look it up in the DB. If not lookup the vendor and product in the DB File format: ipaddress,cpe,family,vendor,product,certainty,osclass """ response.title = "%s :: Mass OS Update" % (settings.title) form = SQLFORM.factory( Field('osfile', 'upload', uploadfolder=os.path.join(request.folder, 'data', 'misc'), label=T('OS CSV File')), ) if form.accepts(request.vars,session): filename = os.path.join(request.folder,'data/misc',form.vars.osfile) import csv from skaldship.cpe import lookup_cpe #from skaldship.general import counter = 0 with open(filename, "rb") as f: for row in csv.reader(f): host_id = get_host_record(row[0]) if not host_id: print "[%s] - Record not found" % (row[0]) continue cpe = row[1] family = row[2] vendor = row[3] product = row[4] certainty = row[5] osclass = row[6] os_id = None if cpe: # we have a cpe entry from xml! hooray! cpe_name = cpe.replace('cpe:/o:', '') os_id = lookup_cpe(cpe_name) #else: # no cpe attribute in xml, go through our messsy lookup # os_id = guess_cpe_os(os_rec) if os_id: db.t_host_os_refs.insert(f_certainty=certainty, f_family=family, f_class=osclass, f_hosts_id=host_id, f_os_id=os_id) db.commit() counter += 1 else: logger.error("OS not found: %s" % (row)) response.flash = "%s Hosts updated with new OS records" % (counter) elif form.errors: response.flash = 'Error in form' return dict(form=form)
def add(): if request.args(0): record = get_host_record(request.args(0)) db.t_netbios.f_hosts_id.default = record.id response.title = "%s :: Add NetBIOS Data" % (settings.title) form = crud.create(db.t_netbios, next='edit/[id]', message="NetBIOS data added") return dict(form=form)
def add(): if request.args(0) is not None: record = get_host_record(request.args(0)) db.t_snmp.f_hosts_id.default = record.id response.title = "%s :: Create SNMP Entry" % (settings.title) form=crud.create(db.t_snmp,next='edit/[id]') db.t_snmp.f_hosts_id.default = None return dict(form=form)
def add(): if request.args(0) is not None: record = get_host_record(request.args(0)) db.t_snmp.f_hosts_id.default = record.id response.title = "%s :: Create SNMP Entry" % (settings.title) form = crud.create(db.t_snmp, next='edit/[id]') db.t_snmp.f_hosts_id.default = None return dict(form=form)
def process_loot_files(loot_list=[]): """ Processes locally stored (to web2py) MSF password loot files into the account database. Args: loot_list: an array of [filename, settings.password_file_types, port, host_id] Returns: An array of [filename, result text] """ from skaldship.passwords.utils import process_password_file, insert_or_update_acct #import os db = current.globalenv['db'] data = [] for loot in loot_list: if isinstance(loot, []): (filename, file_type, port) = loot else: log("Invalid loot sent: %s" % loot, logging.ERROR) continue try: (proto, number) = port.split('/') except AttributeError as e: log("Invalid port sent: %s", port, logging.ERROR) try: pw_data = open(filename, "rb").readlines().split('\n') except IOError as e: log("Error opening %s: %s" % (filename, e), logging.ERROR) accounts = process_password_file( pw_data=pw_data, file_type=file_type, source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == number) query &= (db.t_services.f_proto == proto) query &= (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto=proto, f_number=number, f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added loot accounts for host: %s" % host.f_ipaddr) data.append({loot['host']: resp_text})
def by_host(): """ Returns a list of OS records based upon an host identifier (id, ipv4, ipv6) """ if request.args(0) is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: SNMP Records for %s" % (settings.title, host_title_maker(record)) snmplist = db(db.t_snmp.f_hosts_id==record.id).select() aaData = [] if request.extension == "json": for snmp in snmplist: # datatables json requires aaData to be specificly formatted aaData.append({ '0': A("edit", _target="snmp_update_%s" % (snmp.id), _href=URL('edit',extension='html',args=snmp.id)).xml(), '1': snmp.f_community, '2': snmp.f_version, '3': snmp.f_access, 'DT_RowId': snmp.id, }) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result form = TABLE(THEAD(TR(TH(T('ID'), _width="5%"), TH(T('Community')), TH(T('Version')), TH(T('Access')), ) ), _class="datatable", _id="snmptable", _style="width:100%") add = AddModal( db.t_snmp, 'Add', 'Add', 'Add SNMP String', fields=[ 'f_community', 'f_version', 'f_access'], cmd='snmptable.fnReloadAjax();' ) db.t_snmp.f_hosts_id.default = record.id db.t_snmp.id.comment = add.create() return dict(form=form, host=record, add=add)
def summary_by_host(): """ Returns a list of notes records based upon an host identifier (id, ipv4, ipv6) """ if request.args(0) is None: redirect( URL('default', 'error', vars={'msg': T('No host record provided')})) record = get_host_record(request.args(0)) if record is None: redirect( URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Notes for host %s" % (settings.title, host_title_maker(record)) rows = db(db.t_host_notes.f_hosts_id == record.id)(db.t_host_notes).select( db.t_host_notes.id, db.t_host_notes.f_note) aaData = [] if request.extension == "json": for r in rows: # datatables json requires aaData to be specificly formatted atxt = [] atxt.append('<a href="javascript:void()" onclick="delnotes_summ(' + str(r.id) + ')">X</a>') atxt.append(r.f_note) # add columns after this, don't do anything prior since it'll affect the hidden fields aaData.append(atxt) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result notes = TABLE(THEAD(TR( TH(T('[X]'), _width="5%"), TH(T('Note'), _width="90%"), ), _style="display:none"), _class="table table-condensed", _id="notestable_summary", _style="width:100%") return dict(notes=notes)
def process_file(filename=None, asset_group=None, engineer=None): # Upload and process hping Scan file from skaldship.hosts import get_host_record, do_host_status, add_or_update log(" [*] Processing hping scan file %s" % filename) hoststats = 0 nodefields = {'f_engineer': engineer, 'f_asset_group': asset_group, 'f_confirmed': False} svc_db = db.t_services host_ip = None ICMP_type = '' answer_ip = '' with open(filename) as f: for line in f: if "IP: " in line: host_ip = line.split()[1] if IS_IPADDRESS()(host_ip)[1] == None: nodefields['f_ipaddr'] = host_ip host_rec = add_or_update(nodefields, update=True) hoststats += 1 else: log(" [!] ERROR: Not a valid IP Address (%s)" % host_ip, logging.ERROR) if "[*] " in line: ICMP_type = line.split()[1] if "ip=" in line: ip = line.split('=')[2] answer_ip = ip.split()[0] if "transmitted" in line: packets = line.split() if packets[0] == packets[3]: if answer_ip != host_ip: response = T("No") else: response = T("Yes") else: response = T("No") get_id = get_host_record(host_ip) svc_db.update_or_insert( f_hosts_id=get_id.id, f_proto='ICMP', f_number='0', f_status=response, f_name=ICMP_type ) db.commit() f.close() do_host_status(asset_group=asset_group) log(" [*] Import complete, %s hosts added/updated" % hoststats)
def add_host(self, address=None, ports=None): """Looks up the host and adds the result to the query""" host_rec = get_host_record(address) if host_rec is None: sys.stderr.write("%s invalid address!\n" % (address)) else: q = (db.t_services.f_hosts_id == host_rec.id) for port in ports: q &= (db.t_services.f_proto == port[0]) q &= (db.t_services.f_number == port[1]) if self.host_query is None: self.host_query = q else: self.host_query |= q return
def process_file(filename=None, asset_group=None, engineer=None): # Upload and process hping Scan file from skaldship.hosts import get_host_record, do_host_status, add_or_update from gluon.validators import IS_IPADDRESS log(" [*] Processing hping scan file %s" % filename) hoststats = 0 nodefields = {'f_engineer': engineer, 'f_asset_group': asset_group, 'f_confirmed': False} svc_db = db.t_services host_ip = None ICMP_type = '' answer_ip = '' with open(filename) as f: for line in f: if "IP: " in line: host_ip = line.split()[1] if IS_IPADDRESS()(host_ip)[1] == None: nodefields['f_ipaddr'] = host_ip db.t_hosts.update_or_insert(**nodefields) db.commit() hoststats += 1 else: log(" [!] ERROR: Not a valid IP Address (%s)" % host_ip, logging.ERROR) if "[*] " in line: ICMP_type = line.split()[1] if "ip=" in line: ip = line.split('=')[2] answer_ip = ip.split()[0] if "transmitted" in line: packets = line.split() if packets[0] == packets[3]: if answer_ip != host_ip: response = "No" else: response = "Yes" else: response = "No" get_id = get_host_record(host_ip) svc_db.update_or_insert( f_hosts_id=get_id.id, f_proto='ICMP', f_number='0', f_status=response, f_name=ICMP_type ) db.commit()
def process_pwdump_loot(loot_list=[], msf=None): """ Takes an array of loot records in loot_list, downloads the pwdump file and adds the users. """ from skaldship.passwords.utils import process_password_file, insert_or_update_acct db = current.globalenv['db'] #cache = current.globalenv['cache'] data = [] for loot_id in loot_list: loot = msf.loot_download(loot_id) if loot['ltype'] not in ['host.windows.pwdump', 'windows.hashes']: log("Loot is not a pwdump, it is a %s" % loot['ltype'], logging.ERROR) continue else: # process the pwdump file pw_data = loot['data'].split('\n') accounts = process_password_file( pw_data=pw_data, file_type='PWDUMP', source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == '0') & ( db.t_services.f_proto == 'info') & (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto="info", f_number="0", f_status="info", f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added pwdump records for host: %s" % host.f_ipaddr) data.append({loot['host']: resp_text}) return data
def add_ajax(): record = None if request.vars.has_key('f_hosts_id'): record = get_host_record(request.vars.f_hosts_id) if record: db.t_host_notes.f_hosts_id.default = record.id form=SQLFORM(db.t_host_notes, buttons=[], _action=URL('add_ajax', extension='json'), _id="notes_add_form") if form.accepts(request.vars, formname='t_host_notes_create'): response.flash = 'Note added' response.headers['web2py-component-command'] = 'notesumstable.fnReloadAjax(); notestable.fnReloadAjax();' return elif form.errors: response.flash = "Error in form submission" return TABLE(*[TR(k, v) for k, v in form.errors.items()]) db.t_host_notes.f_hosts_id.default = None return dict(form=form)
def process_pwdump_loot(loot_list=[], msf=None): """ Takes an array of loot records in loot_list, downloads the pwdump file and adds the users. """ from skaldship.passwords import process_password_file, insert_or_update_acct db = current.globalenv['db'] cache = current.globalenv['cache'] logging.debug('loot_list = %s' % (loot_list)) data = [] for loot_id in loot_list: loot = msf.loot_download(loot_id) if loot['ltype'] not in ['host.windows.pwdump', 'windows.hashes']: logging.error("Loot is not a pwdump, it is a %s" % loot['ltype']) continue else: # process the pwdump file pw_data = loot['data'].split('\n') accounts = process_password_file( pw_data=pw_data, file_type='PWDUMP', source='Metasploit', ) # find the info/0 service id for the host host_id = get_host_record(loot['host']) query = (db.t_services.f_number == '0') & (db.t_services.f_proto == 'info') & (db.t_services.f_hosts_id == host_id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto="info", f_number="0", f_status="info", f_hosts_id=host_id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) logging.info("Added pwdump records for host: %s" % (loot['host'])) data.append({ loot['host']: resp_text }) return data
def add_ajax(): record = None if request.vars.has_key('f_hosts_id'): record = get_host_record(request.vars.f_hosts_id) if record: db.t_host_notes.f_hosts_id.default = record.id form = SQLFORM(db.t_host_notes, buttons=[], _action=URL('add_ajax', extension='json'), _id="notes_add_form") if form.accepts(request.vars, formname='t_host_notes_create'): response.flash = 'Note added' response.headers[ 'web2py-component-command'] = 'notesumstable.fnReloadAjax(); notestable.fnReloadAjax();' return elif form.errors: response.flash = "Error in form submission" return TABLE(*[TR(k, v) for k, v in form.errors.items()]) db.t_host_notes.f_hosts_id.default = None return dict(form=form)
def summary_by_host(): """ Returns a list of notes records based upon an host identifier (id, ipv4, ipv6) """ if request.args(0) is None: redirect(URL('default', 'error', vars={'msg': T('No host record provided')})) record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Notes for host %s" % (settings.title, host_title_maker(record)) rows = db(db.t_host_notes.f_hosts_id == record.id)(db.t_host_notes).select(db.t_host_notes.id, db.t_host_notes.f_note) aaData = [] if request.extension == "json": for r in rows: # datatables json requires aaData to be specificly formatted atxt = [] atxt.append('<a href="javascript:void()" onclick="delnotes_summ(' + str(r.id) +')">X</a>') atxt.append(r.f_note) # add columns after this, don't do anything prior since it'll affect the hidden fields aaData.append(atxt) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result notes = TABLE(THEAD(TR(TH(T('[X]'), _width="5%"), TH(T('Note'), _width="90%"), ), _style="display:none" ), _class="table table-condensed", _id="notestable_summary", _style="width:100%") return dict(notes=notes)
def popover(): """ Returns the detail of a host for popovers """ host_rec = get_host_record(request.args(0)) resp = {} if not host_rec: resp['title'] = "Host not found" resp['content'] = "" else: svcs = host_rec.t_services svc_cnt = 0 vuln_cnt = 0 acct_cnt = 0 for svc in svcs.select(): svc_cnt += 1 vuln_cnt += svc.t_service_vulns.count() acct_cnt += svc.t_accounts.count() host_os = (0, 'Unknown') for os_rec in host_rec.t_host_os_refs.select(): if os_rec.f_certainty > host_os[0]: host_os = (os_rec.f_certainty, db.t_os[os_rec.f_os_id].f_title) resp['title'] = host_title_maker(host_rec) resp['content'] = XML( TABLE( TR(TD(T('Asset Group')), TD(host_rec.f_asset_group)), TR(TD(T('Engineer')), TD(db.auth_user[host_rec.f_engineer].username)), TR(TD(T('OS')), TD("%s (%s)" % (host_os[1], host_os[0]))), TR(TD(T('Services')), TD(svc_cnt), _class="success"), TR(TD(T('Vulnerabilities')), TD(vuln_cnt), _class="error"), TR(TD(T('Accounts')), TD(acct_cnt), _class="warning"), _class="table table-condensed", )) return resp
def popover(): """ Returns the detail of a host for popovers """ host_rec = get_host_record(request.args(0)) resp = {} if not host_rec: resp['title'] = "Host not found" resp['content'] = "" else: svcs = host_rec.t_services svc_cnt = 0 vuln_cnt = 0 acct_cnt = 0 for svc in svcs.select(): svc_cnt += 1 vuln_cnt += svc.t_service_vulns.count() acct_cnt += svc.t_accounts.count() host_os = (0, 'Unknown') for os_rec in host_rec.t_host_os_refs.select(): if os_rec.f_certainty > host_os[0]: host_os = (os_rec.f_certainty, db.t_os[os_rec.f_os_id].f_title) resp['title'] = host_title_maker(host_rec) resp['content'] = XML(TABLE( TR(TD(T('Asset Group')), TD(host_rec.f_asset_group)), TR(TD(T('Engineer')), TD(db.auth_user[host_rec.f_engineer].username)), TR(TD(T('OS')), TD("%s (%s)" % (host_os[1], host_os[0]))), TR(TD(T('Services')), TD(svc_cnt), _class="success"), TR(TD(T('Vulnerabilities')), TD(vuln_cnt), _class="error"), TR(TD(T('Accounts')), TD(acct_cnt), _class="warning"), _class="table table-condensed", )) return resp
def vulndata_by_host(): """ Returns a list of vulnerabilties based upon an host identifier (id, ipv4, ipv6) """ from skaldship.metasploit import msf_get_config msf_settings = msf_get_config(session) record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Vulnerabilities for %s" % (settings.title, host_title_maker(record)) services = db(db.t_services.f_hosts_id==record.id).select(db.t_services.id, db.t_services.f_proto, db.t_services.f_number) if request.extension == "json": aaData = [] for svc in services: # service info q = db(db.t_service_vulns.f_services_id == svc.id).select() for vulninfo in q: atxt = {} exploit_list = [] vulndetails = db(db.t_vulndata.id == vulninfo.f_vulndata_id).select(cache=(cache.ram, 300)).first() exploits = db(db.t_exploit_references.f_vulndata_id == vulninfo.f_vulndata_id).select(orderby=~db.t_exploit_references.id) if len(exploits) > 0: expl_count = "Yes (%d)" % (len(exploits)) for expl in exploits: for expl_data in db(db.t_exploits.id == expl.f_exploit_id).select(cache=(cache.ram, 300)): exp_link = expl_data.f_name if expl_data.f_source == 'exploitdb': if db.t_exploitdb[expl_data.f_title]: exploitdb_href = URL('exploitdb', 'detail.html', args=expl_data.f_title) else: exploitdb_href = URL('default', 'redirect', extension='html', vars={'url': 'http://www.exploit-db.com/exploits/%s' % expl_data.f_title}) exp_link = A(IMG(_align="absmiddle", _width=16, _height=16, _src=URL('static','images/exploitdb.ico')), ' exploitdb - ' + expl_data.f_name,_href=exploitdb_href, _target="exploitdb_%s" % (expl_data.f_name)) elif expl_data.f_source == 'metasploit': if session.msf_workspace: msf_uri = os.path.join(msf_settings['url'], 'workspaces', session.msf_workspace_num, 'modules', expl_data.f_title) else: msf_uri = URL('default', 'redirect', extension='html', vars={'url': 'http://www.rapid7.com/db/modules/%s' % expl_data.f_title}) exp_link = A(IMG(_align="absmiddle", _width=16, _height=16, _src=URL('static','images/msf.gif')), ' metasploit - ' + expl_data.f_name, _href=msf_uri, _target="msf_%s" % (expl_data.f_name)) elif expl_data.f_source == 'canvas': exp_link = SPAN(IMG(_align="absmiddle", _width=16, _height=16, _src=URL('static','images/canvas.png')), ' canvas - ' + expl_data.f_name) exploit_list.append("%s : %s (%s/%s)" % (expl_data.f_title, exp_link, expl_data.f_rank, expl_data.f_level)) else: expl_count = "" atxt['0'] = IMG(_src=URL(request.application,'static','images/details_open.png')).xml() atxt['1'] = A('edit', _target="service_vuln_update_%s" % (vulninfo.id), _href=URL('vulns', 'service_vulns_edit', args=vulninfo.id, extension='html')).xml() if vulninfo.f_exploited: atxt['2'] = '<input id="exploited" value="' + str(vulninfo.id) + '" type="checkbox", checked>' else: atxt['2'] = '<input id="exploited" value="' + str(vulninfo.id) + '" type="checkbox">' atxt['3'] = "%s/%s" % (svc.f_proto, svc.f_number) atxt['4'] = A(vulndetails.f_vulnid, _target="vulndata_%s" % (vulndetails.id), _href=URL('vulns', 'vulninfo_by_vulnid', args=vulndetails.f_vulnid, extension='html')).xml() atxt['5'] = vulndetails.f_severity atxt['6'] = vulndetails.f_cvss_score atxt['7'] = SPAN(vulninfo.f_status,_id="vulninfo_status",_vulnstatus=vulninfo.f_status).xml() atxt['8'] = expl_count atxt['9'] = MARKMIN(vulninfo.f_proof).xml() atxt['10'] = MARKMIN(vulndetails.f_description).xml() atxt['11'] = vulndetails.f_title atxt['12'] = "<br />\n".join(exploit_list) atxt['DT_RowId'] = vulninfo.id aaData.append(atxt) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result add = AddModal( db.t_service_vulns, 'Add', 'Add', 'Add Vulnerability', #fields=[ #], cmd='vulntable.fnReloadAjax();' ) #db.t_service_vulns.f_services_id.default = svc.id svc_set = [] for svc in services: svc_set.append([svc.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.id]), svc.f_proto, svc.f_number)]) db.t_service_vulns.f_services_id.requires = IS_IN_SET(svc_set) db.t_service_vulns.id.comment = add.create() form = TABLE(THEAD(TR(TH('', _width="5%"), TH(T(''), _width="5%"), TH(T('Pwned'), width="5%"), TH(T('Port')), TH(T('Vuln ID')), TH(T('Sev')), TH(T('CVSS')), TH(T('Status')), TH(T('Exploits')), TH(T('Proof')), TH(T('Description')), TH(T('Title')), TH(T('Exploit List')), ) ), _class="datatable", _id="vulntable", _style="width:100%") return dict(form=form, host=record, add=add)
def parse_host(self, host): """ Parse an XML host data from ShodanHQ results """ from gluon.validators import IS_IPADDRESS db = current.globalenv['db'] hostfields = {} ipaddr = host.get('ip') if self.ip_only and ipaddr not in self.ip_only: log(" [-] %s is not in the only list... skipping" % (ipaddr)) #sys.stderr.write(msg) self.stats['hosts_skipped'] += 1 return if ipaddr in self.ip_exclude: log(" [-] %s is in exclude list... skipping" % (ipaddr)) if IS_IPADDRESS(is_ipv4=True)(ipaddr)[1] is None: # address is IPv4: hostfields['f_ipv4'] = ipaddr elif IS_IPADDRESS(is_ipv6=True)(ipaddr)[1] is None: hostfields['f_ipv6'] = ipaddr else: log(" [!] Invalid IP Address in report: %s" % (ipaddr)) return hostname = host.findtext('hostnames') if hostname: hostfields['f_hostname'] = hostname # check to see if IP exists in DB already if 'f_ipv4' in hostfields: host_rec = db(db.t_hosts.f_ipv4 == hostfields['f_ipv4']).select().first() else: host_rec = db(db.t_hosts.f_ipv6 == hostfields['f_ipv6']).select().first() if host_rec is None: hostfields['f_asset_group'] = self.asset_group hostfields['f_engineer'] = self.engineer host_id = db.t_hosts.insert(**hostfields) db.commit() self.stats['hosts_added'] += 1 log(" [-] Adding IP: %s" % (ipaddr)) elif host_rec is not None: db.commit() if 'f_ipv4' in hostfields: host_id = db(db.t_hosts.f_ipv4 == hostfields['f_ipv4']).update(**hostfields) db.commit() host_id = get_host_record(hostfields['f_ipv4']) host_id = host_id.id self.stats['hosts_updated'] += 1 log(" [-] Updating IP: %s" % (hostfields['f_ipv4'])) else: host_id = db(db.t_hosts.f_ipv6 == hostfields['f_ipv6']).update(**hostfields) db.commit() host_id = get_host_record(hostfields['f_ipv6']) host_id = host_id.id self.stats['hosts_updated'] += 1 log(" [-] Updating IP: %s" % (hostfields['f_ipv6'])) else: self.stats['hosts_skipped'] += 1 db.commit() log(" [-] Skipped IP: %s" % (ipaddr)) return # process the service / data f_number = host.get('port') if f_number == '161': # only udp provided by shodanhq is snmp f_proto = 'udp' else: f_proto = 'tcp' f_status = 'open' f_name = '' addl_fields = {} # extract the data field for processing port_data = host.findtext('data') # for ssh, telnet and smtp throw data into the banner if f_number == '21': f_banner = "\n".join(self.SMTP_FTP_220.findall(port_data)) f_name = 'FTP' addl_fields = { 'ftp.banner': port_data, } elif f_number == '22': f_banner = port_data f_name = 'SSH' addl_fields = { 'ssh.banner': port_data, } elif f_number == '23': f_banner = port_data f_name = 'Telnet' elif f_number == '25': f_banner = "\n".join(self.SMTP_FTP_220.findall(port_data)) f_name = 'SMTP' addl_fields = { 'smtp.banner': port_data, } elif f_number in HTTP_PORTS: # TODO: parse HTTP headers.. ugly f_banner = port_data f_name = 'HTTP' addl_fields = { 'http.banner': port_data, } elif f_number == '1900': f_banner = port_data f_name = 'UPNP' addl_fields = { 'upnp.banner': port_data, } else: f_banner = port_data query = (db.t_services.f_proto == f_proto) & (db.t_services.f_number == f_number) & (db.t_services.f_hosts_id == host_id) svc_row = db(query).select().first() if svc_row: # we found a service record! Check for similar status, names and banners do_update = False if svc_row.f_status != f_status: svc_row.f_status = f_status do_update = True if svc_row.f_name != f_name: svc_row.f_name = f_name do_update = True if svc_row.f_banner != f_banner: svc_row.f_banner = f_banner do_update = True svc_id = svc_row.id if do_update: svc_row.update_record() db.commit() didwhat = "Updated" self.stats['services_updated'] += 1 else: didwhat = "Unaltered" else: # we have a new service! svc_id = db.t_services.insert( f_proto=f_proto, f_number=f_number, f_status=f_status, f_name=f_name, f_banner=f_banner, f_hosts_id=host_id ) db.commit() didwhat = "Added" self.stats['services_added'] += 1 log(" [-] %s service: (%s) %s/%s" % (didwhat, ipaddr, f_proto, f_number)) for k, v in addl_fields.iteritems(): # add additional field entries as service_info records db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=k, f_text=v, ) db.commit()
logging.warn("No IP Address found in record. Skipping") continue if host_rec is None: host_id = db.t_hosts.insert(**nodefields) db.commit() hoststats['added'] += 1 print(" [-] Adding %s" % (ipaddr)) elif host_rec is not None and update_hosts: db.commit() if iptype == 'ipv4': host_id = db(db.t_hosts.f_ipv4 == nodefields['f_ipv4']).update(**nodefields) elif iptype == 'ipv6': host_id = db(db.t_hosts.f_ipv6 == nodefields['f_ipv6']).update(**nodefields) db.commit() host_id = get_host_record(ipaddr) host_id = host_id.id hoststats['updated'] += 1 print(" [-] Updating %s" % (ipaddr)) else: hoststats['skipped'] += 1 db.commit() print(" [-] Skipped %s" % (ipaddr)) continue hosts.append(host_id) # : for hostscripts in node.findall('hostscript/script'): svc_id = db.t_services.update_or_insert(f_proto='info', f_number=0, f_status='open', f_hosts_id=host_id) db.commit() for script in hostscripts:
def paste(): """ Import and parse password pasted to a textbox into t_accounts """ from skaldship.general import check_datadir check_datadir(request.folder) # Service_id is primary, host_id is secondary, if none then list # all the services svc_set = [] url = URL('accounts', 'paste') if request.vars.has_key('service_id'): try: record = db.t_services[request.vars.service_id] svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) url = URL('accounts', 'paste', vars={'service_id': request.vars.service_id}) except: pass elif request.vars.has_key('host_id'): try: host_record = get_host_record(request.vars.host_id) svc_records = db( db.t_services.f_hosts_id == host_record.id).select( cache=(cache.ram, 30)) url = URL('accounts', 'paste', vars={'host_id': request.vars.host_id}) for record in svc_records: svc_set.append( (record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) except: pass if len(svc_set) == 0: # all services svc_records = db(db.t_services).select(cache=(cache.ram, 30)) svc_set = [] for record in svc_records: svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) if request.extension == "load": buttons = [] else: buttons = ['submit'] form = SQLFORM.factory( Field('f_service', 'string', label=T('Host / Service'), requires=IS_IN_SET(svc_set), default=svc_set[0][0]), Field('f_pwtext', 'text', label=T('Password text')), Field('f_type', 'string', label=T('File type'), default='PWDUMP', requires=IS_IN_SET(settings.password_file_types)), Field('f_source', 'string', label=T('Source (if necessary)')), Field('f_add_to_evidence', 'boolean', label=T('Add file to Evidence')), buttons=buttons, _action=url, _id='accounts_paste_form' #_action=url, _id='accounts_paste_form', formstyle='bootstrap_modal' ) resp_text = "" accounts_added = [] accounts_updated = [] if form.errors: response.flash = 'Error in form' return TABLE(*[TR(k, v) for k, v in form.errors.items()]) elif form.accepts(request.vars, session): from gluon.utils import web2py_uuid host_id = db.t_services[form.vars.f_service].f_hosts_id pwd_file_dir = os.path.join(request.folder, 'data', 'passwords', 'other') if not os.path.exists(pwd_file_dir): from gluon.fileutils import mktree mktree(pwd_file_dir) filename = "%s-pwfile-%s" % (host_id, web2py_uuid()) full_file_path = os.path.join(request.folder, 'data/passwords/other', filename) of = open(full_file_path, "w") of.write(form.vars.f_pwtext) of.close() logger.debug("Processing password file: %s" % (full_file_path)) account_data = process_password_file(pw_file=full_file_path, file_type=request.vars.f_type, source=request.vars.f_source) response.headers[ 'web2py-component-command'] = 'accounttable.fnReloadAjax();' resp_text = insert_or_update_acct(form.vars.f_service, account_data) if form.vars.f_add_to_evidence is True: # add the password file to evidence try: pwdata = open(full_file_path, "r").readlines() except Exception, e: logger.error("Error opening %s: %s" % (full_file_path, e)) resp_text += "Error opening %s: %s\n" % (full_file_path, e) db.t_evidence.insert(f_hosts_id=host_id, f_type='Password File', f_text=form.vars.f_type, f_filename=filename, f_evidence=filename, f_data=pwdata) resp_text += "\n%s added to evidence\n" % (filename) db.commit()
def list(): """ Returns a list of notes records based upon an host identifier (id, ipv4, ipv6) """ aaData = [] response.title = "%s :: Notes" % (settings.title) if request.args(0) is not None: record = get_host_record(request.args(0)) if record: response.title = "%s :: Notes for %s" % (settings.title, host_title_maker(record)) else: record = None if request.extension == "json": if record: rows = db(db.t_host_notes.f_hosts_id == record.id).select() else: rows = db(db.t_host_notes.id > 0).select() for r in rows: aTxt = {} aaData.append({ '0': A('edit', _target="host_notes_%s" % (r.id), _href=URL('edit', args=r.id)).xml(), '1': host_a_maker(r.f_hosts_id).xml(), '2': r.f_note, 'DT_RowId': r.id }) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result if record: add_fields = ['f_note'] else: add_fields = ['f_hosts_id', 'f_note'] add_note = AddModal( db.t_host_notes, 'Add', 'Add Note', 'Add Note', #fields=add_fields, cmd='notestable.fnReloadAjax();', flash="Note added" ) if record: db.t_host_notes.f_hosts_id.default = record.id db.t_host_notes.id.comment = add_note.create() notes = TABLE(THEAD(TR(TH(T('ID'), _width="5%"), TH(T('Host'), _width="20%"), TH(T('Note'), _width="95%"), ) ), _class="datatable", _id="notestable", _style="width:100%") return dict(notes=notes, host=record, add_note=add_note)
# XXX: for now just take the first hostname nodefields['f_hostname'] = node.find('names/name').text # check to see if IP exists in DB already query = (db.t_hosts.f_ipaddr == ip) host_rec = db(query).select().first() if host_rec is None: host_id = db.t_hosts.insert(**nodefields) db.commit() hoststats['added'] += 1 log(" [-] Adding IP: %s" % ip) elif update_hosts: db.commit() db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(**nodefields) db.commit() host_id = get_host_record(nodefields['f_ipaddr']) host_id = host_id.id hoststats['updated'] += 1 log(" [-] Updating IP: %s" % ip) else: hoststats['skipped'] += 1 db.commit() log(" [-] Skipped IP: %s" % ip) continue hosts.append(host_id) # tests that aren't specific to any port we wrap up into a meta service # called "INFO" tests = node.findall("tests/test") if len(tests) > 0: svc_id = db.t_services.update_or_insert(f_proto="info", f_number="0", f_status="info", f_hosts_id=host_id)
def by_host(): """ Returns a list of services + serviceinfo based upon an host identifier (id, ipaddr) """ record = get_host_record(request.args(0)) if record is None: redirect( URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Accounts for %s" % (settings.title, host_title_maker(record)) query = (db.t_services.f_hosts_id == record.id) if request.extension == "json": aaData = [] rows = db(query).select( db.t_accounts.ALL, db.t_services.ALL, left=db.t_services.on( db.t_accounts.f_services_id == db.t_services.id)) for r in rows: if r.t_accounts.f_compromised == True: comprdiv = '<div class="acct_compromised" name="row_id" id="%s"><span class="icon-check"></span></div>' % r.t_accounts.id else: comprdiv = '<div class="acct_uncompromised" name="row_id" id="%s"/>' % r.t_accounts.id aaData.append({ '0': A("edit", _target="accounts_update_%s" % (r.t_accounts.id), _href=URL('accounts', 'edit', args=[r.t_accounts.id], extension='html')).xml(), '1': comprdiv, '2': A("%s/%s" % (r.t_services.f_proto, r.t_services.f_number), _target="services_edit_%s" % (r.t_services.id), _href=URL('services', 'edit', args=[r.t_services.id], extension='html')).xml(), '3': A(r.t_accounts.f_username, _target="accounts_username_%s" % (r.t_accounts.f_username), _href=URL('accounts', 'by_username', vars={'username': r.t_accounts.f_username}, extension='html')).xml(), '4': r.t_accounts.f_fullname, '5': r.t_accounts.f_password, '6': r.t_accounts.f_hash1_type, '7': r.t_accounts.f_hash1, '8': r.t_accounts.f_hash2_type, '9': r.t_accounts.f_hash2, '10': r.t_accounts.f_uid, '11': r.t_accounts.f_gid, '12': r.t_accounts.f_lockout, '13': r.t_accounts.f_duration, '14': r.t_accounts.f_source, '15': r.t_accounts.f_level, '16': r.t_accounts.f_description, 'DT_RowId': r.t_accounts.id, }) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result form = TABLE(THEAD( TR( TH(T(''), _width="5%"), TH(T('Compr'), _width="5%"), TH(T('Port')), TH(T('Username')), TH(T('Fullname')), TH(T('Password')), TH(T('Hash 1 Type')), TH(T('Hash 1')), TH(T('Hash 2 Type')), TH(T('Hash 2')), TH(T('UID')), TH(T('GID')), TH(T('Lockout')), TH(T('Duration')), TH(T('Source')), TH(T('Level')), TH(T('Description')), )), _class="datatable", _id="accounttable", _style="width:100%") add = AddModal( db.t_accounts, 'Add', 'Add', 'Add Account', #fields=[], cmd='accounttable.fnReloadAjax();') services = db(db.t_services.f_hosts_id == record.id).select( cache=(cache.ram, 30)) svc_set = [] for svc in services: svc_set.append([ svc.id, "%s :: %s/%s" % (host_title_maker( db.t_hosts[svc.f_hosts_id]), svc.f_proto, svc.f_number) ]) db.t_accounts.f_services_id.requires = IS_IN_SET(svc_set) db.t_accounts.id.comment = add.create() return dict(form=form, host=record, add=add)
def launch_terminal(record=None, launch_cmd=None): """ Opens a terminal on the Web Server. This only works if the web2py server is running on the user's workstation. The command to execute is stored in the user's settings db under auth_user.f_launch_cmd. Variables translated: _IP_ -- The current IP Address (v4 by default, v6 if exists) _LOGFILE_ -- Session logfile name (we prepend the path) If an IPv6 address is used then ':' is changed to '_' Example: xterm -sb -sl 1500 -vb -T 'manual hacking: _IP_' -n 'manual hacking: _IP_' -e script _LOGFILE_ """ record = get_host_record(record) # only execute launch on requests from localhost! if request.env['remote_addr'] != '127.0.0.1': logger.error("Can only launch from localhost! remote_addr = %s" % (request.env['remote_addr'])) return "Can only launch from localhost" if record is None: return "No record found" import string, os, subprocess import time from gluon.validators import IS_IPADDRESS # if no launch command use the default if not launch_cmd: launch_cmd = "xterm -sb -sl 1500 -vb -T 'manual hacking: _IP_' -n 'manual hacking: _IP_' -e 'script _LOGFILE_'" # check ip address ip = record.f_ipaddr logip = ip if IS_IPADDRESS(is_ipv6=True)(ip)[0] == None: logip = ip.replace(":", "_") logdir = "session-logs" logfilename = "%s-%s.log" % (logip, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) logfile = os.path.join(logdir, logfilename) launch_cmd = launch_cmd.replace("_IP_", ip) launch_cmd = launch_cmd.replace("_LOGFILE_", logfile) from skaldship.general import check_datadir # Check to see if data directories exist, create otherwise check_datadir(request.folder) datadir = os.path.join(os.getcwd(), request.folder, "data") # chdir to datadir! launch_cmd = launch_cmd.replace("_DATADIR_", datadir) os.chdir(datadir) # set environment variables os.environ['IP'] = ip os.environ['HOSTNAME'] = record.f_hostname or "" os.environ['DATADIR'] = datadir try: logger.info("Spawning: %s\n" % (launch_cmd)) print("Spawning: %s" % (launch_cmd)) subprocess.Popen(launch_cmd, shell=True)#, stdout=None, stdin=None, stderr=None) except Exception, e: logger.error("Error spawning launch cmd (%s): %s\n" % (launch_cmd, e)) print("Error spawning launch cmd (%s): %s\n" % (launch_cmd, e))
def import_file(): """ Import and parse password file into t_accounts """ import os from skaldship.general import check_datadir check_datadir(request.folder) # Service_id is primary, host_id is secondary, if none then list # all the services svc_set = [] url = URL('accounts', 'import_file') if request.vars.has_key('service_id'): try: record = db.t_services[request.vars.service_id] svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) url = URL('accounts', 'import_file', vars={'service_id': request.vars.service_id}) except: pass elif request.vars.has_key('host_id'): try: host_record = get_host_record(request.vars.host_id) svc_records = db( db.t_services.f_hosts_id == host_record.id).select( cache=(cache.ram, 30)) url = URL('accounts', 'import_file', vars={'host_id': request.vars.host_id}) for record in svc_records: svc_set.append( (record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) except: pass if len(svc_set) == 0: # all services svc_records = db(db.t_services).select(cache=(cache.ram, 30)) svc_set = [] for record in svc_records: svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) if request.extension == "load": buttons = [] else: buttons = ['submit'] form = SQLFORM.factory( Field('f_service', 'string', label=T('Host / Service'), requires=IS_IN_SET(svc_set), default=svc_set[0][0]), Field('f_filename', 'upload', uploadfolder=os.path.join(request.folder, settings.password_upload_dir), label=T('Password file')), Field('f_type', 'string', label=T('File type'), default='PWDUMP', requires=IS_IN_SET(settings.password_file_types)), Field('f_source', 'string', label=T('Source (if necessary)')), Field('f_add_to_evidence', 'boolean', label=T('Add Evidence')), Field('f_taskit', type='boolean', default=True, label=T('Run in background task')), buttons=buttons, _action=url, _id='accounts_import_form') resp_text = "" accounts_added = [] accounts_updated = [] if form.errors: response.flash = 'Error in form' return TABLE(*[TR(k, v) for k, v in form.errors.items()]) elif form.accepts(request.vars, session): if form.vars.f_filename is not None: orig_filename = request.vars.f_filename.filename filename = os.path.join(request.folder, settings.password_upload_dir, form.vars.f_filename) if form.vars.f_taskit: task = scheduler.queue_task( accounts_import_file, pvars=dict(filename=filename, service=form.vars.f_service, f_type=form.vars.f_type, f_source=form.vars.f_source), group_name=settings.scheduler_group_name, sync_output=5, timeout=settings.scheduler_timeout) if task.id: resp_text = "Submitted file for processing: %s" % (A( "task " + str(task.id), _href=URL(c='tasks', f='status', args=task.id)).xml()) else: resp_text = "Error submitting job: %s" % (task.errors) else: logger.info("Processing password file: %s" % (filename)) account_data = process_password_file(pw_file=filename, file_type=request.vars.f_type, source=request.vars.f_source) resp_text = insert_or_update_acct(form.vars.f_service, account_data) logger.info(resp_text) if form.vars.f_add_to_evidence is True: # add the password file to evidence try: pwdata = open(filename, "r").readlines() except Exception, e: logger.error("Error opening %s: %s" % (filename, e)) db.t_evidence.insert( f_hosts_id=db.t_services[form.vars.f_service].f_hosts_id, f_type='Password File', f_text=form.vars.f_type, f_filename=orig_filename, f_evidence=form.vars.f_filename, f_data=pwdata) db.commit()
def import_file(): """ Import and parse password file into t_accounts """ import os from skaldship.general import check_datadir check_datadir(request.folder) # Service_id is primary, host_id is secondary, if none then list # all the services svc_set = [] url=URL('accounts', 'import_file') if request.vars.has_key('service_id'): try: record = db.t_services[request.vars.service_id] svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) url = URL('accounts', 'import_file', vars={'service_id':request.vars.service_id}) except: pass elif request.vars.has_key('host_id'): try: host_record = get_host_record(request.vars.host_id) svc_records = db(db.t_services.f_hosts_id == host_record.id).select(cache=(cache.ram, 30)) url = URL('accounts', 'import_file', vars={'host_id':request.vars.host_id}) for record in svc_records: svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) except: pass if len(svc_set) == 0: # all services svc_records = db(db.t_services).select(cache=(cache.ram,30)) svc_set = [] for record in svc_records: svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) if request.extension == "load": buttons=[] else: buttons=['submit'] form = SQLFORM.factory( Field('f_service', 'string', label=T('Host / Service'), requires=IS_IN_SET(svc_set), default=svc_set[0][0]), Field('f_filename', 'upload', uploadfolder=os.path.join(request.folder, settings.password_upload_dir), label=T('Password file')), Field('f_type', 'string', label=T('File type'), default='PWDUMP', requires=IS_IN_SET(settings.password_file_types)), Field('f_source', 'string', label=T('Source (if necessary)')), Field('f_add_to_evidence', 'boolean', label=T('Add Evidence')), Field('f_taskit', type='boolean', default=True, label=T('Run in background task')), buttons=buttons, _action=url, _id='accounts_import_form' ) resp_text = "" accounts_added = [] accounts_updated = [] if form.errors: response.flash = 'Error in form' return TABLE(*[TR(k, v) for k, v in form.errors.items()]) elif form.accepts(request.vars, session): if form.vars.f_filename is not None: orig_filename = request.vars.f_filename.filename filename = os.path.join(request.folder, settings.password_upload_dir, form.vars.f_filename) if form.vars.f_taskit: task = scheduler.queue_task( accounts_import_file, pvars=dict( filename=filename, service=form.vars.f_service, f_type=form.vars.f_type, f_source=form.vars.f_source ), group_name=settings.scheduler_group_name, sync_output=5, timeout=settings.scheduler_timeout ) if task.id: resp_text = "Submitted file for processing: %s" % (A("task " + str(task.id), _href=URL(c='tasks', f='status', args=task.id)).xml()) else: resp_text = "Error submitting job: %s" % (task.errors) else: logger.info("Processing password file: %s" % (filename)) account_data = process_password_file( pw_file=filename, file_type=request.vars.f_type, source=request.vars.f_source ) resp_text = insert_or_update_acct(form.vars.f_service, account_data) logger.info(resp_text) if form.vars.f_add_to_evidence is True: # add the password file to evidence try: pwdata = open(filename, "r").readlines() except Exception, e: logger.error("Error opening %s: %s" % (filename, e)) db.t_evidence.insert( f_hosts_id = db.t_services[form.vars.f_service].f_hosts_id, f_type = 'Password File', f_text = form.vars.f_type, f_filename = orig_filename, f_evidence = form.vars.f_filename, f_data = pwdata) db.commit()
def parse(self, host_properties): """ Parse out the <HostProperties> xml content or CSV line. There can be a number of <tag> entries that are either useful to us in t_hosts or other areas. These are processed and returned as dictionary entries in 'hostdata' Args: host_properties: A <HostProperties> section from .nessus or a CSV line Returns: t_hosts.id, { hostdata } """ from gluon.validators import IS_IPADDRESS hostdata = {} if etree.iselement(host_properties): for tag in host_properties.findall('tag'): hostdata[tag.get('name')] = tag.text ipaddr = hostdata.get('host-ip') else: # with CSV each line has all the hostdata fields so we set them here for use later ipaddr = host_properties.get('IP Address') if not ipaddr: # Scanner CSV, use Host ipaddr = host_properties.get('Host') hostdata['mac-address'] = host_properties.get('MAC Address', '') hostdata['host-fqdn'] = host_properties.get('DNS Name', '') hostdata['netbios-name'] = host_properties.get('NetBIOS Name', '') if (ipaddr not in self.ip_include and self.ip_include) or (ipaddr in self.ip_exclude): log("Host in exclude or not in include list, skipping") self.stats['skipped'] += 1 return None, {} host_id = get_host_record(ipaddr) if host_id and not self.update_hosts: return host_id, hostdata # new host found, pull what we need for t_hosts hostfields = {} hostfields['f_engineer'] = self.engineer hostfields['f_asset_group'] = self.asset_group hostfields['f_confirmed'] = False # check ipv4/ipv6 and set hostfields accordingly if IS_IPADDRESS(is_ipv4=True)(ipaddr)[1] is None: hostfields['f_ipv4'] = ipaddr elif IS_IPADDRESS(is_ipv6=True)(ipaddr)[1] is None: hostfields['f_ipv6'] = ipaddr else: log("Invalid IP Address in HostProperties: %s" % ipaddr, logging.ERROR) return None, {} # pull out relevant hostfields for (k,v) in hostdata.iteritems(): if k == 'mac-address': # multiple mac addrs may appear wildly, just pull the first hostfields['f_macaddr'] = v[:v.find('\n')] elif k == 'host-fqdn': hostfields['f_hostname'] = v elif k == 'netbios-name': hostfields['f_netbios_name'] = v if not self.update_hosts and not host_id: result = self.db.t_hosts.validate_and_insert(**hostfields) if not result.id: log("Error adding host to DB: %s" % result.errors, logging.ERROR) return None, {} self.stats['added'] += 1 host_id = result.id log(" [-] Adding host: %s" % ipaddr) elif self.update_hosts: if hostfields['f_ipv4']: host_id = self.db(self.db.t_hosts.f_ipv4 == hostfields['f_ipv4']).update(**hostfields) self.db.commit() host_id = get_host_record(hostfields['f_ipv4']) if host_id: host_id = host_id.id log(" [-] Updating IP: %s" % (hostfields['f_ipv4'])) else: host_id = self.db(self.db.t_hosts.f_ipv6 == hostfields['f_ipv6']).update(**hostfields) self.db.commit() host_id = get_host_record(hostfields['f_ipv6']) host_id = host_id.id log(" [-] Updating IP: %s" % (hostfields['f_ipv6'])) self.stats['updated'] += 1 return host_id, hostfields
def host_to_ip(host_rec): if isinstance(host_rec, (int, str)): host_rec = get_host_record(host_rec) if not host_rec: return None return host_rec.get('f_ipaddr')
def aa_by_host(): """ Returns a list of vulnerabilties per port in a tree view format based upon an host identifier (id, ipv4, ipv6) """ record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) treeul=UL(_id='aatree_ul') db_svcs = db.t_services db_svulns = db.t_service_vulns db_vulns = db.t_vulndata services = db(db_svcs.f_hosts_id==record.id).select(db_svcs.f_number, db_svcs.id, db_svcs.f_proto, db_svcs.f_name,orderby=db_svcs.id) if settings.use_cvss: db_vsevs = db_vulns.f_cvss_score else: db_vsevs = db_vulns.f_severity tree = DIV(_id="aatree") for svc in services: nexlist = [] nexlist_single = [] expl_count = 0 exploit_list = UL() exploitdb = 0 metasploit = 0 canvas = 0 prev_f_status = '' vulnclass = '' for vulninfo in db( (db_svulns.f_services_id == svc.id) & (db_vulns.id == db_svulns.f_vulndata_id) ).select(orderby=~db_svulns.f_status|~db_vsevs, cache=(cache.ram, 120)): #init variables vulndetails = vulninfo.t_vulndata vulninfo = vulninfo.t_service_vulns cur_f_status = vulninfo.f_status #Generating the exploit lists exploits = db(db.t_exploit_references.f_vulndata_id == vulninfo.f_vulndata_id).select(orderby=~db.t_exploit_references.id) exploit_list_single = UL() if len(exploits) > 0: for expl in exploits: for expl_data in db(db.t_exploits.id == expl.f_exploit_id).select(db.t_exploits.f_source, db.t_exploits.f_title, db.t_exploits.f_name, db.t_exploits.f_rank, db.t_exploits.f_level): exp_link = expl_data.f_name if expl_data.f_source == 'exploitdb': exploitdb += 1 if db.t_exploitdb[expl_data.f_title]: exploitdb_href = URL('exploitdb', 'detail.html', args=expl_data.f_title) else: exploitdb_href = URL('default', 'redirect', extension='html', vars={'url': 'http://www.exploit-db.com/exploits/%s' % expl_data.f_title}) exp_link = A(IMG(_align="absmiddle", _width=16, _height=16, _src=URL('static','images/exploitdb.ico')), ' exploitdb - ' + expl_data.f_name,_href=exploitdb_href, _target="exploitdb_%s" % (expl_data.f_name)) elif expl_data.f_source == 'metasploit': metasploit += 1 if session.msf_workspace: msf_uri = os.path.join(msf_settings['url'], session.msf_workspace, 'modules', expl_data.f_title) else: msf_uri = URL('default', 'redirect', extension='html', vars={'url': 'http://www.rapid7.com/db/modules/%s' % expl_data.f_title}) exp_link = A(IMG(_align="absmiddle", _width=16, _height=16, _src=URL('static','images/msf.gif')), ' metasploit - ' + expl_data.f_name, _href=msf_uri, _target="msf_%s" % (expl_data.f_name)) elif expl_data.f_source == 'canvas': canvas += 1 exp_link = SPAN(IMG(_align="absmiddle", _width=16, _height=16, _src=URL('static','images/canvas.png')), ' canvas - ' + expl_data.f_name) #expl_link = ' canvas - ' + expl_data.f_name expl_count += 1 exploit_list_single.append(LI(expl_data.f_title , " : " , exp_link , " (" , expl_data.f_rank , "/" , expl_data.f_level, ")")) textdecoration="" if vulninfo.f_exploited == True and len(exploits) > 0: textdecoration="text-decoration:line-through underline; " elif vulninfo.f_exploited == True and len(exploits) == 0: textdecoration="text-decoration: line-through; " elif (vulninfo.f_exploited == False or vulninfo.f_exploited == None) and len(exploits) == 0: textdecoration="text-decoration: none;" #generation vuln link if settings.use_cvss: severity = int(float(vulndetails.f_cvss_score)) else: severity = int(vulndetails.f_severity) style = textdecoration + "color:" + severity_mapping(severity - 1)[2] vuln_title_link = A(vulndetails.f_vulnid, _title = vulninfo.f_status+ ' Severity: ' + str(severity), _style=style, _target="vulndata_%s" % (vulndetails.id), _href=URL(request.application,'vulns', 'vulninfo_by_vulnid', args=vulndetails.f_vulnid, extension='html')) if cur_f_status != prev_f_status and prev_f_status != '': nexlist.append(SPAN(nexlist_single, _class=vulnclass)) #for a line in the bottom nexlist.append(' ') nexlist_single = [] else: nexlist_single.append(' ') nexlist_single.append(vuln_title_link ) prev_f_status = vulninfo.f_status vulnclass = '' #style for vuln links if vulninfo.f_status == 'vulnerable-version': vulnclass='host_detail_vulnerable-version' if vulninfo.f_status == 'vulnerable-exploited': vulnclass='host_detail_vulnerable-exploited' if vulninfo.f_status == 'potential': vulnclass='host_detail_potential' if len(exploit_list_single) > 0: exploit_list.append(LI(SPAN(vuln_title_link), exploit_list_single)) #attach the last vuln list if len(nexlist_single)>0: nexlist.append(SPAN(nexlist_single, _class=vulnclass)) service_disp=SPAN(svc.f_proto + '/' + svc.f_number + ' - ' + str(svc.f_name)) expl_count = "Exploits - (%d)" % (expl_count) if len(nexlist)>0: if len(exploit_list) == 0: treeul.append(LI(service_disp,UL(LI(nexlist)))) #No exploits else: expl_count = SPAN(expl_count + " : metasploit (%d) exploitdb (%d) canvas (%d)" % (metasploit, exploitdb, canvas),_style="color:red") treeul.append(LI(service_disp,UL(LI(nexlist)), UL(LI(expl_count,exploit_list,_class="closed")))) else: treeul.append(LI(service_disp)) #No vulns tree = DIV(treeul, _id="aatree") return dict(tree=tree)
def refs_by_host(): """ Returns a list of OS records based upon an host identifier (id, ipv4, ipv6) """ if request.args(0) is None: redirect(URL('default', 'error', vars={'msg': T('No host record sent')})) record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: OS Records for %s" % (settings.title, host_title_maker(record)) oslist = db(db.t_host_os_refs.f_hosts_id==record.id).select() aaData = [] if request.extension == "json": for osdetail in oslist: osinfo = db.t_os(osdetail['f_os_id']) # datatables json requires aaData to be specificly formatted atxt = {} atxt['0'] = A('edit', _target="oswindow_%s" % (osdetail.id), _href=URL('refs_edit', args=[osdetail.id], extension='html')).xml() atxt['1'] = osdetail.f_family atxt['2'] = osdetail.f_class atxt['3'] = osdetail.f_certainty atxt['4'] = osinfo.f_cpename atxt['5'] = osinfo.f_title atxt['DT_RowId'] = osdetail.id aaData.append(atxt) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result form = TABLE(THEAD(TR(TH(T(''), _width="5%"), TH(T('Family')), TH(T('Class')), TH(T('Certainty')), TH(T('CPE Name')), TH(T('Title')), ) ), _class="datatable", _id="ostable", _style="width:100%") add_os_refs = AddModal( db.t_host_os_refs, 'Add', 'Add', 'Add OS', fields=['f_certainty', 'f_class', 'f_family', 'f_os_id'], cmd='ostable.fnReloadAjax();' ) db.t_host_os_refs.f_hosts_id.default = record.id db.t_host_os_refs.id.comment = add_os_refs.create() add_non_cpe = AddModal( db.t_os, 'Add Non-CPE OS', 'Add Non-CPE OS', 'Add Non-CPE OS', #fields=[], #cmd='ostable.fnReloadAjax();' ) db.t_os.id.comment = add_non_cpe.create() return dict(form=form, host=record, add_os_refs=add_os_refs, add_non_cpe=add_non_cpe)
def by_host(): """ Returns a list of services + serviceinfo based upon an host identifier (id, ipv4, ipv6) """ record = get_host_record(request.args(0)) if record is None: redirect( URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Services for %s" % (settings.title, host_title_maker(record)) services = db(db.t_services.f_hosts_id == record.id).select( db.t_services.id, db.t_services.f_proto, db.t_services.f_number, db.t_services.f_status, db.t_services.f_name, db.t_services.f_banner) #, cache=(cache.ram,60)) svcq = (db.t_services.f_hosts_id == record.id) infoq = (db.t_service_info.f_services_id == db.t_services.id) if request.extension == "json": #rows = db(svcq).select(db.t_services.ALL, db.t_service_info.ALL, left=db.t_service_info.on(infoq)) aaData = [] for svc in services: # service info atxt = {} q = db(db.t_service_info.f_services_id == svc.id).select() if len(q) > 0: addl = [] for svcinfo in q: addl.append(TR(TD(svcinfo.f_name), TD(svcinfo.f_text))) atxt['0'] = IMG(_src=URL(request.application, 'static', 'images/details_open.png')).xml() atxt['1'] = TABLE(THEAD(TR(TH(T('Name')), TH(T('Text')))), TBODY(addl)).xml() else: atxt['0'] = ("") atxt['1'] = ("") atxt['2'] = A('edit', _target="services_edit_%s" % (svc.id), _href=URL('edit', args=[svc['id']], extension='html')).xml() atxt['3'] = svc.f_proto if svc.f_number in HTTP_PORTS and svc.f_proto == "tcp" or svc.f_name == "HTTP": atxt['4'] = A(svc.f_number, _href="http://%s:%s/" % (record.f_ipaddr, svc.f_number), _target="%s-tcp-%s" % (record.f_ipaddr, svc.f_number)).xml() elif svc.f_number in HTTPS_PORTS and svc.f_proto == "tcp" or svc.f_name == "HTTPS": atxt['4'] = A(svc.f_number, _href="https://%s:%s/" % (record.f_ipaddr, svc.f_number), _target="%s-tcp-%s" % (record.f_ipaddr, svc.f_number)).xml() else: atxt['4'] = svc.f_number atxt['5'] = svc.f_status atxt['6'] = svc.f_name or "" atxt['7'] = svc.f_banner or "" atxt['DT_RowId'] = svc.id aaData.append(atxt) result = { 'sEcho': request.vars._, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result add = AddModal( db.t_services, 'Add', 'Add', 'Add Service', fields=['f_proto', 'f_number', 'f_status', 'f_name', 'f_banner'], cmd='servicetable.fnReloadAjax();') db.t_services.f_hosts_id.default = record.id db.t_services.id.comment = add.create() form = TABLE(THEAD( TR( TH('', _width="5%"), TH('Info'), TH(T('')), TH(T('Protocol')), TH(T('Number')), TH(T('Status')), TH(T('Name')), TH(T('Banner')), )), _class="datatable", _id="servicetable", _style="width:100%") return dict(form=form, host=record, add=add)
def list(): """ Returns a list of notes records based upon an host identifier (id, ipv4, ipv6) """ aaData = [] response.title = "%s :: Notes" % (settings.title) if request.args(0) is not None: record = get_host_record(request.args(0)) if record: response.title = "%s :: Notes for %s" % (settings.title, host_title_maker(record)) else: record = None if request.extension == "json": if record: rows = db(db.t_host_notes.f_hosts_id == record.id).select() else: rows = db(db.t_host_notes.id > 0).select() for r in rows: aTxt = {} aaData.append({ '0': A('edit', _target="host_notes_%s" % (r.id), _href=URL('edit', args=r.id)).xml(), '1': host_a_maker(r.f_hosts_id).xml(), '2': r.f_note, 'DT_RowId': r.id }) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result if record: add_fields = ['f_note'] else: add_fields = ['f_hosts_id', 'f_note'] add_note = AddModal( db.t_host_notes, 'Add', 'Add Note', 'Add Note', #fields=add_fields, cmd='notestable.fnReloadAjax();', flash="Note added") if record: db.t_host_notes.f_hosts_id.default = record.id db.t_host_notes.id.comment = add_note.create() notes = TABLE(THEAD( TR( TH(T('ID'), _width="5%"), TH(T('Host'), _width="20%"), TH(T('Note'), _width="95%"), )), _class="datatable", _id="notestable", _style="width:100%") return dict(notes=notes, host=record, add_note=add_note)
def parse_host(self, host): """ Parse an XML host data from ShodanHQ results """ from gluon.validators import IS_IPADDRESS db = current.globalenv['db'] hostfields = {} ipaddr = host.get('ip') if self.ip_only and ipaddr not in self.ip_only: log(" [-] %s is not in the only list... skipping" % (ipaddr)) #sys.stderr.write(msg) self.stats['hosts_skipped'] += 1 return if ipaddr in self.ip_exclude: log(" [-] %s is in exclude list... skipping" % (ipaddr)) if IS_IPADDRESS(is_ipv4=True)(ipaddr)[1] is None: # address is IPv4: hostfields['f_ipv4'] = ipaddr elif IS_IPADDRESS(is_ipv6=True)(ipaddr)[1] is None: hostfields['f_ipv6'] = ipaddr else: log(" [!] Invalid IP Address in report: %s" % (ipaddr)) return hostname = host.findtext('hostnames') if hostname: hostfields['f_hostname'] = hostname # check to see if IP exists in DB already if 'f_ipv4' in hostfields: host_rec = db( db.t_hosts.f_ipv4 == hostfields['f_ipv4']).select().first() else: host_rec = db( db.t_hosts.f_ipv6 == hostfields['f_ipv6']).select().first() if host_rec is None: hostfields['f_asset_group'] = self.asset_group hostfields['f_engineer'] = self.engineer host_id = db.t_hosts.insert(**hostfields) db.commit() self.stats['hosts_added'] += 1 log(" [-] Adding IP: %s" % (ipaddr)) elif host_rec is not None: db.commit() if 'f_ipv4' in hostfields: host_id = db(db.t_hosts.f_ipv4 == hostfields['f_ipv4']).update( **hostfields) db.commit() host_id = get_host_record(hostfields['f_ipv4']) host_id = host_id.id self.stats['hosts_updated'] += 1 log(" [-] Updating IP: %s" % (hostfields['f_ipv4'])) else: host_id = db(db.t_hosts.f_ipv6 == hostfields['f_ipv6']).update( **hostfields) db.commit() host_id = get_host_record(hostfields['f_ipv6']) host_id = host_id.id self.stats['hosts_updated'] += 1 log(" [-] Updating IP: %s" % (hostfields['f_ipv6'])) else: self.stats['hosts_skipped'] += 1 db.commit() log(" [-] Skipped IP: %s" % (ipaddr)) return # process the service / data f_number = host.get('port') if f_number == '161': # only udp provided by shodanhq is snmp f_proto = 'udp' else: f_proto = 'tcp' f_status = 'open' f_name = '' addl_fields = {} # extract the data field for processing port_data = host.findtext('data') # for ssh, telnet and smtp throw data into the banner if f_number == '21': f_banner = "\n".join(self.SMTP_FTP_220.findall(port_data)) f_name = 'FTP' addl_fields = { 'ftp.banner': port_data, } elif f_number == '22': f_banner = port_data f_name = 'SSH' addl_fields = { 'ssh.banner': port_data, } elif f_number == '23': f_banner = port_data f_name = 'Telnet' elif f_number == '25': f_banner = "\n".join(self.SMTP_FTP_220.findall(port_data)) f_name = 'SMTP' addl_fields = { 'smtp.banner': port_data, } elif f_number in HTTP_PORTS: # TODO: parse HTTP headers.. ugly f_banner = port_data f_name = 'HTTP' addl_fields = { 'http.banner': port_data, } elif f_number == '1900': f_banner = port_data f_name = 'UPNP' addl_fields = { 'upnp.banner': port_data, } else: f_banner = port_data query = (db.t_services.f_proto == f_proto) & ( db.t_services.f_number == f_number) & (db.t_services.f_hosts_id == host_id) svc_row = db(query).select().first() if svc_row: # we found a service record! Check for similar status, names and banners do_update = False if svc_row.f_status != f_status: svc_row.f_status = f_status do_update = True if svc_row.f_name != f_name: svc_row.f_name = f_name do_update = True if svc_row.f_banner != f_banner: svc_row.f_banner = f_banner do_update = True svc_id = svc_row.id if do_update: svc_row.update_record() db.commit() didwhat = "Updated" self.stats['services_updated'] += 1 else: didwhat = "Unaltered" else: # we have a new service! svc_id = db.t_services.insert(f_proto=f_proto, f_number=f_number, f_status=f_status, f_name=f_name, f_banner=f_banner, f_hosts_id=host_id) db.commit() didwhat = "Added" self.stats['services_added'] += 1 log(" [-] %s service: (%s) %s/%s" % (didwhat, ipaddr, f_proto, f_number)) for k, v in addl_fields.iteritems(): # add additional field entries as service_info records db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=k, f_text=v, ) db.commit()
def detail(): if request.args(0) is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.files.append(URL(request.application,'static','js/jquery.sparkline.js')) response.files.append(URL(request.application,'static','jstree/jstree.min.js')) #query = db.t_hosts.id == request.args(0) #query = create_hostfilter_query(session.hostfilter, query) record = get_host_record(request.args(0)) if record is None: redirect(URL('hosts', 'list')) hostipv4=record.f_ipv4 engineername = db.auth_user[record.f_engineer].username # to allow updating of the host record from this page host=crud.read(db.t_hosts,record) host.attributes['_id'] = "host_record" host_points = {} # build the host_points field which will cover: # the top t_host_os_ref cpe string os_list = db(db.t_host_os_refs.f_hosts_id == record.id).select() host_points['os'] = (0, 'Unknown') for os_rec in os_list: if os_rec.f_certainty > host_points['os'][0]: host_points['os'] = (os_rec.f_certainty, db.t_os[os_rec.f_os_id].f_title) host_points['account_cnt'] = 0 host_points['password_cnt'] = 0 host_points['cracked_pct'] = 0 host_points['vuln_cnt'] = 0 host_points['vuln_exploited_cnt'] = 0 host_points['vuln_potential_cnt'] = 0 vulns = {} vuln_list = [] services = db(db.t_services.f_hosts_id == record.id).select() for svc in services: for vuln in db(db.t_service_vulns.f_services_id == svc.id).select(): vulndata = db.t_vulndata[vuln.f_vulndata_id] vulns[vulndata.f_vulnid] = ( vulndata.f_severity, vulndata.f_cvss_score ) vuln_list.append(vulndata) host_points['vuln_exploited_cnt'] += db((db.t_service_vulns.f_services_id==svc.id) & (db.t_service_vulns.f_status.like('%exploited%'))).count() host_points['vuln_potential_cnt'] += db((db.t_service_vulns.f_services_id==svc.id) & (db.t_service_vulns.f_status.like('%potential%'))).count() host_points['vuln_cnt'] += db(db.t_service_vulns.f_services_id==svc.id).count() host_points['account_cnt'] += db(db.t_accounts.f_services_id==svc.id).count() pwq = ((db.t_accounts.f_services_id==svc.id) & (db.t_accounts.f_compromised == True)) #pwq &= (((db.t_accounts.f_password != None) | (db.t_accounts.f_password != '')) | (db.t_accounts.f_compromised == True)) host_points['password_cnt'] += db(pwq).count() try: host_points['cracked_pct'] = 100 * (host_points['password_cnt'] / host_points['account_cnt']) except ZeroDivisionError: host_points['cracked_pct'] = 0 # breakdown of vuln severity sev_sum_dict = {} for a in range(1, 11): sev_sum_dict[a] = 0 for k,v in vulns.iteritems(): # take the severity and increment the sev_sum set item if settings.use_cvss: severity = int(float(v[1])) else: severity = v[0] count = sev_sum_dict.setdefault(severity, 1) count += 1 sev_sum_dict[severity] = count sev_sum_spark = [] sev_sum = [] for k,v in sev_sum_dict.iteritems(): sev_sum_spark.append(str(v)) if v > 0: sev_sum.append("%s: %s" % (k, v)) host_points['sev_sum_spark'] = ",".join(sev_sum_spark) host_points['sev_sum'] = " / ".join(sev_sum) # netbios record (or none if it's empty) netb_record = db(db.t_netbios.f_hosts_id == record.id).select().first() or None if netb_record is not None: netbios=crud.update(db.t_netbios, netb_record, ondelete=lambda netbios: redirect(URL('host_detail', args=[ record.id ]))) host_points['netb_domain'] = netb_record.f_domain host_points['netb_type'] = netb_record.f_type else: db.t_netbios.f_hosts_id.default = record.id netbios = LOAD('netbios', 'add.load', args=[host.record.id], ajax=True, target='netbios_info') host_pagination = pagination(request, record) response.title = "%s :: Host info :: %s" % (settings.title, host_title_maker(record)) return dict(host=host, netbios=netbios, host_points=host_points, host_pagination=host_pagination, hostipv4=hostipv4, engineername=engineername)
def list(): """ Returns a list of evidence based on a host (id, ipv4, ipv6) or all """ import os, string if request.args(0) is not None: record = get_host_record(request.args(0)) if record is None: redirect( URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Evidence for host %s" % ( settings.title, host_title_maker(record)) else: response.title = "%s :: Evidence listing" % (settings.title) record = None aaData = [] if request.extension == "json": if record is None: rows = db(db.t_evidence).select( db.t_evidence.id, db.t_evidence.f_hosts_id, db.t_evidence.f_type, db.t_evidence.f_other_type, db.t_evidence.f_text, db.t_evidence.f_filename, db.t_evidence.f_evidence, db.t_evidence.f_data.len() + 1) else: rows = db(db.t_evidence.f_hosts_id == record.id).select( db.t_evidence.id, db.t_evidence.f_hosts_id, db.t_evidence.f_type, db.t_evidence.f_other_type, db.t_evidence.f_text, db.t_evidence.f_filename, db.t_evidence.f_evidence, db.t_evidence.f_data.len() + 1) for r in rows: atxt = {} cnt = 0 atxt[cnt] = A('edit', _target="evidence_edit_%s" % (r.t_evidence.id), _href=URL('edit', extension='html', args=r.t_evidence.id)).xml() cnt += 1 if record is None: atxt[cnt] = host_a_maker(r.t_evidence.f_hosts_id).xml() cnt += 1 if r.t_evidence.f_other_type: atxt[cnt] = "Other: %s" % (r.t_evidence.f_other_type) else: atxt[cnt] = r.t_evidence.f_type cnt += 1 atxt[cnt] = r.t_evidence.f_text cnt += 1 if r.t_evidence.f_filename is not None: if string.lower(os.path.splitext( r.t_evidence.f_filename)[1]) in ('.png', '.jpeg', '.jpg', '.gif'): atxt[cnt] = A(IMG(_src=URL('download', args=[r.t_evidence.f_evidence]), _width="50%", _height="20%"), _href=URL('download', args=[r.t_evidence.f_evidence]), _target="evidence_image_%s" % (r.t_evidence.id), _id="evidence_image").xml() cnt += 1 atxt[cnt] = "%sb" % ( r._extra['(LENGTH(t_evidence.f_data) + 1)']) cnt += 1 else: atxt[cnt] = A( r.t_evidence.f_filename, _target="evidence_other_%s" % (r.t_evidence.id), _id="evidence_other", _href=URL('download', args=[r.t_evidence.f_evidence])).xml() cnt += 1 atxt[cnt] = "%sb" % ( r._extra['(LENGTH(t_evidence.f_data) + 1)']) cnt += 1 else: atxt[cnt] = r.t_evidence.f_filename cnt += 1 atxt['DT_RowId'] = r.t_evidence.id aaData.append(atxt) return { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } if record: th_rows = ( TH(T(''), _width="5%"), TH(T('Type')), TH(T('Text')), TH(T('Evidence')), TH(T('File Size')), ) else: th_rows = ( TH(T(''), _width="5%"), TH(T('Host')), TH(T('Type')), TH(T('Text')), TH(T('Evidence'), _width="35%"), TH(T('File Size')), ) evidence = TABLE(THEAD(TR(th_rows)), _class="datatable", _id="evidencetable", _style="width:100%") return dict(evidence=evidence, host=record)
def launch_terminal(record=None, launch_cmd=None): """ Opens a terminal on the Web Server. This only works if the web2py server is running on the user's workstation. The command to execute is stored in the user's settings db under auth_user.f_launch_cmd. Variables translated: _IP_ -- The current IP Address (v4 by default, v6 if exists) _LOGFILE_ -- Session logfile name (we prepend the path) If an IPv6 address is used then ':' is changed to '_' Example: xterm -sb -sl 1500 -vb -T 'manual hacking: _IP_' -n 'manual hacking: _IP_' -e script _LOGFILE_ """ record = get_host_record(record) # only execute launch on requests from localhost! if request.env['remote_addr'] != '127.0.0.1': logger.error("Can only launch from localhost! remote_addr = %s" % (request.env['remote_addr'])) return "Can only launch from localhost" if record is None: return "No record found" import string, os, subprocess import time from gluon.validators import IS_IPADDRESS # if no launch command use the default if not launch_cmd: launch_cmd = "xterm -sb -sl 1500 -vb -T 'manual hacking: _IP_' -n 'manual hacking: _IP_' -e 'script _LOGFILE_'" # check ip address ip = record.f_ipaddr logip = ip if IS_IPADDRESS(is_ipv6=True)(ip)[0] == None: logip = ip.replace(":", "_") logdir = "session-logs" logfilename = "%s-%s.log" % ( logip, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) logfile = os.path.join(logdir, logfilename) launch_cmd = launch_cmd.replace("_IP_", ip) launch_cmd = launch_cmd.replace("_LOGFILE_", logfile) from skaldship.general import check_datadir # Check to see if data directories exist, create otherwise check_datadir(request.folder) datadir = os.path.join(os.getcwd(), request.folder, "data") # chdir to datadir! launch_cmd = launch_cmd.replace("_DATADIR_", datadir) os.chdir(datadir) # set environment variables os.environ['IP'] = ip os.environ['HOSTNAME'] = record.f_hostname or "" os.environ['DATADIR'] = datadir try: logger.info("Spawning: %s\n" % (launch_cmd)) print("Spawning: %s" % (launch_cmd)) subprocess.Popen(launch_cmd, shell=True) #, stdout=None, stdin=None, stderr=None) except Exception, e: logger.error("Error spawning launch cmd (%s): %s\n" % (launch_cmd, e)) print("Error spawning launch cmd (%s): %s\n" % (launch_cmd, e))
ip_dict[ip] = ip_accts # run through the ip_accts now to add/update them to the database from skaldship.hosts import get_host_record for k, v in ip_dict.iteritems(): for ip_acct in v: # build a query to find the service for this host/port combo query = (db.t_hosts.f_ipv4 == k) & (db.t_services.f_hosts_id == db.t_hosts.id) query &= (db.t_services.f_proto == ip_acct['f_proto']) & ( db.t_services.f_number == ip_acct['f_number']) svc = db(query).select(db.t_services.id, cache=(cache.ram, 60)).first() if svc is None: # no service found, get the host record based on the IP host_rec = get_host_record(k) if host_rec is None and add_hosts: # add host to the database, unfortunately all we know is the IP address so it's pretty bare. # assign it to the current user and asset group of "new_hosts_medusa" fields = { 'f_ipv4': k, 'f_engineer': user_id, 'f_asset_group': 'new_hosts_medusa', } host_rec = db.t_hosts.insert(**fields) db.commit() logger.info("Added new host from Medusa output: %s" % (k)) new_hosts += 1 elif host_rec is None: # no host and not asking to add hosts so print message and continue logger.error("Unable to find host_rec for %s" % (k))
def list(): """ Returns a list of evidence based on a host (id, ipv4, ipv6) or all """ import os, string if request.args(0) is not None: record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Evidence for host %s" % (settings.title, host_title_maker(record)) else: response.title = "%s :: Evidence listing" % (settings.title) record = None aaData = [] if request.extension == "json": if record is None: rows = db(db.t_evidence).select(db.t_evidence.id, db.t_evidence.f_hosts_id, db.t_evidence.f_type, db.t_evidence.f_other_type, db.t_evidence.f_text, db.t_evidence.f_filename, db.t_evidence.f_evidence, db.t_evidence.f_data.len()+1) else: rows = db(db.t_evidence.f_hosts_id == record.id).select(db.t_evidence.id, db.t_evidence.f_hosts_id, db.t_evidence.f_type, db.t_evidence.f_other_type, db.t_evidence.f_text, db.t_evidence.f_filename, db.t_evidence.f_evidence, db.t_evidence.f_data.len()+1) for r in rows: atxt = {} cnt = 0 atxt[cnt] = A('edit', _target="evidence_edit_%s" % (r.t_evidence.id), _href=URL('edit', extension='html', args=r.t_evidence.id)).xml() cnt += 1 if record is None: atxt[cnt] = host_a_maker(r.t_evidence.f_hosts_id).xml() cnt += 1 if r.t_evidence.f_other_type: atxt[cnt] = "Other: %s" % (r.t_evidence.f_other_type) else: atxt[cnt] = r.t_evidence.f_type cnt += 1 atxt[cnt] = r.t_evidence.f_text cnt += 1 if r.t_evidence.f_filename is not None: if string.lower(os.path.splitext(r.t_evidence.f_filename)[1]) in ('.png', '.jpeg', '.jpg', '.gif'): atxt[cnt] = A(IMG(_src=URL('download', args=[r.t_evidence.f_evidence]), _width="50%", _height="20%"), _href=URL('download', args=[r.t_evidence.f_evidence]), _target="evidence_image_%s" % (r.t_evidence.id), _id="evidence_image").xml() cnt += 1 atxt[cnt] = "%sb" % (r._extra['(LENGTH(t_evidence.f_data) + 1)']) cnt += 1 else: atxt[cnt] = A(r.t_evidence.f_filename, _target="evidence_other_%s" % (r.t_evidence.id), _id="evidence_other", _href=URL('download', args=[r.t_evidence.f_evidence])).xml() cnt += 1 atxt[cnt] = "%sb" % (r._extra['(LENGTH(t_evidence.f_data) + 1)']) cnt += 1 else: atxt[cnt] = r.t_evidence.f_filename cnt += 1 atxt['DT_RowId'] = r.t_evidence.id aaData.append(atxt) return { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } if record: th_rows = (TH(T(''), _width="5%"), TH(T('Type')), TH(T('Text')), TH(T('Evidence')), TH(T('File Size')), ) else: th_rows = (TH(T(''), _width="5%"), TH(T('Host')), TH(T('Type')), TH(T('Text')), TH(T('Evidence'), _width="35%"), TH(T('File Size')), ) evidence = TABLE(THEAD(TR(th_rows)), _class="datatable", _id="evidencetable", _style="width:100%") return dict(evidence=evidence, host=record)
except AttributeError, e: log("Invalid port sent: %s", port, logging.ERROR) try: pw_data = open(filename, "rb").readlines().split('\n') except IOError, e: log("Error opening %s: %s" % (filename, e), logging.ERROR) accounts = process_password_file( pw_data=pw_data, file_type=file_type, source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == number) query &= (db.t_services.f_proto == proto) query &= (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto=proto, f_number=number, f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added loot accounts for host: %s" % host.f_ipaddr)
def list_lootfiles(): """ Lists local loot files for import processing into Kvasir. This does not use the Metasploit API and depends upon a directory being local to the web2py server instance. The API is used to check if pro is installed and sets the loot_dir to Linux or Windows path """ import os import re response.title = "%s :: Metasploit Loots" % (settings.title) msf_settings = msf_get_config(session) dbsvcs = db.t_services loot_dir = request.args(0) if not loot_dir: try: from MetasploitProAPI import MetasploitProAPI, MSFProAPIError msf = MetasploitProAPI(host=msf_settings['url'], apikey=msf_settings['key']) if msf.pro_about(): if platform in ["linux", "linux2"]: loot_dir = "/opt/metasploit_pro/apps/pro/loot" else: loot_dir = "C:\\Metasploit\\apps\\pro\\loot" except ImportError as error: pass if not loot_dir: from sys import platform if platform in ["linux", "linux2", "darwin", "freebsd"]: loot_dir = os.path.join(os.environ.get('HOME'), '.msf4/loot') elif platform in ["win32", "cygwin"]: loot_dir = '$FINDYOUR/msf4/loot/path' try: os.chdir(loot_dir) loot_files = os.listdir(loot_dir) except OSError: loot_files = [] loot_file_details = [] for loot in loot_files: try: (timestamp, workspace, ipaddr, filetype, extension) = re.split('_', loot) except ValueError: logging.warn("Invalid loot file: %s" % (loot)) continue # TODO: service_list = get_services(ipaddr) host_rec = get_host_record(ipaddr) services = [] for service in db(dbsvcs.f_hosts_id == host_rec).select( dbsvcs.id, dbsvcs.f_proto, dbsvcs.f_number, cache=(cache.ram, 120)): services.append( [service.id, "%s/%s" % (service.f_proto, service.f_number)]) loot_file_details.append([workspace, ipaddr, services, filetype]) form_lootdir = SQLFORM.factory( Field('lootdir', 'string', default=loot_dir, requires=IS_NOT_EMPTY(), label=T('Metasploit Loot Directory')), ) return dict(form_lootdir=form_lootdir, loot_file_details=loot_file_details)
def parse(self, host_properties): """ Parse out the <HostProperties> xml content or CSV line. There can be a number of <tag> entries that are either useful to us in t_hosts or other areas. These are processed and returned as dictionary entries in 'hostdata' Args: host_properties: A <HostProperties> section from .nessus or a CSV line Returns: t_hosts.id, { hostdata } """ from gluon.validators import IS_IPADDRESS hostdata = {} if etree.iselement(host_properties): for tag in host_properties.findall('tag'): hostdata[tag.get('name')] = tag.text ipaddr = hostdata.get('host-ip') else: # with CSV each line has all the hostdata fields so we set them here for use later ipaddr = host_properties.get('IP Address') if not ipaddr: # Scanner CSV, use Host ipaddr = host_properties.get('Host') hostdata['mac-address'] = host_properties.get('MAC Address', '') hostdata['host-fqdn'] = host_properties.get('DNS Name', '') hostdata['netbios-name'] = host_properties.get('NetBIOS Name', '') if (ipaddr not in self.ip_include and self.ip_include) or (ipaddr in self.ip_exclude): log("Host in exclude or not in include list, skipping") self.stats['skipped'] += 1 return None, {} host_id = get_host_record(ipaddr) if host_id and not self.update_hosts: return host_id, hostdata # new host found, pull what we need for t_hosts hostfields = {} hostfields['f_engineer'] = self.engineer hostfields['f_asset_group'] = self.asset_group hostfields['f_confirmed'] = False # check ipv4/ipv6 and set hostfields accordingly if IS_IPADDRESS(is_ipv4=True)(ipaddr)[1] is None: hostfields['f_ipv4'] = ipaddr elif IS_IPADDRESS(is_ipv6=True)(ipaddr)[1] is None: hostfields['f_ipv6'] = ipaddr else: log("Invalid IP Address in HostProperties: %s" % ipaddr, logging.ERROR) return None, {} # pull out relevant hostfields for (k, v) in hostdata.iteritems(): if k == 'mac-address': # multiple mac addrs may appear wildly, just pull the first hostfields['f_macaddr'] = v[:v.find('\n')] elif k == 'host-fqdn': hostfields['f_hostname'] = v elif k == 'netbios-name': hostfields['f_netbios_name'] = v if not self.update_hosts and not host_id: result = self.db.t_hosts.validate_and_insert(**hostfields) if not result.id: log("Error adding host to DB: %s" % result.errors, logging.ERROR) return None, {} self.stats['added'] += 1 host_id = result.id log(" [-] Adding host: %s" % ipaddr) elif self.update_hosts: if hostfields['f_ipv4']: host_id = self.db( self.db.t_hosts.f_ipv4 == hostfields['f_ipv4']).update( **hostfields) self.db.commit() host_id = get_host_record(hostfields['f_ipv4']) if host_id: host_id = host_id.id log(" [-] Updating IP: %s" % (hostfields['f_ipv4'])) else: host_id = self.db( self.db.t_hosts.f_ipv6 == hostfields['f_ipv6']).update( **hostfields) self.db.commit() host_id = get_host_record(hostfields['f_ipv6']) host_id = host_id.id log(" [-] Updating IP: %s" % (hostfields['f_ipv6'])) self.stats['updated'] += 1 return host_id, hostfields
def process_xml( filename=None, addnoports=False, asset_group=None, engineer=None, msf_workspace=False, ip_ignore_list=None, ip_include_list=None, update_hosts=False, ): # Upload and process Qualys XML Scan file import os, time, re, html.parser from io import StringIO from MetasploitProAPI import MetasploitProAPI from skaldship.hosts import html_to_markmin, get_host_record, do_host_status from skaldship.cpe import lookup_cpe parser = html.parser.HTMLParser() # output regexes RE_NETBIOS_NAME = re.compile('NetBIOS name: (?P<d>.*),') RE_NETBIOS_MAC = re.compile( 'NetBIOS MAC: (?P<d>([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2}))') RE_IPV4 = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$') if msf_workspace: msf = MetasploitProAPI(host=user_id.f_msf_pro_url, apikey=user_id.f_msf_pro_key) if msf.login(): logger.info(" [-] Authenticated to Metasploit PRO") else: logger.error( " [!] Unable to login to Metasploit PRO, check your API key") msf = None else: logger.warn(" [-] No Metasploit workspace provided!") msf = None try: from lxml import etree except ImportError: try: import xml.etree.cElementTree as etree except ImportError: try: import xml.etree.ElementTree as etree except: raise Exception("Unable to find valid ElementTree module.") # build the hosts only/exclude list ip_exclude = [] if ip_ignore_list: ip_exclude = ip_ignore_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals ip_only = [] if ip_include_list: ip_only = ip_include_list.split('\r\n') # TODO: check for ip subnet/range and break it out to individuals print((" [*] Processing Qualys scan file %s" % (filename))) try: nmap_xml = etree.parse(filename) except etree.ParseError as e: print((" [!] Invalid XML file (%s): %s " % (filename, e))) return root = nmap_xml.getroot() db = current.globalenv['db'] cache = current.globalenv['cache'] existing_vulnids = db(db.t_vulndata()).select( db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid') #print(" [*] Found %d vulnerabilities in the database already." % (len(existing_vulnids))) # check for any CPE OS data if db(db.t_cpe_os).count() > 0: have_cpe = True else: have_cpe = False user_id = db.auth_user(engineer) or auth.user.id # parse the hosts, where all the goodies are nodes = root.findall('IP') print((" [-] Parsing %d hosts" % (len(nodes)))) hoststats = {} hoststats['added'] = 0 hoststats['skipped'] = 0 hoststats['updated'] = 0 hoststats['errored'] = 0 hosts = [] # array of host_id fields vulns_added = 0 vulns_skipped = 0 for node in nodes: nodefields = {} ipaddr = node.get('value') nodefields['f_ipaddr'] = ipaddr nodefields['f_hostname'] = node.get('hostname') nodefields['f_netbios_name'] = node.findtext('NETBIOS_HOSTNAME') # nodefields['f_macaddr'] = address.get('addr') """ status = node.find('status').get('state') print(" [-] Host %s status is: %s" % (ipaddr, status)) if status != "up": hoststats['skipped'] += 1 continue """ if ipaddr in ip_exclude: print(" [-] Host is in exclude list... skipping") hoststats['skipped'] += 1 continue if len(ip_only) > 0 and ipaddr not in ip_only: print(" [-] Host is not in the only list... skipping") hoststats['skipped'] += 1 continue ports = node.findall('INFOS') if len(ports) < 1 and not addnoports: print( " [-] No ports open and not asked to add those kind... skipping" ) hoststats['skipped'] += 1 continue nodefields['f_engineer'] = user_id nodefields['f_asset_group'] = asset_group nodefields['f_confirmed'] = False # check to see if IPv4/IPv6 exists in DB already if 'f_ipaddr' in nodefields: host_rec = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).select().first() else: logging.warn("No IP Address found in record. Skipping") continue if host_rec is None: host_id = db.t_hosts.insert(**nodefields) db.commit() hoststats['added'] += 1 print((" [-] Adding %s" % (ipaddr))) elif host_rec is not None and update_hosts: db.commit() host_id = db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update( **nodefields) db.commit() host_id = get_host_record(ipaddr) host_id = host_id.id hoststats['updated'] += 1 print((" [-] Updating %s" % (ipaddr))) else: hoststats['skipped'] += 1 db.commit() print((" [-] Skipped %s" % (ipaddr))) continue hosts.append(host_id) # : for hostscripts in node.findall('hostscript/script'): svc_id = db.t_services.update_or_insert(f_proto='info', f_number=0, f_status='open', f_hosts_id=host_id) db.commit() for script in hostscripts: script_id = script.get('id') output = script.get('output') svc_info = db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=script_id, f_text=output) db.commit() # add ports and resulting vulndata for port in node.findall("ports/port"): f_proto = port.get('protocol') f_number = port.get('portid') f_status = port.find('state').get('state') port_svc = port.find('service') if port_svc: f_name = port_svc.get('name') f_product = port_svc.get('product') svc_fp = port_svc.get('servicefp') else: f_name = None f_product = None svc_fp = None print( (" [-] Adding port: %s/%s (%s)" % (f_proto, f_number, f_name))) svc_id = db.t_services.update_or_insert(f_proto=f_proto, f_number=f_number, f_status=f_status, f_hosts_id=host_id, f_name=f_name) if f_product: version = port.find('service').get('version', None) if version: f_product += " (%s)" % (version) svc_info = db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=f_name, f_text=f_product) db.commit() if svc_fp: svc_info = db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=svc_fp, f_text=svc_fp) db.commit() # Process <script> service entries for script in port.findall('service/script'): svc_info = db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=script.get('id'), f_text=script.get('output')) db.commit() # Process <cpe> service entries for port_cpe in port.findall('service/cpe'): cpe_id = port_cpe.text.replace('cpe:/', '') if cpe_id[0] == "a": # process CPE Applications print((" [-] Found Application CPE data: %s" % (cpe_id))) svc_info = db.t_service_info.update_or_insert( f_services_id=svc_id, f_name='CPE ID', f_text="cpe:/%s" % (cpe_id)) db.commit() elif cpe_id[0] == "o": # process CPE Operating System os_id = lookup_cpe(cpe_id[2:]) if os_id is not None: db.t_host_os_refs.insert(f_certainty='0.9', f_family='Unknown', f_class='Other', f_hosts_id=host_id, f_os_id=os_id) db.commit() else: # So no CPE or existing OS data, lets split up the CPE data and make our own print(" [!] No os_id found, this is odd !!!") for config in port.findall("configuration/config"): cfg_id = db.t_service_info.update_or_insert( f_services_id=svc_id, f_name=config.attrib['name'], f_text=config.text) db.commit() if re.match('\w+.banner$', config.attrib['name']): db.t_services[svc_id] = dict(f_banner=config.text) db.commit() if config.attrib['name'] == 'mac-address': # update the mac address of the host db.t_hosts[host_id] = dict(f_macaddr=config.text) db.commit() if "advertised-name" in config.attrib['name']: # netbios computer name d = config.text.split(" ")[0] if "Computer Name" in config.text: data = {} data['f_netbios_name'] = d # if hostname isn't defined then lowercase netbios name and put it in if db.t_hosts[host_id].f_hostname is None: data['f_hostname'] = d.lower() db(db.t_hosts.id == host_id).update(**data) elif "Domain Name" in config.text: db(db.t_netbios.f_hosts_id == host_id).update( f_domain=d) or db.t_netbios.insert( f_hosts_id=host_id, f_domain=d) db.commit() for script in port.findall("script"): # process <script> results. This data contains both info # and vulnerability data. For now we'll take a list of # known nmap vuln checks from private/nmap_vulns.csv and # use that to separate between service_info and vulndata. pass if msf is not None: # send the downloaded nexpose file to MSF for importing try: res = msf.pro_import_file( msf_workspace, filename, { 'DS_REMOVE_FILE': False, 'tag': asset_group, }, ) print((" [*] Added file to MSF Pro: %s" % (res))) except MSFAPIError as e: logging.error("MSFAPI Error: %s" % (e)) pass # any new nexpose vulns need to be checked against exploits table and connected print(" [*] Connecting exploits to vulns and performing do_host_status") do_host_status(asset_group=asset_group) print(( " [*] Import complete: hosts: %s added, %s skipped, %s errors - vulns: %s added, %s skipped" % (hoststats['added'], hoststats['skipped'], hoststats['errored'], vulns_added, vulns_skipped)))
def by_host(): """ Returns a list of services + serviceinfo based upon an host identifier (id, ipaddr) """ record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Accounts for %s" % (settings.title, host_title_maker(record)) query = (db.t_services.f_hosts_id == record.id) if request.extension == "json": aaData = [] rows = db(query).select(db.t_accounts.ALL, db.t_services.ALL, left=db.t_services.on(db.t_accounts.f_services_id==db.t_services.id)) for r in rows: if r.t_accounts.f_compromised == True: comprdiv = '<div class="acct_compromised" name="row_id" id="%s"><span class="icon-check"></span></div>' % r.t_accounts.id else: comprdiv = '<div class="acct_uncompromised" name="row_id" id="%s"/>' % r.t_accounts.id aaData.append({ '0': A("edit", _target="accounts_update_%s" % (r.t_accounts.id), _href=URL('accounts', 'edit', args=[r.t_accounts.id], extension='html')).xml(), '1': comprdiv, '2': A("%s/%s" % (r.t_services.f_proto, r.t_services.f_number), _target="services_edit_%s" % (r.t_services.id), _href=URL('services', 'edit', args=[r.t_services.id], extension='html')).xml(), '3': A(r.t_accounts.f_username, _target="accounts_username_%s" % (r.t_accounts.f_username), _href=URL('accounts', 'by_username', vars={'username': r.t_accounts.f_username}, extension='html')).xml(), '4': r.t_accounts.f_fullname, '5': r.t_accounts.f_password, '6': r.t_accounts.f_hash1_type, '7': r.t_accounts.f_hash1, '8': r.t_accounts.f_hash2_type, '9': r.t_accounts.f_hash2, '10': r.t_accounts.f_uid, '11': r.t_accounts.f_gid, '12': r.t_accounts.f_lockout, '13': r.t_accounts.f_duration, '14': r.t_accounts.f_source, '15': r.t_accounts.f_level, '16': r.t_accounts.f_description, 'DT_RowId': r.t_accounts.id, } ) result = { 'sEcho': request.vars.sEcho, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result form = TABLE(THEAD(TR(TH(T(''), _width="5%"), TH(T('Compr'), _width="5%"), TH(T('Port')), TH(T('Username')), TH(T('Fullname')), TH(T('Password')), TH(T('Hash 1 Type')), TH(T('Hash 1')), TH(T('Hash 2 Type')), TH(T('Hash 2')), TH(T('UID')), TH(T('GID')), TH(T('Lockout')), TH(T('Duration')), TH(T('Source')), TH(T('Level')), TH(T('Description')), ) ), _class="datatable", _id="accounttable", _style="width:100%") add = AddModal( db.t_accounts, 'Add', 'Add', 'Add Account', #fields=[], cmd='accounttable.fnReloadAjax();' ) services = db(db.t_services.f_hosts_id == record.id).select(cache=(cache.ram,30)) svc_set = [] for svc in services: svc_set.append([svc.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[svc.f_hosts_id]), svc.f_proto, svc.f_number)]) db.t_accounts.f_services_id.requires = IS_IN_SET(svc_set) db.t_accounts.id.comment = add.create() return dict(form=form, host=record, add=add)
'f_message': mass_pw_data.get('msg'), 'f_source': message, 'f_compromised': True }) ip_dict[ip] = ip_accts # run through the ip_accts now to add/update them to the database from skaldship.hosts import get_host_record for k,v in ip_dict.iteritems(): for ip_acct in v: # build a query to find the service for this host/port combo query = (db.t_hosts.f_ipv4 == k) & (db.t_services.f_hosts_id == db.t_hosts.id) query &= (db.t_services.f_proto==ip_acct['f_proto']) & (db.t_services.f_number==ip_acct['f_number']) svc = db(query).select(db.t_services.id, cache=(cache.ram, 60)).first() if svc is None: # no service found, get the host record based on the IP host_rec = get_host_record(k) if host_rec is None and add_hosts: # add host to the database, unfortunately all we know is the IP address so it's pretty bare. # assign it to the current user and asset group of "new_hosts_medusa" fields = { 'f_ipv4': k, 'f_engineer': user_id, 'f_asset_group': 'new_hosts_medusa', } host_rec = db.t_hosts.insert(**fields) db.commit() logger.info("Added new host from Medusa output: %s" % (k)) new_hosts += 1 elif host_rec is None: # no host and not asking to add hosts so print message and continue logger.error("Unable to find host_rec for %s" % (k))
def paste(): """ Import and parse password pasted to a textbox into t_accounts """ from skaldship.general import check_datadir check_datadir(request.folder) # Service_id is primary, host_id is secondary, if none then list # all the services svc_set = [] url=URL('accounts', 'paste') if request.vars.has_key('service_id'): try: record = db.t_services[request.vars.service_id] svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) url = URL('accounts', 'paste', vars={'service_id':request.vars.service_id}) except: pass elif request.vars.has_key('host_id'): try: host_record = get_host_record(request.vars.host_id) svc_records = db(db.t_services.f_hosts_id == host_record.id).select(cache=(cache.ram, 30)) url = URL('accounts', 'paste', vars={'host_id':request.vars.host_id}) for record in svc_records: svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) except: pass if len(svc_set) == 0: # all services svc_records = db(db.t_services).select(cache=(cache.ram,30)) svc_set = [] for record in svc_records: svc_set.append((record.id, "%s :: %s/%s" % (host_title_maker(db.t_hosts[record.f_hosts_id]), record.f_proto, record.f_number))) if request.extension == "load": buttons=[] else: buttons=['submit'] form = SQLFORM.factory( Field('f_service', 'string', label=T('Host / Service'), requires=IS_IN_SET(svc_set), default=svc_set[0][0]), Field('f_pwtext', 'text', label=T('Password text')), Field('f_type', 'string', label=T('File type'), default='PWDUMP', requires=IS_IN_SET(settings.password_file_types)), Field('f_source', 'string', label=T('Source (if necessary)')), Field('f_add_to_evidence', 'boolean', label=T('Add file to Evidence')), buttons=buttons, _action=url, _id='accounts_paste_form' #_action=url, _id='accounts_paste_form', formstyle='bootstrap_modal' ) resp_text = "" accounts_added = [] accounts_updated = [] if form.errors: response.flash = 'Error in form' return TABLE(*[TR(k, v) for k, v in form.errors.items()]) elif form.accepts(request.vars, session): from gluon.utils import web2py_uuid host_id = db.t_services[form.vars.f_service].f_hosts_id pwd_file_dir = os.path.join(request.folder, 'data', 'passwords', 'other') if not os.path.exists(pwd_file_dir): from gluon.fileutils import mktree mktree(pwd_file_dir) filename = "%s-pwfile-%s" % (host_id, web2py_uuid()) full_file_path = os.path.join(request.folder, 'data/passwords/other', filename) of = open(full_file_path, "w") of.write(form.vars.f_pwtext) of.close() logger.debug("Processing password file: %s" % (full_file_path)) account_data = process_password_file(pw_file=full_file_path, file_type=request.vars.f_type, source=request.vars.f_source) response.headers['web2py-component-command'] = 'accounttable.fnReloadAjax();' resp_text = insert_or_update_acct(form.vars.f_service, account_data) if form.vars.f_add_to_evidence is True: # add the password file to evidence try: pwdata = open(full_file_path, "r").readlines() except Exception, e: logger.error("Error opening %s: %s" % (full_file_path, e)) resp_text += "Error opening %s: %s\n" % (full_file_path, e) db.t_evidence.insert( f_hosts_id = host_id, f_type = 'Password File', f_text = form.vars.f_type, f_filename = filename, f_evidence = filename, f_data = pwdata) resp_text += "\n%s added to evidence\n" % (filename) db.commit()
def list(): """ Returns a list of evidence based on a host (id, ipv4, ipv6) or all """ import os, string if request.args(0) is not None: record = get_host_record(request.args(0)) if record is None: redirect(URL("default", "error", vars={"msg": T("Host record not found")})) response.title = "%s :: Evidence for host %s" % (settings.title, host_title_maker(record)) else: response.title = "%s :: Evidence listing" % (settings.title) record = None aaData = [] if request.extension == "json": if record is None: rows = db(db.t_evidence).select( db.t_evidence.id, db.t_evidence.f_hosts_id, db.t_evidence.f_type, db.t_evidence.f_other_type, db.t_evidence.f_text, db.t_evidence.f_filename, db.t_evidence.f_evidence, db.t_evidence.f_data.len() + 1, ) else: rows = db(db.t_evidence.f_hosts_id == record.id).select( db.t_evidence.id, db.t_evidence.f_hosts_id, db.t_evidence.f_type, db.t_evidence.f_other_type, db.t_evidence.f_text, db.t_evidence.f_filename, db.t_evidence.f_evidence, db.t_evidence.f_data.len() + 1, ) for r in rows: atxt = {} cnt = 0 atxt[cnt] = A( "edit", _target="evidence_edit_%s" % (r.t_evidence.id), _href=URL("edit", extension="html", args=r.t_evidence.id), ).xml() cnt += 1 if record is None: atxt[cnt] = host_a_maker(r.t_evidence.f_hosts_id).xml() cnt += 1 if r.t_evidence.f_other_type: atxt[cnt] = "Other: %s" % (r.t_evidence.f_other_type) else: atxt[cnt] = r.t_evidence.f_type cnt += 1 atxt[cnt] = r.t_evidence.f_text cnt += 1 if r.t_evidence.f_filename is not None: if string.lower(os.path.splitext(r.t_evidence.f_filename)[1]) in (".png", ".jpeg", ".jpg", ".gif"): atxt[cnt] = A( IMG(_src=URL("download", args=[r.t_evidence.f_evidence]), _width="50%", _height="20%"), _href=URL("download", args=[r.t_evidence.f_evidence]), _target="evidence_image_%s" % (r.t_evidence.id), _id="evidence_image", ).xml() cnt += 1 atxt[cnt] = "%sb" % (r._extra["(LENGTH(t_evidence.f_data) + 1)"]) cnt += 1 else: atxt[cnt] = A( r.t_evidence.f_filename, _target="evidence_other_%s" % (r.t_evidence.id), _id="evidence_other", _href=URL("download", args=[r.t_evidence.f_evidence]), ).xml() cnt += 1 atxt[cnt] = "%sb" % (r._extra["(LENGTH(t_evidence.f_data) + 1)"]) cnt += 1 else: atxt[cnt] = r.t_evidence.f_filename cnt += 1 atxt["DT_RowId"] = r.t_evidence.id aaData.append(atxt) return {"sEcho": request.vars.sEcho, "iTotalRecords": len(aaData), "aaData": aaData} if record: th_rows = (TH(T(""), _width="5%"), TH(T("Type")), TH(T("Text")), TH(T("Evidence")), TH(T("File Size"))) else: th_rows = ( TH(T(""), _width="5%"), TH(T("Host")), TH(T("Type")), TH(T("Text")), TH(T("Evidence"), _width="35%"), TH(T("File Size")), ) evidence = TABLE(THEAD(TR(th_rows)), _class="datatable", _id="evidencetable", _style="width:100%") return dict(evidence=evidence, host=record)
def by_host(): """ Returns a list of services + serviceinfo based upon an host identifier (id, ipv4, ipv6) """ record = get_host_record(request.args(0)) if record is None: redirect(URL('default', 'error', vars={'msg': T('Host record not found')})) response.title = "%s :: Services for %s" % (settings.title, host_title_maker(record)) services = db(db.t_services.f_hosts_id==record.id).select(db.t_services.id, db.t_services.f_proto, db.t_services.f_number, db.t_services.f_status, db.t_services.f_name, db.t_services.f_banner)#, cache=(cache.ram,60)) svcq = (db.t_services.f_hosts_id==record.id) infoq = (db.t_service_info.f_services_id == db.t_services.id) if request.extension == "json": #rows = db(svcq).select(db.t_services.ALL, db.t_service_info.ALL, left=db.t_service_info.on(infoq)) aaData = [] for svc in services: # service info atxt = {} q = db(db.t_service_info.f_services_id == svc.id).select() if len(q) > 0: addl = [] for svcinfo in q: addl.append(TR(TD(svcinfo.f_name), TD(svcinfo.f_text))) atxt['0'] = IMG(_src=URL(request.application,'static','images/details_open.png')).xml() atxt['1'] = TABLE(THEAD(TR(TH(T('Name')), TH(T('Text')))), TBODY(addl)).xml() else: atxt['0'] = ("") atxt['1'] = ("") atxt['2'] = A('edit', _target="services_edit_%s" % (svc.id), _href=URL('edit', args=[svc['id']], extension='html')).xml() atxt['3'] = svc.f_proto if svc.f_number in HTTP_PORTS and svc.f_proto == "tcp" or svc.f_name == "HTTP": atxt['4'] = A(svc.f_number, _href="http://%s:%s/" % (record.f_ipaddr, svc.f_number), _target="%s-tcp-%s" % (record.f_ipaddr, svc.f_number)).xml() elif svc.f_number in HTTPS_PORTS and svc.f_proto == "tcp" or svc.f_name == "HTTPS": atxt['4'] = A(svc.f_number, _href="https://%s:%s/" % (record.f_ipaddr, svc.f_number), _target="%s-tcp-%s" % (record.f_ipaddr, svc.f_number)).xml() else: atxt['4'] = svc.f_number atxt['5'] = svc.f_status atxt['6'] = svc.f_name or "" atxt['7'] = svc.f_banner or "" atxt['DT_RowId'] = svc.id aaData.append(atxt) result = { 'sEcho': request.vars._, 'iTotalRecords': len(aaData), 'aaData': aaData, } return result add = AddModal( db.t_services, 'Add', 'Add', 'Add Service', fields=[ 'f_proto', 'f_number', 'f_status', 'f_name', 'f_banner' ], cmd='servicetable.fnReloadAjax();' ) db.t_services.f_hosts_id.default = record.id db.t_services.id.comment = add.create() form = TABLE(THEAD(TR(TH('', _width="5%"), TH('Info'), TH(T('')), TH(T('Protocol')), TH(T('Number')), TH(T('Status')), TH(T('Name')), TH(T('Banner')), ) ), _class="datatable", _id="servicetable", _style="width:100%") return dict(form=form, host=record, add=add)
except AttributeError, e: log("Invalid port sent: %s", port, logging.ERROR) try: pw_data = open(filename, "rb").readlines().split('\n') except IOError, e: log("Error opening %s: %s" % (filename, e), logging.ERROR) accounts = process_password_file( pw_data=pw_data, file_type=file_type, source='Metasploit', ) # find the info/0 service id for the host host = get_host_record(loot['host']) query = (db.t_services.f_number == number) query &= (db.t_services.f_proto == proto) query &= (db.t_services.f_hosts_id == host.id) svc_id = db(query).select().first() if svc_id is None: # info/0 not found.. add it! svc_id = db.t_services.insert(f_proto=proto, f_number=number, f_hosts_id=host.id) db.commit() # insert or update the account records resp_text = insert_or_update_acct(svc_id.id, accounts) log("Added loot accounts for host: %s" % host.f_ipaddr) data.append({loot['host']: resp_text})