def process_msfcsv(line):
"""
Process a metasploit creds output csv file and returns a dictionary. Looks up hash values if provided.
:param line: Line from a metasploit creds csv file
:return: {'ip': ipaddress, 'port': port, 'user': username, 'pass': password, 'hash': ntlm_hash, 'msg': message}
>>> process_msfcsv()
"""
# host,port,user,pass,type,active?
retval = {}
hash_types = ['smb', 'rakp_hmac_sha1_hash', 'smb_challenge']
import csv
for data in csv.reader([line]):
retval['ip'] = data[0]
retval['port'] = data[1]
retval['user'] = data[2]
retval['pass'] = data[3]
retval['type'] = data[4]
retval['msg'] = 'from metasploit'
# isactive = data[5] # unused
if retval['type'] in hash_types:
retval['hash'] = retval['pass']
retval['pass'] = lookup_hash(retval['hash'])
else:
retval['hash'] = None
retval['error'] = False
return retval
def process_msfcsv(line):
"""
Process a metasploit creds output csv file and returns a dictionary. Looks up hash values if provided.
:param line: Line from a metasploit creds csv file
:return: {'ip': ipaddress, 'port': port, 'user': username, 'pass': password, 'hash': ntlm_hash, 'msg': message}
>>> process_msfcsv()
"""
# host,port,user,pass,type,active?
retval = {}
hash_types = ['smb', 'rakp_hmac_sha1_hash', 'smb_challenge']
import csv
for data in csv.reader([line]):
retval['ip'] = data[0]
retval['port'] = data[1]
retval['user'] = data[2]
retval['pass'] = data[3]
retval['type'] = data[4]
retval['msg'] = 'from metasploit'
# isactive = data[5] # unused
if retval['type'] in hash_types:
retval['hash'] = retval['pass']
retval['pass'] = lookup_hash(retval['hash'])
else:
retval['hash'] = None
retval['error'] = False
return retval
def process_hydra(line):
"""
Process a hydra line and return a dictionary:
{ 'ip' : ip address
'port': port info - can be port # or module name
'user': username,
'pass': password,
'hash': ntlm hash if smbnt hash used
'msg' : status message
}
"""
# line: [22][ssh] host: 1.1.1.1 login: username password: pw1234
retval = {}
try:
data = line.split()
except Exception as e:
log("Error processing hydra line: %s -- %s" % (e, line), logging.ERROR)
return retval
if data[1] == "host:":
# these fields are always there.. sometimes password is not
retval['port'] = data[0][1:data[0].find("]")]
retval['ip'] = data[2]
retval['user'] = data[4]
if "password:"******"[smb]" in data and "Error:" in data:
if len(retval['pass']) == 68 and retval['pass'][65:68] == ":::":
# we have an ntlm hash cap'n
retval['hash'] = ":".join(retval['pass'].split(':')[:2])
retval['pass'] = lookup_hash(retval['hash'])
retval['type'] = 'smb'
else:
retval['type'] = 'cleartext'
retval['hash'] = None
retval['error'] = False
return retval
# handle credential data
f_password = None
f_compromised = False
cred_type = cred.findtext('ptype')
if cred_type == "smb_hash":
# add smb hashes to info/0 service
svc_fields = {
'f_number': '0',
'f_proto': 'info',
'f_hosts_id': host_rec.id,
}
svc_rec = services.get_record(create_or_update=True, **svc_fields)
pwhash = cred.findtext('pass')
f_password = lookup_hash(pwhash)
(lm, nt) = pwhash.split(':')
user = cred.findtext('user')
query = (db.t_accounts.f_services_id == svc_rec.id) & (db.t_accounts.f_username.upper() == user.upper())
acct_row = db(query).select().first()
if acct_row:
# we have an account already, lets see if the hashes are in there
h1 = acct_row.f_hash1
if isinstance(h1, str):
if acct_row.f_hash1.upper() != lm.upper():
acct_row.f_hash1=lm.upper()
acct_row.f_hash1_type = "LM"
acct_row.f_hash2=nt.upper()
acct_row.f_hash2_type = "NT"
if f_password:
acct_row.f_compromised = True
f_password = None
f_compromised = False
cred_type = cred.findtext('ptype')
if cred_type == "smb_hash":
# add smb hashes to info/0 service
svc_fields = {
'f_number': '0',
'f_proto': 'info',
'f_hosts_id': host_rec.id,
}
svc_rec = services.get_record(create_or_update=True,
**svc_fields)
pwhash = cred.findtext('pass')
f_password = lookup_hash(pwhash)
(lm, nt) = pwhash.split(':')
user = cred.findtext('user')
query = (db.t_accounts.f_services_id == svc_rec.id) & (
db.t_accounts.f_username.upper() == user.upper())
acct_row = db(query).select().first()
if acct_row:
# we have an account already, lets see if the hashes are in there
h1 = acct_row.f_hash1
if isinstance(h1, str):
if acct_row.f_hash1.upper() != lm.upper():
acct_row.f_hash1 = lm.upper()
acct_row.f_hash1_type = "LM"
acct_row.f_hash2 = nt.upper()
acct_row.f_hash2_type = "NT"
if f_password:
# no password provided, adjust the field modulator cap'n
retval['pass'] = None
if len(data) == 6:
retval['msg'] = data[5]
else:
retval['pass'] = data[6]
if len(data) == 8:
retval['msg'] = data[7]
# handle specific SMB errors:
#if "[smb]" in data and "Error:" in data:
if len(retval['pass']) == 68 and retval['pass'][65:68] == ":::":
# we have an ntlm hash cap'n
retval['hash'] = ":".join(retval['pass'].split(':')[:2])
retval['pass'] = lookup_hash(retval['hash'])
retval['type'] = 'smb'
else:
retval['type'] = 'cleartext'
retval['hash'] = None
retval['error'] = False
return retval
##-------------------------------------------------------------------------
def _doctest():
import doctest
doctest.testmod()
if __name__ == "__main__":
def process_report_xml(
filename=None,
ip_ignore_list=None,
ip_include_list=None,
engineer=1,
asset_group="Metasploit Import",
update_hosts=True,
):
"""
Processes a Metasploit XML Export for the following data and adds to the db:
- Hosts and services
- Credentials
Generate the XML report by using db_export -t xml filename.xml or through WebUI
TODO: Auto-exploits successful exploit attempts if matching CVE/VulnDB entry found
"""
from gluon.validators import IS_IPADDRESS
from skaldship.passwords.utils import lookup_hash
from skaldship.hosts import get_host_record, get_or_create_record
from skaldship.services import Services
services = Services()
db = current.globalenv['db']
#cache = current.globalenv['cache']
try:
from lxml import etree
except ImportError:
try:
import xml.etree.cElementTree as etree
except ImportError:
try:
import xml.etree.ElementTree as etree
except:
raise Exception("Unable to find valid ElementTree module.")
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Metasploit Pro report file: %s" % (filename))
try:
xml = etree.parse(filename)
except etree.ParseError as e:
raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
root = xml.getroot()
# parse the hosts now
hosts = root.findall("hosts/host")
log(" [-] Parsing %d hosts" % (len(hosts)))
stats = {}
stats['hosts_added'] = 0
stats['hosts_skipped'] = 0
stats['hosts_updated'] = 0
stats['services_added'] = 0
stats['services_updated'] = 0
stats['accounts_added'] = 0
stats['accounts_updated'] = 0
for host in hosts:
didwhat = "Unknown"
if host.findtext('state') != "alive":
stats['hosts_skipped'] += 1
continue
hostfields = {}
ipaddr = host.findtext('address')
if len(ip_only) > 0 and ipaddr not in ip_only:
log(" [-] Node is not in the only list... skipping")
stats['hosts_skipped'] += 1
continue
if IS_IPADDRESS()(ipaddr)[1] is not None:
logger.error("Invalid IP Address in report: %s" % ipaddr)
log(" [!] Invalid IP Address in report: %s" % ipaddr)
continue
macaddr = host.findtext('mac')
if macaddr:
hostfields['f_macaddr'] = macaddr
hostname = host.findtext('name')
if hostname:
hostfields['f_hostname'] = hostname
# check to see if IP exists in DB already
hostfields['f_asset_group'] = asset_group
hostfields['f_engineer'] = engineer
if update_hosts:
# update or add, doesn't matter which
host_rec = get_or_create_record(ipaddr, **hostfields)
stats['hosts_added'] += 1
else:
# weird logic.. get a host record, if it doesn't exist create it otherwise skip because update_hosts=False
host_rec = get_host_record(ipaddr)
if not host_rec:
host_rec = get_or_create_record(ipaddr, **hostfields)
stats['hosts_added'] += 1
log(" [-] Adding IP: %s" % (ipaddr))
else:
stats['hosts_skipped'] += 1
log(" [-] Skipped IP: %s" % (ipaddr))
continue
# add the <info> and <comments> as a note to the host
info_note = host.findtext('info') or None
if info_note and info_note.startswith('Domain controller for '):
db.t_netbios.update_or_insert(f_hosts_id=host_rec.id,
f_type="PDC",
f_domain=info_note[22:].upper())
elif info_note:
db.t_host_notes.update_or_insert(
f_hosts_id=host_rec.id,
f_note=info_note,
)
db.commit()
for comment in host.findall('comments/comment'):
db.t_host_notes.update_or_insert(
f_hosts_id=host_rec.id,
f_note=comment.text,
)
# process the services, adding any new
for svc in host.findall('services/service'):
svc_fields = {
'f_number': svc.findtext('port'),
'f_proto': svc.findtext('proto'),
'f_status': svc.findtext('state'),
'f_name': svc.findtext('name') or '',
'f_banner': svc.findtext('info') or '',
'f_hosts_id': host_rec.id,
}
if svc_fields['f_name'] in ['http', 'https']:
svc_fields['f_name'] = svc_fields['f_name'].upper()
svc_rec = services.get_record(create_or_update=True, **svc_fields)
for cred in host.findall('creds/cred'):
# handle credential data
f_password = None
f_compromised = False
cred_type = cred.findtext('ptype')
if cred_type == "smb_hash":
# add smb hashes to info/0 service
svc_fields = {
'f_number': '0',
'f_proto': 'info',
'f_hosts_id': host_rec.id,
}
svc_rec = services.get_record(create_or_update=True,
**svc_fields)
pwhash = cred.findtext('pass')
f_password = lookup_hash(pwhash)
(lm, nt) = pwhash.split(':')
user = cred.findtext('user')
query = (db.t_accounts.f_services_id == svc_rec.id) & (
db.t_accounts.f_username.upper() == user.upper())
acct_row = db(query).select().first()
if acct_row:
# we have an account already, lets see if the hashes are in there
h1 = acct_row.f_hash1
if isinstance(h1, str):
if acct_row.f_hash1.upper() != lm.upper():
acct_row.f_hash1 = lm.upper()
acct_row.f_hash1_type = "LM"
acct_row.f_hash2 = nt.upper()
acct_row.f_hash2_type = "NT"
if f_password:
acct_row.f_compromised = True
acct_row.f_password = f_password
if not acct_row.f_source:
acct_row.f_source = "Metasploit Import"
acct_row.update_record()
db.commit()
stats['accounts_updated'] += 1
didwhat = "Updated"
else:
# add a new account record
if f_password:
f_compromised = True
else:
f_compromised = False
acct_data = dict(f_services_id=svc_rec.id,
f_username=user,
f_password=f_password,
f_compromised=f_compromised,
f_hash1=lm.upper(),
f_hash1_type='LM',
f_hash2=nt.upper(),
f_hash2_type='NT',
f_source="Metasploit Import")
acct_id = db.t_accounts.insert(**acct_data)
db.commit()
stats['accounts_added'] += 1
didwhat = "Added"
elif cred_type == 'smb_challenge':
# add smb challenge hashes to info/0 service
svc_fields = {
'f_number': '0',
'f_proto': 'info',
'f_hosts_id': host_rec.id,
}
svc_rec = services.get_record(create_or_update=True,
**svc_fields)
user = cred.findtext('user')
query = (db.t_accounts.f_services_id == svc_rec.id) & (
db.t_accounts.f_username.upper() == user.upper())
acct_row = db(query).select().first()
if acct_row:
# we have an account already, lets see if the hashes are in there
h1 = acct_row.f_hash1
if isinstance(h1, str):
if acct_row.f_hash1.upper() != lm.upper():
acct_row.f_password = f_password
acct_row.f_hash1 = pwhash.upper()
acct_row.f_hash1_type = 'NTCHALLENGE'
acct_row.f_domain = cred.findtext('proof')
if not acct_row.f_source:
acct_row.f_source = "Metasploit Capture"
acct_row.update_record()
db.commit()
stats['accounts_updated'] += 1
didwhat = "Updated"
else:
# new account record
f_password = lookup_hash(pwhash)
if f_password:
f_compromised = True
else:
f_compromised = False
acct_data = dict(f_services_id=svc_rec.id,
f_username=user,
f_password=f_password,
f_compromised=f_compromised,
f_hash1=pwhash.upper(),
f_hash1_type='NTCHALLENGE',
f_source="Metasploit Capture")
acct_id = db.t_accounts.insert(**acct_data)
db.commit()
stats['accounts_added'] += 1
didwhat = "Added"
elif cred_type == 'rakp_hmac_sha1_hash':
# IPMI 2.0 RAKP Remote SHA1 Hashes
f_hash1 = cred.findtext('pass')
f_hash1_type = cred.findtext('ptype')
user = cred.findtext('user')
svcname = cred.findtext('sname')
query = (db.t_accounts.f_services_id == svc_rec.id) & (
db.t_accounts.f_username.upper() == user.upper())
acct_row = db(query).select().first()
f_source = "Metasploit Import"
if acct_row:
# we have an account already, lets see if the hashes are in there
if acct_row.f_hash1 != f_hash1:
acct_row.f_hash1 = f_hash1
acct_row.f_hash1_type = f_hash1_type
if not acct_row.f_source:
acct_row.f_source = f_source
acct_row.update_record()
db.commit()
stats['accounts_updated'] += 1
didwhat = "Updated"
else:
# new account record
acct_data = dict(f_services_id=svc_rec.id,
f_username=user,
f_hash1=f_hash1,
f_hash1_type=f_hash1_type,
f_source=f_source,
f_compromised=True)
acct_id = db.t_accounts.insert(**acct_data)
db.commit()
stats['accounts_added'] += 1
didwhat = "Added"
else:
# for cred_type == 'password' or 'exploit':
# add regular password
if svc_fields['f_number'] == '445':
svc_fields['f_proto'] = 'info'
svc_fields['f_number'] = '0'
svc_rec = services.get_record(create_or_update=True,
**svc_fields)
f_password = cred.findtext('pass')
if f_password == "*BLANK PASSWORD*":
f_password = ''
user = cred.findtext('user')
svcname = cred.findtext('sname')
# do some case mangling for known variations we want in all upper case
if svcname == "vnc":
user = "******"
query = (db.t_accounts.f_services_id == svc_rec.id) & (
db.t_accounts.f_username.upper() == user.upper())
acct_row = db(query).select().first()
f_source = cred.findtext('type')
if f_source == 'captured':
f_source = "Metasploit Capture"
else:
f_source = "Metasploit Import"
if acct_row:
# we have an account already, lets see if the hashes are in there
if acct_row.f_password != f_password:
acct_row.f_password = f_password
acct_row.f_compromised = True
if not acct_row.f_source:
acct_row.f_source = f_source
acct_row.update_record()
db.commit()
stats['accounts_updated'] += 1
didwhat = "Updated"
else:
# new account record
acct_data = dict(f_services_id=svc_rec.id,
f_username=user,
f_password=f_password,
f_source=f_source,
f_compromised=True)
acct_id = db.t_accounts.insert(**acct_data)
db.commit()
stats['accounts_added'] += 1
didwhat = "Added"
log(" [-] Account %s: (%s) %s" % (didwhat, ipaddr, user))
do_host_status()
msg = " [*] Import complete: hosts: (%s/A, %s/U, %s/S) - services: (%s/A, %s/U), creds: (%s/A, %s/U)"\
% (
stats['hosts_added'],
stats['hosts_updated'],
stats['hosts_skipped'],
stats['services_added'],
stats['services_updated'],
stats['accounts_added'],
stats['accounts_updated']
)
log(msg)
return msg