def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['STREAMS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
d = find_session(md5hash)
folder = d[2]
else:
folder = config['working/directory']
l = len(folder) + 11
raw = pcap[l:-5]
raw = raw.split('-')
banner = 'Protocol:%s\nSource:%s\nDestination:%s' % (raw[0], raw[1], raw[2])
e = pcapStream(banner)
response += e
return response
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['STREAMS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
d = find_session(md5hash)
folder = d[2]
else:
folder = config['working/directory']
l = len(folder) + 11
raw = pcap[l:-5]
raw = raw.split('-')
banner = 'Protocol:%s\nSource:%s\nDestination:%s' % (raw[0], raw[1],
raw[2])
e = pcapStream(banner)
response += e
return response
def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['DNS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
# Get the session and/or pcap id
d = find_session(md5hash)
pcap_id = d[0]
session_id = d[1]
else:
pass
try:
pkts = rdpcap(pcap)
dns_requests = []
for p in pkts:
if p.haslayer(DNSQR):
timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[DNSQR].qname[:-1]
tld = tldextract.extract(r)
domain = tld.registered_domain
if usedb > 0:
dns = OrderedDict({'PCAP ID': pcap_id, 'Stream ID': session_id,
'Time Stamp': timestamp,
'Type': 'Request', 'IP': {'src': p[IP].src, 'dst': p[IP].dst, 'length': p[IP].len},
'Request Details': {'Query Type': p[DNSQR].qtype, 'Query Name': r, 'Domain': domain}})
t = x.DNS.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(dns)
else:
pass
if r not in dns_requests:
dns_requests.append(domain)
else:
pass
for d in dns_requests:
x = Domain(d)
response += x
return response
except Exception as e:
if usedb > 0:
error_logging(str(e), 'DNS Requests')
else:
return response + UIMessage(str(e))
def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['HTTP']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
d = find_session(md5hash)
pcap_id = d[0]
else:
pass
# Find HTTP Requests
pkts = rdpcap(pcap)
http_requests = []
for p in pkts:
if p.haslayer(HTTPRequest):
timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[HTTPRequest].Host
if usedb > 0:
http = OrderedDict({'PCAP ID': pcap_id,
'Time Stamp': timestamp,
'Type': 'HTTP Request', 'IP': {'src': p[IP].src, 'dst': p[IP].dst},
'HTTP': {'Method': p[HTTPRequest].Method, 'URI': p[HTTPRequest].Path,
'Referer': p[HTTPRequest].Referer, 'Host': p[HTTPRequest].Host}})
# Check if record already exists
s = x.HTTP.find({'Time Stamp': timestamp}).count()
if s > 0:
pass
else:
c.insert(http)
if r not in http_requests:
http_requests.append(r)
else:
pass
for i in http_requests:
h = Website(i)
response += h
return response
def dotransform(request, response):
pcap = request.value
lookfor = ['MAIL FROM:', 'RCPT TO:']
pkts = rdpcap(pcap)
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
d = mongo_connect()
c = d['CREDS']
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
x = find_session(md5pcap)
pcap_id = x[0]
else:
pass
addr = []
try:
for p in pkts:
for m in lookfor:
if p.haslayer(TCP) and p.haslayer(Raw):
raw = p[Raw].load
if m in raw:
for s in re.finditer('<([\S.-]+@[\S-]+)>', raw):
addr.append(s.group(1))
except Exception as e:
return response + UIMessage(str(e))
for x in addr:
if usedb > 0:
data = {'PCAP ID': pcap_id, 'Type': 'Email Address', 'Record': x}
t = d.CREDS.find({'Record': x}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
e = EmailAddress(x)
response += e
return response
def dotransform(request, response):
pcap = request.value
lookfor = ['MAIL FROM:', 'RCPT TO:']
pkts = rdpcap(pcap)
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
d = mongo_connect()
c = d['CREDS']
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
x = find_session(md5pcap)
pcap_id = x[0]
else:
pass
addr = []
try:
for p in pkts:
for m in lookfor:
if p.haslayer(TCP) and p.haslayer(Raw):
raw = p[Raw].load
if m in raw:
for s in re.finditer('<([\S.-]+@[\S-]+)>', raw):
addr.append(s.group(1))
except Exception as e:
return response + UIMessage(str(e))
for x in addr:
if usedb > 0:
data = {'PCAP ID': pcap_id, 'Type': 'Email Address', 'Record': x}
t = d.CREDS.find({'Record': x}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
e = EmailAddress(x)
response += e
return response
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['CREDS']
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
d = find_session(md5pcap)
pcap_id = d[0]
else:
pass
d = smtp_creds(pcap)
if len(d) == 0:
return response + UIMessage('No SMTP Credentials found..sorry')
for n in d:
if usedb > 0:
data = {'PCAP ID': pcap_id, 'Type': 'Email Credential', 'Record': n}
t = x.CREDS.find({'Record': n}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
e = Credential(n)
response += e
return response
def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['DNS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
# Get the session and/or pcap id
d = find_session(md5hash)
pcap_id = d[0]
session_id = d[1]
else:
pass
try:
pkts = rdpcap(pcap)
dns_requests = []
for p in pkts:
if p.haslayer(DNSQR):
timestamp = datetime.datetime.fromtimestamp(
p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[DNSQR].qname[:-1]
tld = tldextract.extract(r)
domain = tld.registered_domain
if usedb > 0:
dns = OrderedDict({
'PCAP ID': pcap_id,
'Stream ID': session_id,
'Time Stamp': timestamp,
'Type': 'Request',
'IP': {
'src': p[IP].src,
'dst': p[IP].dst,
'length': p[IP].len
},
'Request Details': {
'Query Type': p[DNSQR].qtype,
'Query Name': r,
'Domain': domain
}
})
t = x.DNS.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(dns)
else:
pass
if r not in dns_requests:
dns_requests.append(domain)
else:
pass
for d in dns_requests:
x = Domain(d)
response += x
return response
except Exception as e:
if usedb > 0:
error_logging(str(e), 'DNS Requests')
else:
return response + UIMessage(str(e))
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
d = mongo_connect()
c = d['SSL']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
d = find_session(md5hash)
pcap_id = d[0]
else:
pass
# Load the packets
pkts = rdpcap(pcap)
# Look for SSL packets and pull out the required information.
servers = []
try:
for p in pkts:
if p.haslayer(IP) and p.haslayer(TCP) and p.haslayer(Raw):
x = p[Raw].load
x = hexstr(x)
x = x.split(' ')
if x[0] == '16':
timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
stype = 'Handshake'
if x[5] == '01':
htype = 'Client Hello'
slen = int(''.join(x[131:133]), 16)
s = 133 + slen
sname = binascii.unhexlify(''.join(x[133:s]))
if sname not in servers:
servers.append(sname)
if usedb > 0:
data = {'PCAP ID': pcap_id, 'SSL Type': stype, 'Handshake Type': htype,
'Time Stamp': timestamp,
'Source IP': p[IP].src, 'Source Port': p[TCP].sport, 'Destination IP': p[IP].dst,
'Destination Port': p[TCP].dport, 'Server Name': sname}
t = d.SSL.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
if x[5] == '02':
htype = 'Server Hello'
ctype = ''.join(x[76:78])
if usedb > 0:
data = {'PCAP ID': pcap_id, 'SSL Type': stype, 'Handshake Type': htype,
'Time Stamp': timestamp,
'Source IP': p[IP].src, 'Source Port': p[TCP].sport, 'Destination IP': p[IP].dst,
'Destination Port': p[TCP].dport, 'Cipher Suite': ctype}
t = d.SSL.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
else:
pass
else:
pass
except Exception as e:
return response + UIMessage(str(e))
# Return Maltego entities based on the SSL server name
for s in servers:
e = Website(s)
response += e
return response
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
d = mongo_connect()
c = d['ARTIFACTS']
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
x = find_session(md5pcap)
pcap_id = x[0]
folder = x[2]
else:
w = config['working/directory'].strip('\'')
try:
if w != '':
w = w + '/' + str(uuid.uuid4())[:12].replace('-', '')
if not os.path.exists(w):
os.makedirs(w)
folder = w
else:
return response + UIMessage(
'No working directory set, check your config file')
except Exception as e:
return response + UIMessage(e)
folder = '%s/%s' % (folder, 'artifacts')
if not os.path.exists(folder):
os.makedirs(folder)
dissector = Dissector() # instance of dissector class
dissector.change_dfolder(folder)
dissector.dissect_pkts(pcap)
list_files = glob.glob(folder + '/*')
# print list_files
# Loop through the stored files and create the database/maltego objects
for g in list_files:
try:
md5hash = md5_for_file(g)
sha1hash = sha1_for_file(g)
ftype = check_file(g)
n = len(folder) + 1
l = len(g)
filename = g[n:l]
if usedb > 0:
data = {
'PCAP ID': pcap_id,
'Path': folder,
'File Name': filename,
'File Type': ftype,
'MD5 Hash': md5hash,
'SHA1 Hash': sha1hash
}
t = d.ARTIFACTS.find({
'MD5 Hash': md5hash,
"File Name": filename
}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
# Create the Maltego entities
a = Artifact(filename)
a.ftype = ftype
a.fhash = md5hash
a += Field('path', folder, displayname='Path')
response += a
except Exception as e:
print str(e)
return response
def dotransform(request, response):
pcap = request.value
folder = ''
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['STREAMS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
d = find_session(md5hash)
pcap_id = d[0]
folder = d[2]
except Exception as e:
return response + UIMessage(str(e))
else:
w = config['working/directory'].strip('\'')
try:
if w != '':
w = w + '/' + str(uuid.uuid4())[:12].replace('-', '')
if not os.path.exists(w):
os.makedirs(w)
folder = w
else:
return response + UIMessage('No working directory set, check your config file')
except Exception as e:
return response + UIMessage(e)
# Create TCP/UDP stream files
s = create_streams(pcap, folder)
if usedb > 0:
for i in s:
# Create StreamID
streamid = str(uuid.uuid4())[:8]
# Get a count of packets available
try:
pkcount = packet_count(i)
except Exception as e:
return response + UIMessage(str(e))
# Get the start/end time of packets
try:
pcap_time = get_time(i)
except Exception as e:
return response + UIMessage(str(e))
# Hash the pcap file
try:
md5hash = md5_for_file(i)
sha1hash = sha1_for_file(i)
except Exception as e:
return response + UIMessage(str(e))
# Pull out the details of the packets
l = len(folder) + 1
raw = i[l:-5]
pkt = raw.replace('-', ' ').replace(':', ' ').split()
# Create the dictonary object to insert into database
data = OrderedDict({'PCAP ID': pcap_id, 'Stream ID': streamid, 'Folder': folder, 'Packet Count': pkcount,
'File Name': i, 'First Packet': pcap_time[0], 'Last Packet': pcap_time[1],
'MD5 Hash': md5hash, 'SHA1 Hash': sha1hash,
'Packet': {'Protocol': pkt[0], 'Source IP': pkt[1], 'Source Port': pkt[2],
'Destination IP': pkt[3], 'Destination Port': pkt[4]}})
# Check to see if the record exists
try:
t = x.STREAMS.find({"File Name": i}).count()
if t > 0:
pass
else:
c.insert(data)
except Exception as e:
return response + UIMessage(str(e))
else:
pass
# Create Maltego entities for each pcap file
for p in s:
e = pcapFile(p)
response += e
return response
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
d = mongo_connect()
c = d['ARTIFACTS']
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
x = find_session(md5pcap)
pcap_id = x[0]
folder = x[2]
else:
w = config['working/directory'].strip('\'')
try:
if w != '':
w = w + '/' + str(uuid.uuid4())[:12].replace('-', '')
if not os.path.exists(w):
os.makedirs(w)
folder = w
else:
return response + UIMessage('No working directory set, check your config file')
except Exception as e:
return response + UIMessage(e)
folder = '%s/%s' % (folder, 'artifacts')
if not os.path.exists(folder):
os.makedirs(folder)
dissector = Dissector() # instance of dissector class
dissector.change_dfolder(folder)
dissector.dissect_pkts(pcap)
list_files = glob.glob(folder+'/*')
# print list_files
# Loop through the stored files and create the database/maltego objects
for g in list_files:
try:
md5hash = md5_for_file(g)
sha1hash = sha1_for_file(g)
ftype = check_file(g)
n = len(folder) + 1
l = len(g)
filename = g[n:l]
if usedb > 0:
data = {'PCAP ID': pcap_id, 'Path': folder, 'File Name': filename, 'File Type': ftype, 'MD5 Hash': md5hash,
'SHA1 Hash': sha1hash}
t = d.ARTIFACTS.find({'MD5 Hash': md5hash, "File Name": filename}).count()
if t > 0:
pass
else:
c.insert(data)
else:
pass
# Create the Maltego entities
a = Artifact(filename)
a.ftype = ftype
a.fhash = md5hash
a += Field('path', folder, displayname='Path')
response += a
except Exception as e:
print str(e)
return response
def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['HTTP']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
d = find_session(md5hash)
pcap_id = d[0]
else:
pass
# Find HTTP Requests
pkts = rdpcap(pcap)
http_requests = []
for p in pkts:
if p.haslayer(HTTPRequest):
timestamp = datetime.datetime.fromtimestamp(
p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[HTTPRequest].Host
if usedb > 0:
http = OrderedDict({
'PCAP ID': pcap_id,
'Time Stamp': timestamp,
'Type': 'HTTP Request',
'IP': {
'src': p[IP].src,
'dst': p[IP].dst
},
'HTTP': {
'Method': p[HTTPRequest].Method,
'URI': p[HTTPRequest].Path,
'Referer': p[HTTPRequest].Referer,
'Host': p[HTTPRequest].Host
}
})
# Check if record already exists
s = x.HTTP.find({'Time Stamp': timestamp}).count()
if s > 0:
pass
else:
c.insert(http)
if r not in http_requests:
http_requests.append(r)
else:
pass
for i in http_requests:
h = Website(i)
response += h
return response
def dotransform(request, response):
pcap = request.value
folder = ''
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['STREAMS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
d = find_session(md5hash)
pcap_id = d[0]
folder = d[2]
except Exception as e:
return response + UIMessage(str(e))
else:
w = config['working/directory'].strip('\'')
try:
if w != '':
w = w + '/' + str(uuid.uuid4())[:12].replace('-', '')
if not os.path.exists(w):
os.makedirs(w)
folder = w
else:
return response + UIMessage(
'No working directory set, check your config file')
except Exception as e:
return response + UIMessage(e)
# Create TCP/UDP stream files
s = create_streams(pcap, folder)
if usedb > 0:
for i in s:
# Create StreamID
streamid = str(uuid.uuid4())[:8]
# Get a count of packets available
try:
pkcount = packet_count(i)
except Exception as e:
return response + UIMessage(str(e))
# Get the start/end time of packets
try:
pcap_time = get_time(i)
except Exception as e:
return response + UIMessage(str(e))
# Hash the pcap file
try:
md5hash = md5_for_file(i)
sha1hash = sha1_for_file(i)
except Exception as e:
return response + UIMessage(str(e))
# Pull out the details of the packets
l = len(folder) + 1
raw = i[l:-5]
pkt = raw.replace('-', ' ').replace(':', ' ').split()
# Create the dictonary object to insert into database
data = OrderedDict({
'PCAP ID': pcap_id,
'Stream ID': streamid,
'Folder': folder,
'Packet Count': pkcount,
'File Name': i,
'First Packet': pcap_time[0],
'Last Packet': pcap_time[1],
'MD5 Hash': md5hash,
'SHA1 Hash': sha1hash,
'Packet': {
'Protocol': pkt[0],
'Source IP': pkt[1],
'Source Port': pkt[2],
'Destination IP': pkt[3],
'Destination Port': pkt[4]
}
})
# Check to see if the record exists
try:
t = x.STREAMS.find({"File Name": i}).count()
if t > 0:
pass
else:
c.insert(data)
except Exception as e:
return response + UIMessage(str(e))
else:
pass
# Create Maltego entities for each pcap file
for p in s:
e = pcapFile(p)
response += e
return response
