def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['DNS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
# Get the session and/or pcap id
d = find_session(md5hash)
pcap_id = d[0]
session_id = d[1]
else:
pass
try:
pkts = rdpcap(pcap)
dns_requests = []
for p in pkts:
if p.haslayer(DNSQR):
timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[DNSQR].qname[:-1]
tld = tldextract.extract(r)
domain = tld.registered_domain
if usedb > 0:
dns = OrderedDict({'PCAP ID': pcap_id, 'Stream ID': session_id,
'Time Stamp': timestamp,
'Type': 'Request', 'IP': {'src': p[IP].src, 'dst': p[IP].dst, 'length': p[IP].len},
'Request Details': {'Query Type': p[DNSQR].qtype, 'Query Name': r, 'Domain': domain}})
t = x.DNS.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(dns)
else:
pass
if r not in dns_requests:
dns_requests.append(domain)
else:
pass
for d in dns_requests:
x = Domain(d)
response += x
return response
except Exception as e:
if usedb > 0:
error_logging(str(e), 'DNS Requests')
else:
return response + UIMessage(str(e))
def dotransform(request, response):
# Store the pcap file as a variable
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb > 0:
# Connect to the database so we can insert the record created below
x = mongo_connect()
c = x['DNS']
# Hash the pcap file
try:
md5hash = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
# Get the session and/or pcap id
d = find_session(md5hash)
pcap_id = d[0]
session_id = d[1]
else:
pass
try:
pkts = rdpcap(pcap)
dns_requests = []
for p in pkts:
if p.haslayer(DNSQR):
timestamp = datetime.datetime.fromtimestamp(
p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
r = p[DNSQR].qname[:-1]
tld = tldextract.extract(r)
domain = tld.registered_domain
if usedb > 0:
dns = OrderedDict({
'PCAP ID': pcap_id,
'Stream ID': session_id,
'Time Stamp': timestamp,
'Type': 'Request',
'IP': {
'src': p[IP].src,
'dst': p[IP].dst,
'length': p[IP].len
},
'Request Details': {
'Query Type': p[DNSQR].qtype,
'Query Name': r,
'Domain': domain
}
})
t = x.DNS.find({'Time Stamp': timestamp}).count()
if t > 0:
pass
else:
c.insert(dns)
else:
pass
if r not in dns_requests:
dns_requests.append(domain)
else:
pass
for d in dns_requests:
x = Domain(d)
response += x
return response
except Exception as e:
if usedb > 0:
error_logging(str(e), 'DNS Requests')
else:
return response + UIMessage(str(e))
def dotransform(request, response):
pcap = request.value
usedb = config['working/usedb']
# Check to see if we are using the database or not
if usedb == 0:
return response + UIMessage('You have chosen not to use a database')
else:
pass
d = mongo_connect()
c = d['PACKETS']
y = d['PACKETSUMMARY']
url = config['web/server'].strip('\'')
port = config['web/port'].strip('\'')
# Hash the pcap file
try:
md5pcap = md5_for_file(pcap)
except Exception as e:
return response + UIMessage(str(e))
def convert_encoding(data, encoding='utf-8'):
if isinstance(data, dict):
return dict((convert_encoding(key), convert_encoding(value)) \
for key, value in data.iteritems())
elif isinstance(data, list):
return [convert_encoding(element) for element in data]
elif isinstance(data, unicode):
return data.encode(encoding, errors='replace')
else:
return data
# Get the PCAP ID for the pcap file
try:
s = d.INDEX.find({"MD5 Hash": md5pcap}).count()
if s == 0:
t = d.STREAMS.find({"MD5 Hash": md5pcap}).count()
if t > 0:
r = d.STREAMS.find({"MD5 Hash": md5pcap}, {"PCAP ID": 1, "Stream ID": 1, "_id": 0})
for i in r:
pcap_id = i['PCAP ID']
streamid = i['Stream ID']
else:
return response + UIMessage('No PCAP ID, you need to index the pcap file')
if s > 0:
r = d.INDEX.find({"MD5 Hash": md5pcap}, {"PCAP ID": 1, "_id": 0})
for i in r:
pcap_id = i['PCAP ID']
streamid = i['PCAP ID']
except Exception as e:
return response + UIMessage(str(e))
stream_url = 'http://%s:%s/pcap/%s/packets' % (url, port, streamid)
pkts = loadpackets(pcap)
# Dump the full packets into the database for later use.
x = find_layers(pkts, pcap, pcap_id, streamid)
try:
for s in x:
tstamp = s['Buffer']['timestamp']
q = d.PACKETS.find({"Buffer.timestamp": tstamp}).count()
if q > 0:
pass
else:
v = OrderedDict(json.loads(json.dumps(convert_encoding(s), encoding='latin-1', ensure_ascii=False)))
c.insert(v)
except Exception as e:
error_logging(str(e), 'Packets')
# Build the packet summary so we can make pretty pages.
count = 1
packet = OrderedDict()
try:
for p in pkts:
tstamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f')
p_header = {"PCAP ID": pcap_id, "Buffer": {"timestamp": tstamp, "packetnumber": count, "pcapfile": pcap,
"packet_length": p.len, "StreamID": streamid}}
packet.update(p_header)
if p.haslayer(IP):
p_ip = {"IP": {"ip_src": p[IP].src, "ip_dst": p[IP].dst, "ip_ttl": p[IP].ttl}}
packet.update(p_ip)
layers = []
counter = 0
while True:
layer = p.getlayer(counter)
if layer != None:
if layer.name == 'HTTP':
pass
else:
layers.append(layer.name)
else:
break
counter += 1
p_layers = {"Layers": layers}
packet.update(p_layers)
view_url = 'http://%s:%s/pcap/%s/%s/packets/%s' % (url, port, pcap_id, streamid, count)
p_view = {"View": view_url}
packet.update(p_view)
t = d.PACKETSUMMARY.find({"Buffer.timestamp": tstamp}).count()
if t > 0:
pass
else:
y.insert(packet)
count += 1
packet.clear()
except Exception as e:
error_logging(str(e), 'PacketSummary')
# Return the Maltego Entity
a = pcapStream(stream_url)
response += a
return response