def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['STREAMS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) d = find_session(md5hash) folder = d[2] else: folder = config['working/directory'] l = len(folder) + 11 raw = pcap[l:-5] raw = raw.split('-') banner = 'Protocol:%s\nSource:%s\nDestination:%s' % (raw[0], raw[1], raw[2]) e = pcapStream(banner) response += e return response
def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['STREAMS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) d = find_session(md5hash) folder = d[2] else: folder = config['working/directory'] l = len(folder) + 11 raw = pcap[l:-5] raw = raw.split('-') banner = 'Protocol:%s\nSource:%s\nDestination:%s' % (raw[0], raw[1], raw[2]) e = pcapStream(banner) response += e return response
def dotransform(request, response): # Store the pcap file as a variable pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['DNS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) # Get the session and/or pcap id d = find_session(md5hash) pcap_id = d[0] session_id = d[1] else: pass try: pkts = rdpcap(pcap) dns_requests = [] for p in pkts: if p.haslayer(DNSQR): timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f') r = p[DNSQR].qname[:-1] tld = tldextract.extract(r) domain = tld.registered_domain if usedb > 0: dns = OrderedDict({'PCAP ID': pcap_id, 'Stream ID': session_id, 'Time Stamp': timestamp, 'Type': 'Request', 'IP': {'src': p[IP].src, 'dst': p[IP].dst, 'length': p[IP].len}, 'Request Details': {'Query Type': p[DNSQR].qtype, 'Query Name': r, 'Domain': domain}}) t = x.DNS.find({'Time Stamp': timestamp}).count() if t > 0: pass else: c.insert(dns) else: pass if r not in dns_requests: dns_requests.append(domain) else: pass for d in dns_requests: x = Domain(d) response += x return response except Exception as e: if usedb > 0: error_logging(str(e), 'DNS Requests') else: return response + UIMessage(str(e))
def dotransform(request, response): # Store the pcap file as a variable pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['HTTP'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) d = find_session(md5hash) pcap_id = d[0] else: pass # Find HTTP Requests pkts = rdpcap(pcap) http_requests = [] for p in pkts: if p.haslayer(HTTPRequest): timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f') r = p[HTTPRequest].Host if usedb > 0: http = OrderedDict({'PCAP ID': pcap_id, 'Time Stamp': timestamp, 'Type': 'HTTP Request', 'IP': {'src': p[IP].src, 'dst': p[IP].dst}, 'HTTP': {'Method': p[HTTPRequest].Method, 'URI': p[HTTPRequest].Path, 'Referer': p[HTTPRequest].Referer, 'Host': p[HTTPRequest].Host}}) # Check if record already exists s = x.HTTP.find({'Time Stamp': timestamp}).count() if s > 0: pass else: c.insert(http) if r not in http_requests: http_requests.append(r) else: pass for i in http_requests: h = Website(i) response += h return response
def dotransform(request, response): pcap = request.value lookfor = ['MAIL FROM:', 'RCPT TO:'] pkts = rdpcap(pcap) usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: d = mongo_connect() c = d['CREDS'] # Hash the pcap file try: md5pcap = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) x = find_session(md5pcap) pcap_id = x[0] else: pass addr = [] try: for p in pkts: for m in lookfor: if p.haslayer(TCP) and p.haslayer(Raw): raw = p[Raw].load if m in raw: for s in re.finditer('<([\S.-]+@[\S-]+)>', raw): addr.append(s.group(1)) except Exception as e: return response + UIMessage(str(e)) for x in addr: if usedb > 0: data = {'PCAP ID': pcap_id, 'Type': 'Email Address', 'Record': x} t = d.CREDS.find({'Record': x}).count() if t > 0: pass else: c.insert(data) else: pass e = EmailAddress(x) response += e return response
def dotransform(request, response): pcap = request.value lookfor = ['MAIL FROM:', 'RCPT TO:'] pkts = rdpcap(pcap) usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: d = mongo_connect() c = d['CREDS'] # Hash the pcap file try: md5pcap = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) x = find_session(md5pcap) pcap_id = x[0] else: pass addr = [] try: for p in pkts: for m in lookfor: if p.haslayer(TCP) and p.haslayer(Raw): raw = p[Raw].load if m in raw: for s in re.finditer('<([\S.-]+@[\S-]+)>', raw): addr.append(s.group(1)) except Exception as e: return response + UIMessage(str(e)) for x in addr: if usedb > 0: data = {'PCAP ID': pcap_id, 'Type': 'Email Address', 'Record': x} t = d.CREDS.find({'Record': x}).count() if t > 0: pass else: c.insert(data) else: pass e = EmailAddress(x) response += e return response
def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['CREDS'] # Hash the pcap file try: md5pcap = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) d = find_session(md5pcap) pcap_id = d[0] else: pass d = smtp_creds(pcap) if len(d) == 0: return response + UIMessage('No SMTP Credentials found..sorry') for n in d: if usedb > 0: data = {'PCAP ID': pcap_id, 'Type': 'Email Credential', 'Record': n} t = x.CREDS.find({'Record': n}).count() if t > 0: pass else: c.insert(data) else: pass e = Credential(n) response += e return response
def dotransform(request, response): # Store the pcap file as a variable pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['DNS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) # Get the session and/or pcap id d = find_session(md5hash) pcap_id = d[0] session_id = d[1] else: pass try: pkts = rdpcap(pcap) dns_requests = [] for p in pkts: if p.haslayer(DNSQR): timestamp = datetime.datetime.fromtimestamp( p.time).strftime('%Y-%m-%d %H:%M:%S.%f') r = p[DNSQR].qname[:-1] tld = tldextract.extract(r) domain = tld.registered_domain if usedb > 0: dns = OrderedDict({ 'PCAP ID': pcap_id, 'Stream ID': session_id, 'Time Stamp': timestamp, 'Type': 'Request', 'IP': { 'src': p[IP].src, 'dst': p[IP].dst, 'length': p[IP].len }, 'Request Details': { 'Query Type': p[DNSQR].qtype, 'Query Name': r, 'Domain': domain } }) t = x.DNS.find({'Time Stamp': timestamp}).count() if t > 0: pass else: c.insert(dns) else: pass if r not in dns_requests: dns_requests.append(domain) else: pass for d in dns_requests: x = Domain(d) response += x return response except Exception as e: if usedb > 0: error_logging(str(e), 'DNS Requests') else: return response + UIMessage(str(e))
def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below d = mongo_connect() c = d['SSL'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) d = find_session(md5hash) pcap_id = d[0] else: pass # Load the packets pkts = rdpcap(pcap) # Look for SSL packets and pull out the required information. servers = [] try: for p in pkts: if p.haslayer(IP) and p.haslayer(TCP) and p.haslayer(Raw): x = p[Raw].load x = hexstr(x) x = x.split(' ') if x[0] == '16': timestamp = datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f') stype = 'Handshake' if x[5] == '01': htype = 'Client Hello' slen = int(''.join(x[131:133]), 16) s = 133 + slen sname = binascii.unhexlify(''.join(x[133:s])) if sname not in servers: servers.append(sname) if usedb > 0: data = {'PCAP ID': pcap_id, 'SSL Type': stype, 'Handshake Type': htype, 'Time Stamp': timestamp, 'Source IP': p[IP].src, 'Source Port': p[TCP].sport, 'Destination IP': p[IP].dst, 'Destination Port': p[TCP].dport, 'Server Name': sname} t = d.SSL.find({'Time Stamp': timestamp}).count() if t > 0: pass else: c.insert(data) else: pass if x[5] == '02': htype = 'Server Hello' ctype = ''.join(x[76:78]) if usedb > 0: data = {'PCAP ID': pcap_id, 'SSL Type': stype, 'Handshake Type': htype, 'Time Stamp': timestamp, 'Source IP': p[IP].src, 'Source Port': p[TCP].sport, 'Destination IP': p[IP].dst, 'Destination Port': p[TCP].dport, 'Cipher Suite': ctype} t = d.SSL.find({'Time Stamp': timestamp}).count() if t > 0: pass else: c.insert(data) else: pass else: pass else: pass except Exception as e: return response + UIMessage(str(e)) # Return Maltego entities based on the SSL server name for s in servers: e = Website(s) response += e return response
def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below d = mongo_connect() c = d['ARTIFACTS'] # Hash the pcap file try: md5pcap = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) x = find_session(md5pcap) pcap_id = x[0] folder = x[2] else: w = config['working/directory'].strip('\'') try: if w != '': w = w + '/' + str(uuid.uuid4())[:12].replace('-', '') if not os.path.exists(w): os.makedirs(w) folder = w else: return response + UIMessage( 'No working directory set, check your config file') except Exception as e: return response + UIMessage(e) folder = '%s/%s' % (folder, 'artifacts') if not os.path.exists(folder): os.makedirs(folder) dissector = Dissector() # instance of dissector class dissector.change_dfolder(folder) dissector.dissect_pkts(pcap) list_files = glob.glob(folder + '/*') # print list_files # Loop through the stored files and create the database/maltego objects for g in list_files: try: md5hash = md5_for_file(g) sha1hash = sha1_for_file(g) ftype = check_file(g) n = len(folder) + 1 l = len(g) filename = g[n:l] if usedb > 0: data = { 'PCAP ID': pcap_id, 'Path': folder, 'File Name': filename, 'File Type': ftype, 'MD5 Hash': md5hash, 'SHA1 Hash': sha1hash } t = d.ARTIFACTS.find({ 'MD5 Hash': md5hash, "File Name": filename }).count() if t > 0: pass else: c.insert(data) else: pass # Create the Maltego entities a = Artifact(filename) a.ftype = ftype a.fhash = md5hash a += Field('path', folder, displayname='Path') response += a except Exception as e: print str(e) return response
def dotransform(request, response): pcap = request.value folder = '' usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['STREAMS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) d = find_session(md5hash) pcap_id = d[0] folder = d[2] except Exception as e: return response + UIMessage(str(e)) else: w = config['working/directory'].strip('\'') try: if w != '': w = w + '/' + str(uuid.uuid4())[:12].replace('-', '') if not os.path.exists(w): os.makedirs(w) folder = w else: return response + UIMessage('No working directory set, check your config file') except Exception as e: return response + UIMessage(e) # Create TCP/UDP stream files s = create_streams(pcap, folder) if usedb > 0: for i in s: # Create StreamID streamid = str(uuid.uuid4())[:8] # Get a count of packets available try: pkcount = packet_count(i) except Exception as e: return response + UIMessage(str(e)) # Get the start/end time of packets try: pcap_time = get_time(i) except Exception as e: return response + UIMessage(str(e)) # Hash the pcap file try: md5hash = md5_for_file(i) sha1hash = sha1_for_file(i) except Exception as e: return response + UIMessage(str(e)) # Pull out the details of the packets l = len(folder) + 1 raw = i[l:-5] pkt = raw.replace('-', ' ').replace(':', ' ').split() # Create the dictonary object to insert into database data = OrderedDict({'PCAP ID': pcap_id, 'Stream ID': streamid, 'Folder': folder, 'Packet Count': pkcount, 'File Name': i, 'First Packet': pcap_time[0], 'Last Packet': pcap_time[1], 'MD5 Hash': md5hash, 'SHA1 Hash': sha1hash, 'Packet': {'Protocol': pkt[0], 'Source IP': pkt[1], 'Source Port': pkt[2], 'Destination IP': pkt[3], 'Destination Port': pkt[4]}}) # Check to see if the record exists try: t = x.STREAMS.find({"File Name": i}).count() if t > 0: pass else: c.insert(data) except Exception as e: return response + UIMessage(str(e)) else: pass # Create Maltego entities for each pcap file for p in s: e = pcapFile(p) response += e return response
def dotransform(request, response): pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below d = mongo_connect() c = d['ARTIFACTS'] # Hash the pcap file try: md5pcap = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) x = find_session(md5pcap) pcap_id = x[0] folder = x[2] else: w = config['working/directory'].strip('\'') try: if w != '': w = w + '/' + str(uuid.uuid4())[:12].replace('-', '') if not os.path.exists(w): os.makedirs(w) folder = w else: return response + UIMessage('No working directory set, check your config file') except Exception as e: return response + UIMessage(e) folder = '%s/%s' % (folder, 'artifacts') if not os.path.exists(folder): os.makedirs(folder) dissector = Dissector() # instance of dissector class dissector.change_dfolder(folder) dissector.dissect_pkts(pcap) list_files = glob.glob(folder+'/*') # print list_files # Loop through the stored files and create the database/maltego objects for g in list_files: try: md5hash = md5_for_file(g) sha1hash = sha1_for_file(g) ftype = check_file(g) n = len(folder) + 1 l = len(g) filename = g[n:l] if usedb > 0: data = {'PCAP ID': pcap_id, 'Path': folder, 'File Name': filename, 'File Type': ftype, 'MD5 Hash': md5hash, 'SHA1 Hash': sha1hash} t = d.ARTIFACTS.find({'MD5 Hash': md5hash, "File Name": filename}).count() if t > 0: pass else: c.insert(data) else: pass # Create the Maltego entities a = Artifact(filename) a.ftype = ftype a.fhash = md5hash a += Field('path', folder, displayname='Path') response += a except Exception as e: print str(e) return response
def dotransform(request, response): # Store the pcap file as a variable pcap = request.value usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['HTTP'] # Hash the pcap file try: md5hash = md5_for_file(pcap) except Exception as e: return response + UIMessage(str(e)) d = find_session(md5hash) pcap_id = d[0] else: pass # Find HTTP Requests pkts = rdpcap(pcap) http_requests = [] for p in pkts: if p.haslayer(HTTPRequest): timestamp = datetime.datetime.fromtimestamp( p.time).strftime('%Y-%m-%d %H:%M:%S.%f') r = p[HTTPRequest].Host if usedb > 0: http = OrderedDict({ 'PCAP ID': pcap_id, 'Time Stamp': timestamp, 'Type': 'HTTP Request', 'IP': { 'src': p[IP].src, 'dst': p[IP].dst }, 'HTTP': { 'Method': p[HTTPRequest].Method, 'URI': p[HTTPRequest].Path, 'Referer': p[HTTPRequest].Referer, 'Host': p[HTTPRequest].Host } }) # Check if record already exists s = x.HTTP.find({'Time Stamp': timestamp}).count() if s > 0: pass else: c.insert(http) if r not in http_requests: http_requests.append(r) else: pass for i in http_requests: h = Website(i) response += h return response
def dotransform(request, response): pcap = request.value folder = '' usedb = config['working/usedb'] # Check to see if we are using the database or not if usedb > 0: # Connect to the database so we can insert the record created below x = mongo_connect() c = x['STREAMS'] # Hash the pcap file try: md5hash = md5_for_file(pcap) d = find_session(md5hash) pcap_id = d[0] folder = d[2] except Exception as e: return response + UIMessage(str(e)) else: w = config['working/directory'].strip('\'') try: if w != '': w = w + '/' + str(uuid.uuid4())[:12].replace('-', '') if not os.path.exists(w): os.makedirs(w) folder = w else: return response + UIMessage( 'No working directory set, check your config file') except Exception as e: return response + UIMessage(e) # Create TCP/UDP stream files s = create_streams(pcap, folder) if usedb > 0: for i in s: # Create StreamID streamid = str(uuid.uuid4())[:8] # Get a count of packets available try: pkcount = packet_count(i) except Exception as e: return response + UIMessage(str(e)) # Get the start/end time of packets try: pcap_time = get_time(i) except Exception as e: return response + UIMessage(str(e)) # Hash the pcap file try: md5hash = md5_for_file(i) sha1hash = sha1_for_file(i) except Exception as e: return response + UIMessage(str(e)) # Pull out the details of the packets l = len(folder) + 1 raw = i[l:-5] pkt = raw.replace('-', ' ').replace(':', ' ').split() # Create the dictonary object to insert into database data = OrderedDict({ 'PCAP ID': pcap_id, 'Stream ID': streamid, 'Folder': folder, 'Packet Count': pkcount, 'File Name': i, 'First Packet': pcap_time[0], 'Last Packet': pcap_time[1], 'MD5 Hash': md5hash, 'SHA1 Hash': sha1hash, 'Packet': { 'Protocol': pkt[0], 'Source IP': pkt[1], 'Source Port': pkt[2], 'Destination IP': pkt[3], 'Destination Port': pkt[4] } }) # Check to see if the record exists try: t = x.STREAMS.find({"File Name": i}).count() if t > 0: pass else: c.insert(data) except Exception as e: return response + UIMessage(str(e)) else: pass # Create Maltego entities for each pcap file for p in s: e = pcapFile(p) response += e return response