def filter_pcap_helper(ip, data, packet):
# Set the TCP data
tcp = packet.data
src_ip = PcapHandler.inet_to_str(packet.src)
dst_ip = PcapHandler.inet_to_str(packet.dst)
#sport = packet.data.sport
#dport = packet.data.dport
if src_ip == ip or dst_ip == ip:
# print 'Found: ' + dst_ip
try:
request = dpkt.http.Request(tcp.data)
data = str(data).replace('[', '')
data = data.replace(']', '')
if 'GET ' in data:
data = data.replace('GET ', '')
elif 'POST ' in data:
data = data.replace('POST ', '')
data = data.replace(' ', '')
if data in request.uri:
return True
else:
# print 'Not matched: ' + ip + ', ' + data + ', ' + request.uri
return False
except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
return False
return False
def filter_pcap_helper(ip, port, packet):
src_ip = PcapHandler.inet_to_str(packet.src)
dst_ip = PcapHandler.inet_to_str(packet.dst)
sport = packet.data.sport
dport = packet.data.dport
if str(sport) == str(port) or str(dport) == str(port) \
and src_ip == ip or dst_ip == ip:
# print 'Found: ' + dst_ip
return True
return False
def pcap2jsons(pcap_dir, out_dir, filter_func=None, *args):
filtered = []
for root, dirs, files in os.walk(pcap_dir, topdown=True):
for name in files:
# print(os.path.join(root, name))
if str(name).endswith('.pcap'):
pcap_path = os.path.join(root, name)
if '/0/' in pcap_path:
label = 0
elif '/1/' in pcap_path:
label = 1
else:
label = 'unknown'
"""Open up a test pcap file and print out the packets"""
print pcap_path
filtered += PcapHandler.http_requests(pcap_path,
filter_flow=filter_func,
args=args)
print out_dir
if not os.path.exists(out_dir):
os.makedirs(out_dir)
for flow in filtered:
timestamp = flow['timestamp'].replace(':', '-')
timestamp = timestamp.replace('.', '-')
timestamp = timestamp.replace(' ', '_')
filename = str(flow['domain'] + '_' + timestamp + '.json').replace(
':', '_').replace('/', '_')
with open(os.path.join(out_dir, filename), 'w') as outfile:
try:
json.dump(flow, outfile)
except UnicodeDecodeError as e:
print e
return filtered
def event_duration(out_dir):
timestamps = []
for root, dirs, files in os.walk(out_dir, topdown=True):
for name in files:
# print(os.path.join(root, name))
if str(name).endswith('.pcap'):
pcap_path = os.path.join(root, name)
timestamps.append(PcapHandler.duration_pcap(pcap_path))
print timestamps
'''
def durations(base_dir, timestamps, label):
input_dir = os.path.join(base_dir, label)
for root, dirs, files in os.walk(input_dir, topdown=True):
for name in files:
# print(os.path.join(root, name))
if str(name).endswith('.pcap'):
pcap_path = os.path.join(root, name)
min_val, max_val = PcapHandler.duration_pcap(pcap_path)
data = dict()
data['min'] = datetime.datetime.utcfromtimestamp(min_val)
data['max'] = datetime.datetime.utcfromtimestamp(max_val)
data['category'] = label
timestamps.append(data)
def extract_flow_pcap_helper(taint, pcap_path):
'''
Given a taint record, extract the flow in the pcap file and output the pcap flow.
:param taint:
:return:
'''
print pcap_path
ip = taint['dst']
if 'data=' in taint['message']:
data = taint['message'].split('data=')[1]
elif 'data' in taint['message']:
data = taint['message'].split('data')[1]
else:
raise Exception
try:
#if True:
# Get filtered http requests based on Taintlogs (ip, data)
flows = PcapHandler.http_requests(
pcap_path,
filter_func=TaintDroidLogProcessor.filter_pcap,
args=[ip, data])
print len(flows)
# Output to pcaps
for flow in flows:
print flow
pkts = PcapHandler.get_packets(pcap_path)
PcapHandler.filter_pcap(os.path.dirname(pcap_path),
pkts,
flow['dest'],
flow['sport'],
tag=TaintDroidLogProcessor.gen_tag(
taint['src']))
return flows
#return PcapHandler.match_http_requests(pcap_path, TaintDroidLogProcessor.filter_pcap, [ip, data],
# gen_pcap=True, tag=TaintDroidLogProcessor.gen_tag(taint['src']))
except Exception as e:
print e
return []
def filter_tcp_streams(pcap_path,
out_dir,
json_dir,
gen_streams=False,
tag=''):
"""
Parse json and filter the pcap to generte subpcap
:param json_dir:
:param pcap_path:
:param out_dir:
:return:
"""
#pkts = PcapHandler.get_packets(pcap_path)
if gen_streams:
if not os.path.exists(out_dir):
os.makedirs(out_dir)
pkts = rdpcap(pcap_path)
jsons = dir2jsons(json_dir)
print len(jsons)
for json in jsons:
if json['src'] != '147.32.84.165' and json[
'dest'] != '147.32.84.165':
continue
print json
PcapHandler.filter_pcap(out_dir,
pkts,
json['dest'],
json['sport'],
tag=tag)
"""
streams = PcapHandler.tcp_streams(pcap_path)
jsons = dir2jsons(json_dir)
print len(jsons)
for json in jsons:
if json['src'] != '147.32.84.165' and json['dest'] != '147.32.84.165':
continue
print json
ip_src = stream['src']
ip_dest = stream['dest']
tcp_sport = stream['sport']
tcp_dport = stream['dport']
ip = json['dest']
port = json['sport']
for stream in streams:
if (ip_src == ip or ip_dest == ip) and (
tcp_dport == port or tcp_sport == port):
i = stream['index']
out_pcap = os.path.join(out_dir, ip + '_filtered_' + tag + '_' + str(port) + '.pcap')
cmd = 'tshark -r ' + pcap_path + ' -Y "tcp.stream==' + str(i) + '" -w ' + out_pcap
print cmd
os.system(cmd)
"""
return
ts_pks = []
for root, dirs, files in os.walk(out_dir, topdown=True):
for name in files:
# print(os.path.join(root, name))
if '_ts_' in name and str(name).endswith('.pcap'):
ts_pcap_path = os.path.join(root, name)
if os.path.exists(ts_pcap_path):
try:
pkts = rdpcap(pcap_path)
ts_pks.append(pkts)
except:
continue
PcapHandler.duration_pcap(ts_pcap_path)
if __name__ == '__main__':
'''
dir = '/mnt/Documents/flows/CTU-13/CTU-13-1/'
dir_0 = dir + '0/'
label = 'Ad' #TCP-CC' #SPAM'
for root, dirs, files in os.walk(dir_0, topdown=True):
for name in files:
# print(os.path.join(root, name))
if str(name).endswith('binetflow.2format'):
headers, csv_packets = read_csv(os.path.join(dir_0, name),
csv_filter_http, label=label)
flows = pcap2jsons(dir_0, dir_0 + '/' + label, filter_flow, csv_packets)
print len(csv_packets), len(flows)
for csv_packet in csv_packets:
print csv_packet['DstAddr'], csv_packet['Sport'], csv_packet['StartTime'], csv_packet['LastTime']
print '_________________________________'
for flow in flows:
print flow['dest'], flow['sport'], flow['uri']
'''
#dir_1 = '/mnt/Documents/flows/FlowIntent/Address'
#pcap2jsons(dir_1, dir_1)
label = 'Ad'
pcap_path = '/mnt/Documents/flows/Event/147-32-84-165/147-32-84-165.pcap'
#filter_tcp_streams(pcap_path,
# '/mnt/Documents/flows/Event/147-32-84-165/' + label, '/mnt/Documents/flows/Event/' + label,
# tag=label, gen_streams=True)
streams = PcapHandler.tcp_streams(
pcap_path, out_dir='/mnt/Documents/flows/Event/147-32-84-165/')