def get_frames_hashes(capfile, port='', stream='', options=[]):
frames_hashes = {}
#Run tshark once to get a list of frames with ssl app data in them
filterstr = 'ssl.record.content_type==23'
if (port):
filterstr += ' and tcp.port==' + str(port)
if (stream):
filterstr += (' and tcp.stream==' + stream)
try:
frames_str = tshark(capfile,field='frame.number', \
filter= filterstr,options=options)
except:
print 'Exception in tshark'
return -1
ssl_frames = shared.pisp(frames_str)
shared.debug(1, ['need to process this many frames:', len(ssl_frames)])
if not any(ssl_frames):
return None
for frame in ssl_frames:
frames_hashes[frame] = [
] #array will contain all hashes for that frame
ssl_app_data = tshark(capfile,field='ssl.app_data',frames=ssl_frames, \
options=options)
#ssl.app_data will return all encrypted segments separated by commas
#WITHIN each frame, and each frame consecutively is separated by a newline
#so first we split ONLY on the (platform dependent) newline to get
#a chunk of ssl app data for EACH FRAME
app_data_output = shared.pisp(ssl_app_data)
shared.debug(4, app_data_output)
#now app_data_output is a list, each element of which is the complete
#output of ssl.app_data for each frame consecutively
x = 0
for frame in ssl_frames:
#now we are looking at the ssl app data for a single frame -
#need to split it into segments
segments = app_data_output[x].split(',')
# make sure that segments is empty if there's no data!
segments = filter(None, segments)
frames_hashes[frame].extend(
get_ssl_hashes_from_ssl_app_data_list(segments))
shared.debug(3, ["Frame: ", frame, frames_hashes[frame]])
x += 1
return frames_hashes
def editcap(infile, outfile, reverse_flag=0, filter='', frames=[]):
editcap_exepath = g("Exepaths", "editcap_exepath")
frame_list = []
if reverse_flag:
args = [editcap_exepath, '-r', infile]
else:
args = [editcap_exepath, infile]
if (frames):
frame_list = frames
else:
tshark_out = tshark(infile, field='frame.number', filter=filter)
frame_list = shared.pisp(tshark_out)
shared.debug(3, ["Frames are: ", frame_list])
#TODO: This won't work if -r flag not used;
#may need some reconsideration
if (len(frame_list) > MAX_FRAME_FILTER):
editcap_inner(args, frame_list, outfile)
else:
args.append(outfile)
args.extend(frame_list)
shared.debug(2, ["Calling editcap with these arguments: ", args])
subprocess.check_output(args)
def editcap(infile, outfile, reverse_flag = 0, filter='', frames=[]):
editcap_exepath = g("Exepaths","editcap_exepath")
frame_list=[]
if reverse_flag:
args = [editcap_exepath, '-r', infile]
else:
args = [editcap_exepath,infile]
if (frames):
frame_list = frames
else:
tshark_out = tshark(infile,field = 'frame.number',filter = filter)
frame_list = shared.pisp(tshark_out)
shared.debug(3,["Frames are: ", frame_list])
#TODO: This won't work if -r flag not used;
#may need some reconsideration
if (len(frame_list)>MAX_FRAME_FILTER):
editcap_inner(args,frame_list,outfile)
else:
args.append(outfile)
args.extend(frame_list)
shared.debug(2,["Calling editcap with these arguments: ",args])
subprocess.check_output(args)
def get_ssl_hashes_from_capfile(capfile,
port=-1,
stream='',
options=[],
frames=[]):
frames_str = ''
ssl_frames = []
#Run tshark to get a list of frames with ssl app data in them
#EDITED to test escrow
filterstr = build_filterstr(stream=stream, port=port)
if (frames):
ssl_frames = frames
else:
try:
frames_str = tshark(capfile,field='frame.number', \
filter= filterstr,options=options)
except:
#this could be caused by a corrupt file from stcppipe
#or by a malformed query string,etc. - but in the former
#case we should NOT exit, hence this approach
shared.debug(0, ["tshark failed - see stderr for message"])
shared.debug(0, ["return code from tshark: ", frames_str])
return None
ssl_frames = shared.pisp(frames_str)
#gracefully handle null result (i.e. blank tshark output):
ssl_frames = filter(None, ssl_frames)
if not ssl_frames:
return None
#Now we definitely have ssl frames in this capture file
shared.debug(1, ['need to process this many frames:', len(ssl_frames)])
ssl_app_data = tshark(capfile,field='ssl.app_data',frames=ssl_frames,\
options=options)
#ssl.app_data will return all encrypted segments separated by commas
#but also, lists of segments from different frames will be separated by
#platform dependent newlines
ssl_app_data_list = shared.pisp(ssl_app_data.replace(',', shared.PINL))
#remove any blank OR duplicate entries in the ssl app data list
ssl_app_data_list = filter(None, list(set(ssl_app_data_list)))
shared.debug(4,
["Full dump of ssl application data:\n", ssl_app_data_list])
shared.debug(1,["Length of list of ssl segments for file ",capfile," was: " \
,len(ssl_app_data_list)])
return get_ssl_hashes_from_ssl_app_data_list(ssl_app_data_list)
def get_frames_hashes(capfile,port='',stream='',options=[]):
frames_hashes = {}
#Run tshark once to get a list of frames with ssl app data in them
filterstr = 'ssl.record.content_type==23'
if (port):
filterstr += ' and tcp.port=='+str(port)
if (stream):
filterstr += (' and tcp.stream==' + stream)
try:
frames_str = tshark(capfile,field='frame.number', \
filter= filterstr,options=options)
except:
print 'Exception in tshark'
return -1
ssl_frames = shared.pisp(frames_str)
shared.debug(1,['need to process this many frames:', len(ssl_frames)])
if not any(ssl_frames):
return None
for frame in ssl_frames:
frames_hashes[frame]= [] #array will contain all hashes for that frame
ssl_app_data = tshark(capfile,field='ssl.app_data',frames=ssl_frames, \
options=options)
#ssl.app_data will return all encrypted segments separated by commas
#WITHIN each frame, and each frame consecutively is separated by a newline
#so first we split ONLY on the (platform dependent) newline to get
#a chunk of ssl app data for EACH FRAME
app_data_output = shared.pisp(ssl_app_data)
shared.debug(4,app_data_output)
#now app_data_output is a list, each element of which is the complete
#output of ssl.app_data for each frame consecutively
x=0
for frame in ssl_frames:
#now we are looking at the ssl app data for a single frame -
#need to split it into segments
segments = app_data_output[x].split(',')
# make sure that segments is empty if there's no data!
segments = filter(None,segments)
frames_hashes[frame].extend(get_ssl_hashes_from_ssl_app_data_list(segments))
shared.debug(3,["Frame: ", frame, frames_hashes[frame]])
x += 1
return frames_hashes
def get_ssl_hashes_from_capfile(capfile,port=-1,stream='',options=[],frames=[]):
frames_str=''
ssl_frames=[]
#Run tshark to get a list of frames with ssl app data in them
#EDITED to test escrow
filterstr = build_filterstr(stream=stream,port=port)
if (frames):
ssl_frames=frames
else:
try:
frames_str = tshark(capfile,field='frame.number', \
filter= filterstr,options=options)
except:
#this could be caused by a corrupt file from stcppipe
#or by a malformed query string,etc. - but in the former
#case we should NOT exit, hence this approach
shared.debug(0,["tshark failed - see stderr for message"])
shared.debug(0,["return code from tshark: ",frames_str])
return None
ssl_frames = shared.pisp(frames_str)
#gracefully handle null result (i.e. blank tshark output):
ssl_frames = filter(None,ssl_frames)
if not ssl_frames:
return None
#Now we definitely have ssl frames in this capture file
shared.debug(1,['need to process this many frames:', len(ssl_frames)])
ssl_app_data = tshark(capfile,field='ssl.app_data',frames=ssl_frames,\
options=options)
#ssl.app_data will return all encrypted segments separated by commas
#but also, lists of segments from different frames will be separated by
#platform dependent newlines
ssl_app_data_list = shared.pisp(ssl_app_data.replace(',',shared.PINL))
#remove any blank OR duplicate entries in the ssl app data list
ssl_app_data_list = filter(None,list(set(ssl_app_data_list)))
shared.debug(4,["Full dump of ssl application data:\n",ssl_app_data_list])
shared.debug(1,["Length of list of ssl segments for file ",capfile," was: " \
,len(ssl_app_data_list)])
return get_ssl_hashes_from_ssl_app_data_list(ssl_app_data_list)
def debug_get_streams(capfile, filterstr='', options=[]):
streams_output = tshark(capfile,field='tcp.stream',filter=filterstr,\
options=options)
streams = list(set(shared.pisp(streams_output)))
shared.debug(2, ["This is the list of tcp streams: ", streams])
return streams
def get_GET_http_requests(capfile, options):
hexdigits = shared.hexdigits
frames_list = shared.pisp(tshark(capfile,filter='ssl and http.request',\
field='frame.number',options=options))
#data structure to store all the GETs found:
if not any(frames_list): return None
GETs = {}
for frame in frames_list:
binary_html = bytearray()
#TODO: this is too slow as it calls tshark many many times
#(and with decryption enabled which REALLY slows it down) -
#we will have to find a way
#to parse -x output for all frames at once, but it will be nasty!
ascii_dump = tshark(capfile,
frames=[frame],
field='x',
options=options)
if ascii_dump == '':
shared.debug(0, ['empty frame dump'])
return
#the section we're looking for will usually start "Reassembled SSL.."
#but if no reassembly was necessary, it will start with "Decrypted.."
#(requests are not ever compressed AFAIK, although the technology exists)
#TODO check this?
reassembled_pos = ascii_dump.rfind('Reassembled SSL')
if reassembled_pos == -1:
reassembled_pos = ascii_dump.rfind('Decrypted SSL')
if reassembled_pos != -1:
lines = ascii_dump[reassembled_pos:].split('\n')
line_length = len(lines[1]) + 1
line_numbering_length = len(lines[1].split()[0])
hexlist = [line.split()[1:17] for line in lines[1:]]
#flatten the nested lists acc.to http://stackoverflow.com/\
#questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
flathexlist = [item for sublist in hexlist for item in sublist]
#convert the list into a single string
hexstring = ''.join(flathexlist)
#modified to read GET headers
start_pos_in_hex = 0
#The very first hex is line numbering,it is followed by 2 spaces
#each hex number in a line takes up 2 alphanum chars + 1 space char
#we skip the very first line 'Reassembled SSL ...' by finding a newline.
newline_offset = ascii_dump[reassembled_pos:].find('\n')
body_start = reassembled_pos + newline_offset + 1 + line_numbering_length + 2
lines = ascii_dump[body_start:].split('\n')
#treat the first line specially - not needed now, remove TODO
#this is different to HTML case, because we DO want the headers
binary_html += bytearray.fromhex(lines[0][0:48])
for line in lines[1:]:
#convert ascii representation of hex into binary
#only deal with lines where first 4 chars are hexdigits
if all(c in hexdigits for c in line[:4]):
m_array = bytearray.fromhex(line[6:54])
binary_html += m_array
else:
break
if len(binary_html) == 0:
shared.debug(0, ['empty binary array'])
return
#we now have the entire request stored in binary_html; check for GET
if binary_html.find('GET') == 0:
GETs[frame] = str(binary_html)
#now we have all frames with gets in a list of dicts frame num: request
return GETs
def debug_get_streams(capfile,filterstr='',options=[]):
streams_output = tshark(capfile,field='tcp.stream',filter=filterstr,\
options=options)
streams = list(set(shared.pisp(streams_output)))
shared.debug(2,["This is the list of tcp streams: ", streams])
return streams
def get_GET_http_requests(capfile,options):
hexdigits = shared.hexdigits
frames_list = shared.pisp(tshark(capfile,filter='ssl and http.request',\
field='frame.number',options=options))
#data structure to store all the GETs found:
if not any(frames_list): return None
GETs = {}
for frame in frames_list:
binary_html = bytearray()
#TODO: this is too slow as it calls tshark many many times
#(and with decryption enabled which REALLY slows it down) -
#we will have to find a way
#to parse -x output for all frames at once, but it will be nasty!
ascii_dump = tshark(capfile,frames=[frame],field='x',options=options)
if ascii_dump == '':
shared.debug(0,['empty frame dump'])
return
#the section we're looking for will usually start "Reassembled SSL.."
#but if no reassembly was necessary, it will start with "Decrypted.."
#(requests are not ever compressed AFAIK, although the technology exists)
#TODO check this?
reassembled_pos = ascii_dump.rfind('Reassembled SSL')
if reassembled_pos == -1:
reassembled_pos = ascii_dump.rfind('Decrypted SSL')
if reassembled_pos != -1:
lines = ascii_dump[reassembled_pos:].split('\n')
line_length = len(lines[1])+1
line_numbering_length = len(lines[1].split()[0])
hexlist = [line.split()[1:17] for line in lines[1:]]
#flatten the nested lists acc.to http://stackoverflow.com/\
#questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
flathexlist = [item for sublist in hexlist for item in sublist]
#convert the list into a single string
hexstring = ''.join(flathexlist)
#modified to read GET headers
start_pos_in_hex = 0
#The very first hex is line numbering,it is followed by 2 spaces
#each hex number in a line takes up 2 alphanum chars + 1 space char
#we skip the very first line 'Reassembled SSL ...' by finding a newline.
newline_offset = ascii_dump[reassembled_pos:].find('\n')
body_start = reassembled_pos+newline_offset+1+line_numbering_length+2
lines = ascii_dump[body_start:].split('\n')
#treat the first line specially - not needed now, remove TODO
#this is different to HTML case, because we DO want the headers
binary_html += bytearray.fromhex(lines[0][0:48])
for line in lines[1:]:
#convert ascii representation of hex into binary
#only deal with lines where first 4 chars are hexdigits
if all(c in hexdigits for c in line [:4]):
m_array = bytearray.fromhex(line[6:54])
binary_html += m_array
else:
break
if len(binary_html) == 0:
shared.debug(0,['empty binary array'])
return
#we now have the entire request stored in binary_html; check for GET
if binary_html.find('GET') ==0:
GETs[frame]=str(binary_html)
#now we have all frames with gets in a list of dicts frame num: request
return GETs