def dump(module_data, d):
if module_data['mongo']:
module_data['db'].insert(d)
chop.prnt(d)
chop.json(d)
if module_data['carve_request'] and 'body' in d['request']:
chop.prnt(
"DUMPING REQUEST: %s (%i)" %
(sanitize_filename(d['request']['uri']['path'][1:] + '.request.' +
str(module_data['counter'])),
len(d['request']['body'])))
chop.savefile(
sanitize_filename(d['request']['uri']['path'][1:] + '.request.' +
str(module_data['counter'])),
d['request']['body'])
module_data['counter'] += 1
if module_data['carve_response'] and 'body' in d['response']:
chop.prnt(
"DUMPING RESPONSE: %s (%i)" %
(sanitize_filename(d['request']['uri']['path'][1:] + '.response.' +
str(module_data['counter'])),
len(d['response']['body'])))
chop.savefile(
sanitize_filename(d['request']['uri']['path'][1:] + '.response.' +
str(module_data['counter'])),
d['response']['body'])
module_data['counter'] += 1
# In case pipelining is going on remove these.
d['request'] = {'headers': {}}
d['response'] = {'headers': {}}
def dump(module_data, d):
if module_data['mongo']:
module_data['db'].insert(d)
if module_data['carve_request'] and 'body' in d['request']:
chop.prnt("DUMPING REQUEST: %s (%i)" % (sanitize_filename(d['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), len(d['request']['body'])))
chop.savefile(sanitize_filename(d['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), d['request']['body'])
module_data['counter'] += 1
if module_data['carve_response'] and 'body' in d['response']:
chop.prnt("DUMPING RESPONSE: %s (%i)" % (sanitize_filename(d['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), len(d['response']['body'])))
chop.savefile(sanitize_filename(d['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), d['response']['body'])
module_data['counter'] += 1
# Convert the body to base64 encoded data, if it exists.
if 'body' in d['request']:
d['request']['body'] = b64encode(d['request']['body'])
d['request']['body_encoding'] = 'base64'
if 'body' in d['response']:
d['response']['body'] = b64encode(d['response']['body'])
d['response']['body_encoding'] = 'base64'
chop.prnt(d)
chop.json(d)
# In case pipelining is going on remove these.
d['request'] = { 'headers': {} }
d['response'] = { 'headers': {} }
def get_name_and_size(msg, tcp):
(hsize, lsize) = struct.unpack("<II", msg[:8])
size = winsizeize(hsize, lsize)
fname = msg[8:-1]
if tcp.module_data["savefiles"]:
tcp.stream_data["fsize"] = size
tcp.stream_data["fname"] = sanitize_filename(fname)
chop.prnt(tcp.stream_data["fname"])
tcp.stream_data["byteswritten"] = 0
return (fname, size)
def get_name_and_size(msg, tcp):
(hsize, lsize) = struct.unpack('<II', msg[:8])
size = winsizeize(hsize, lsize)
fname = msg[8:-1]
if tcp.module_data['savefiles']:
tcp.stream_data['fsize'] = size
tcp.stream_data['fname'] = sanitize_filename(fname)
chop.prnt(tcp.stream_data['fname'])
tcp.stream_data['byteswritten'] = 0
return (fname, size)
def get_name_and_size(msg, tcp):
(hsize, lsize) = struct.unpack('<II', msg[:8])
size = winsizeize(hsize, lsize)
fname = msg[8:-1]
if tcp.module_data['savefiles']:
tcp.stream_data['fsize'] = size
tcp.stream_data['fname'] = sanitize_filename(fname)
chop.prnt(tcp.stream_data['fname'])
tcp.stream_data['byteswritten'] = 0
return (fname, size)
def dump(module_data, d):
if module_data['prnt']:
chop.prnt(d)
if module_data['mongo']:
module_data['db'].insert(d)
if module_data['json']:
chop.json(d)
if module_data['carve_request'] and 'body' in d['request']:
chop.prnt("DUMPING REQUEST: %s (%i)" % (sanitize_filename(d['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), len(d['request']['body'])))
chop.savefile(sanitize_filename(d['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), d['request']['body'])
module_data['counter'] += 1
if module_data['carve_response'] and 'body' in d['response']:
chop.prnt("DUMPING RESPONSE: %s (%i)" % (sanitize_filename(d['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), len(d['response']['body'])))
chop.savefile(sanitize_filename(d['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), d['response']['body'])
module_data['counter'] += 1
# In case pipelining is going on remove these.
d['request'] = { 'headers': {} }
d['response'] = { 'headers': {} }
def handleProtocol(protocol):
if protocol.type != 'http':
chop.prnt("Error")
return
module_data = protocol.module_data
data = {'request': protocol.clientData, 'response': protocol.serverData}
if data['request']['body'] is None:
del data['request']['body']
del data['request']['body_hash']
elif module_data['hash_body']:
del data['request']['body']
if data['response']['body'] is None:
del data['response']['body']
del data['response']['body_hash']
elif module_data['hash_body']:
del data['response']['body']
del data['request']['truncated']
del data['request']['body_len']
del data['request']['hash_fn']
del data['response']['truncated']
del data['response']['body_len']
del data['response']['hash_fn']
fields = module_data['fields']
if fields:
req_fields = fields + ['uri', 'method']
new_headers = {}
for header in data['request']['headers']:
if header in req_fields:
new_headers[header] = data['request']['headers'][header]
for element in data['request'].keys():
if element not in req_fields:
del data['request'][element]
#Set the new headers dictionary
data['request']['headers'] = new_headers
res_fields = fields + ['status']
new_headers = {}
for header in data['response']['headers']:
if header in res_fields:
new_headers[header] = data['response']['headers'][header]
for element in data['response'].keys():
if element not in res_fields:
del data['response'][element]
data['response']['headers'] = new_headers
if module_data['carve_request'] and 'body' in data['request']:
chop.prnt("DUMPING REQUEST: %s (%i)" % (sanitize_filename(data['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), len(data['request']['body'])))
chop.savefile(sanitize_filename(data['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), data['request']['body'])
module_data['counter'] += 1
if module_data['carve_response'] and 'body' in data['response']:
chop.prnt("DUMPING RESPONSE: %s (%i)" % (sanitize_filename(data['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), len(data['response']['body'])))
chop.savefile(sanitize_filename(data['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), data['response']['body'])
module_data['counter'] += 1
# Convert the body to base64 encoded data, if it exists.
if 'body' in data['request']:
data['request']['body'] = b64encode(data['request']['body'])
data['request']['body_encoding'] = 'base64'
if 'body' in data['response']:
data['response']['body'] = b64encode(data['response']['body'])
data['response']['body_encoding'] = 'base64'
chop.prnt(data)
chop.json(data)
return
def handleProtocol(protocol):
if protocol.type != 'http':
chop.prnt("Error")
return
module_data = protocol.module_data
data = {'request': protocol.clientData, 'response': protocol.serverData}
if data['request']['body'] is None:
del data['request']['body']
del data['request']['body_hash']
elif module_data['hash_body']:
del data['request']['body']
if data['response']['body'] is None:
del data['response']['body']
del data['response']['body_hash']
elif module_data['hash_body']:
del data['response']['body']
del data['request']['truncated']
del data['request']['body_len']
del data['request']['hash_fn']
del data['response']['truncated']
del data['response']['body_len']
del data['response']['hash_fn']
fields = module_data['fields']
if fields:
req_fields = fields + ['uri', 'method']
new_headers = {}
for header in data['request']['headers']:
if header in req_fields:
new_headers[header] = data['request']['headers'][header]
for element in data['request'].keys():
if element not in req_fields:
del data['request'][element]
#Set the new headers dictionary
data['request']['headers'] = new_headers
res_fields = fields + ['status']
new_headers = {}
for header in data['response']['headers']:
if header in res_fields:
new_headers[header] = data['response']['headers'][header]
for element in data['response'].keys():
if element not in res_fields:
del data['response'][element]
data['response']['headers'] = new_headers
if module_data['carve_request'] and 'body' in data['request']:
chop.prnt(
"DUMPING REQUEST: %s (%i)" %
(sanitize_filename(data['request']['uri']['path'][1:] +
'.request.' + str(module_data['counter'])),
len(data['request']['body'])))
chop.savefile(
sanitize_filename(data['request']['uri']['path'][1:] +
'.request.' + str(module_data['counter'])),
data['request']['body'])
module_data['counter'] += 1
if module_data['carve_response'] and 'body' in data['response']:
chop.prnt(
"DUMPING RESPONSE: %s (%i)" %
(sanitize_filename(data['request']['uri']['path'][1:] +
'.response.' + str(module_data['counter'])),
len(data['response']['body'])))
chop.savefile(
sanitize_filename(data['request']['uri']['path'][1:] +
'.response.' + str(module_data['counter'])),
data['response']['body'])
module_data['counter'] += 1
# Convert the body to base64 encoded data, if it exists.
if 'body' in data['request']:
data['request']['body'] = b64encode(data['request']['body'])
data['request']['body_encoding'] = 'base64'
if 'body' in data['response']:
data['response']['body'] = b64encode(data['response']['body'])
data['response']['body_encoding'] = 'base64'
chop.prnt(data)
chop.json(data)
return
def _stream_ended_(frame, direction, tcp):
hash_fn = tcp.module_data['options']['hash_fn']
transaction = {
'timestamp': tcp.stream_data['stream_cache'][frame.stream_id]['start'],
'request': {
'truncated': False, #TODO support this
'body': None,
'body_len': 0,
'body_hash': '',
'hash_fn': hash_fn,
'protocol': '2'
},
'response': {
'truncated': False,
'body': None,
'body_len': 0,
'body_hash': '',
'hash_fn': hash_fn,
}
}
if tcp.stream_data['stream_cache'][frame.stream_id][direction][
'stream_ended'] and tcp.stream_data['stream_cache'][
frame.stream_id][_opposite_direction_(
direction)]['stream_ended']:
#chop.prnt("Stream: %d" % (frame.stream_id))
for d in ['request', 'response']:
headers = copy.deepcopy(
tcp.stream_data['stream_cache'][frame.stream_id][d]['headers'])
if d == 'request':
headers[':stream_id'] = frame.stream_id
method = headers.get(':method', None)
path = headers.get(':path', None)
if ':method' in headers:
del headers[':method']
if ':path' in headers:
del headers[':path']
transaction[d]['method'] = method
transaction[d]['uri'] = path
else:
status = headers.get(':status', None)
if ':status' in headers:
del headers[':status']
transaction[d]['status'] = status
transaction[d]['headers'] = headers
#chop.prnt("\tHeaders: %s" % (headers))
if tcp.stream_data['stream_cache'][
frame.stream_id][d]['data'] is not None:
content_encoding = headers.get('content-encoding', None)
mimetype = headers.get('content-type', None)
if mimetype is not None:
mimetype = mimetype.split(';', 1)[0]
content_disposition = headers.get('content-disposition', None)
if content_encoding == 'gzip':
try:
dataStream = BytesIO(tcp.stream_data['stream_cache'][
frame.stream_id][d]['data'])
gdata = gzip.GzipFile(fileobj=dataStream, mode='rb')
data = gdata.read()
except Exception as e:
chop.prnt("Warning: Unable to gzip file")
data = tcp.stream_data['stream_cache'][
frame.stream_id][d]['data']
else:
data = tcp.stream_data['stream_cache'][
frame.stream_id][d]['data']
if mimetype is None:
try:
import magic
except ImportError:
pass
else:
try:
mimetype = magic.from_buffer(data, mime=True)
except Exception as e:
chop.prnt(
"Warning: Unable to get mime type of file: %s"
% (str(e)))
filename = 'noname'
if d == 'response':
if content_disposition is None:
if transaction['request']['uri'] is not None:
raw_path = transaction['request']['uri']
raw_path = raw_path.split('?', 1)[0]
path_parts = os.path.split(raw_path)
outPath = None
while (len(path_parts) > 0):
if path_parts[-1] == '':
path_parts = path_parts[:-1]
continue
else:
outPath = os.path.basename(path_parts[-1])
break
if outPath is None or outPath == '':
outPath = 'index'
filename = sanitize_filename(outPath)
else:
filename = sanitize_filename(content_disposition)
#chop.prnt("\tData Name: %s, Length: %d, Type: %s" % (filename, len(data), mimetype))
((src, sport), (dst, dport)) = parse_addr(tcp)
chop.savefile(
"%d-%s-%d-%s-%d-%d-%s" %
(tcp.timestamp, src, sport, dst, dport, frame.stream_id,
filename), data)
transaction[d]['body'] = data
transaction[d]['body_len'] = len(data)
transaction[d]['body_hash'] = __hash_function__(
data).hexdigest()
#chop.prnt("\n")
#chop.prnt('\n')
del tcp.stream_data['stream_cache'][frame.stream_id]
#chop.prnt(tcp.stream_data['stream_cache'].keys())
#chop.prnt(json.dumps(transaction, indent=4))
chopp = ChopProtocol('http')
chopp.setClientData(transaction['request'])
chopp.setServerData(transaction['response'])
chopp.setTimeStamp(transaction['timestamp'])
chopp.setAddr(tcp.addr)
chopp.flowStart = tcp.stream_data['flowStart']
return chopp
else:
return None
