def teardownProtocol(protocol): if protocol.type != 'http': chop.prnt("Error") return module_data = protocol.module_data data = {'request': protocol.clientData, 'response': protocol.serverData} if module_data['base64_encode']: if (data['request'] is not None and 'body' in data['request'] and data['request']['body'] is not None): data['request']['body'] = b64encode(data['request']['body']) data['request']['body_encoding'] = 'base64' if (data['response'] is not None and 'body' in data['response'] and data['response']['body'] is not None): data['response']['body'] = b64encode(data['response']['body']) data['response']['body_encoding'] = 'base64' chopp = ChopProtocol('http_meta') chopp.data = data chopp.flowStart = protocol.flowStart chopp.setTimeStamp(protocol.timestamp) chopp.setAddr(protocol.addr) return chopp
def handleStream(tcp): chopp = ChopProtocol('http') ((src, sport), (dst, dport)) = parse_addr(tcp) tcp.stream_data['htpy_obj'].timestamp = tcp.timestamp if tcp.server.count_new > 0: if tcp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s (%i)" % (src, sport, dst, dport, tcp.server.count_new)) if tcp.server.count_new == 1 and tcp.module_data['options']['beast']: tcp.discard(0) else: data_size = tcp.server.count - tcp.server.offset try: tcp.stream_data['connparser'].\ req_data(tcp.server.data[:data_size]) except htpy.stop: tcp.stop() except htpy.error: if tcp.module_data['options']['verbose']: chop.tsprnt("Stream error in htpy.") tcp.stop() tcp.discard(data_size) elif tcp.client.count_new > 0: if tcp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s (%i)" % (src, sport, dst, dport, tcp.client.count_new)) if tcp.client.count_new == 1 and tcp.module_data['options']['beast']: tcp.discard(0) else: data_size = tcp.client.count - tcp.client.offset try: tcp.stream_data['connparser'].\ res_data(tcp.client.data[:data_size]) except htpy.stop: tcp.stop() except htpy.error: if tcp.module_data['options']['verbose']: chop.tsprnt("Stream error in htpy.") tcp.stop() tcp.discard(data_size) if tcp.stream_data['htpy_obj'].ready: trans = tcp.stream_data['htpy_obj'].transaction chopp.setClientData(trans['request']) chopp.setServerData(trans['response']) chopp.setTimeStamp(trans['timestamp']) chopp.setAddr(tcp.addr) chopp.flowStart = tcp.stream_data['htpy_obj'].flowStart tcp.stream_data['htpy_obj'].ready = False tcp.stream_data['htpy_obj'].temp = {} tcp.stream_data['htpy_obj'].transaction = {} return chopp return None
def handleProtocol(chopp): if chopp.type != 'sslim': return stream_data = chopp.stream_data if 'htpy_obj' not in stream_data: stream_data['htpy_obj'] = { 'options': chopp.module_data['options'], 'timestamp': None, 'temp': {}, 'transaction': {}, 'lines': Queue.Queue(), 'ready': False, 'flowStart': chopp.timestamp } stream_data['connparser'] = register_connparser() stream_data['connparser'].set_obj(stream_data['htpy_obj']) ((src, sport),(dst,dport)) = chopp.addr stream_data['htpy_obj']['timestamp'] = chopp.timestamp if chopp.clientData: if chopp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s" % (src, sport, dst, dport)) try: stream_data['connparser'].req_data(chopp.clientData) except htpy.stop: chopp.stop() except htpy.error: chop.prnt("Stream error in htpy.") chopp.stop() return if chopp.serverData: if chopp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s" % (dst, dport, src, sport)) try: stream_data['connparser'].res_data(chopp.serverData) except htpy.stop: chopp.stop() except htpy.error: chop.prnt("Stream error in htpy.") chopp.stop() return if stream_data['htpy_obj']['ready']: new_chopp = ChopProtocol('http') trans = stream_data['htpy_obj']['transaction'] new_chopp.setClientData(trans['request']) new_chopp.setServerData(trans['response']) new_chopp.setTimeStamp(trans['timestamp']) new_chopp.setAddr(chopp.addr) new_chopp.flowStart = stream_data['htpy_obj']['flowStart'] stream_data['htpy_obj']['ready'] = False return new_chopp
def handleProtocol(chopp): if chopp.type != 'sslim': return stream_data = chopp.stream_data if 'htpy_obj' not in stream_data: stream_data['htpy_obj'] = { 'options': chopp.module_data['options'], 'timestamp': None, 'temp': {}, 'transaction': {}, 'lines': Queue.Queue(), 'ready': False, 'flowStart': chopp.timestamp } stream_data['connparser'] = register_connparser() stream_data['connparser'].set_obj(stream_data['htpy_obj']) ((src, sport), (dst, dport)) = chopp.addr stream_data['htpy_obj']['timestamp'] = chopp.timestamp if chopp.clientData: if chopp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s" % (src, sport, dst, dport)) try: stream_data['connparser'].req_data(chopp.clientData) except htpy.stop: chopp.stop() except htpy.error: chop.prnt("Stream error in htpy.") chopp.stop() return if chopp.serverData: if chopp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s" % (dst, dport, src, sport)) try: stream_data['connparser'].res_data(chopp.serverData) except htpy.stop: chopp.stop() except htpy.error: chop.prnt("Stream error in htpy.") chopp.stop() return if stream_data['htpy_obj']['ready']: new_chopp = ChopProtocol('http') trans = stream_data['htpy_obj']['transaction'] new_chopp.setClientData(trans['request']) new_chopp.setServerData(trans['response']) new_chopp.setTimeStamp(trans['timestamp']) new_chopp.setAddr(chopp.addr) new_chopp.flowStart = stream_data['htpy_obj']['flowStart'] stream_data['htpy_obj']['ready'] = False return new_chopp
def handleProtocol(chopp): if chopp.type != 'sslim': return stream_data = chopp.stream_data if 'htpy_obj' not in stream_data: stream_data['htpy_obj'] = \ __htpyObj__(chopp.module_data['options'], chopp.timestamp) stream_data['connparser'] = register_connparser() stream_data['connparser'].set_obj(stream_data['htpy_obj']) ((src, sport), (dst, dport)) = chopp.addr stream_data['htpy_obj'].timestamp = chopp.timestamp if chopp.clientData: if chopp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s" % (src, sport, dst, dport)) try: stream_data['connparser'].req_data(chopp.clientData) except htpy.stop: chopp.stop() except htpy.error: if chopp.module_data['options']['verbose']: chop.tsprnt("Stream error in htpy.") chopp.stop() return if chopp.serverData: if chopp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s" % (dst, dport, src, sport)) try: stream_data['connparser'].res_data(chopp.serverData) except htpy.stop: chopp.stop() except htpy.error: if chopp.module_data['options']['verbose']: chop.tsprnt("Stream error in htpy.") chopp.stop() return if stream_data['htpy_obj'].ready: new_chopp = ChopProtocol('http') trans = stream_data['htpy_obj'].transaction new_chopp.setClientData(trans['request']) new_chopp.setServerData(trans['response']) new_chopp.setTimeStamp(trans['timestamp']) new_chopp.setAddr(chopp.addr) new_chopp.flowStart = stream_data['htpy_obj'].flowStart stream_data['htpy_obj'].ready = False stream_data['htpy_obj'].temp = {} stream_data['htpy_obj'].transaction = {} return new_chopp
def teardown(tcp): chopp = ChopProtocol('http') ((src, sport), (dst, dport)) = tcp.addr tcp.stream_data['htpy_obj'].timestamp = tcp.timestamp # There's data collected in temp if len(tcp.stream_data['htpy_obj'].temp.keys()) > 1: # we don't care if only start is populated t = tcp.stream_data['htpy_obj'].temp if 'request' in t: if len(t['request'].keys()) == 0: try: req = tcp.stream_data['htpy_obj'].lines.get(False) except Queue.Empty: req = None else: req = t['request'] if 'tmp_hash' in t['request']: if t['request']['body_len'] > 0: t['request']['body_hash'] = \ t['request']['tmp_hash'].hexdigest() else: t['request']['body_hash'] = "" del t['request']['tmp_hash'] if 'response' in t: resp = t['response'] if 'tmp_hash' in t['response']: if t['response']['body_len'] > 0: t['response']['body_hash'] = \ t['response']['tmp_hash'].hexdigest() else: t['response']['body_hash'] = "" del t['response']['tmp_hash'] else: resp = None if req is not None or resp is not None: chopp.setClientData(req) chopp.setServerData(resp) chopp.setTimeStamp(t['start']) chopp.setAddr(tcp.addr) chopp.setTeardown() chopp.flowStart = tcp.stream_data['htpy_obj'].flowStart tcp.stream_data['htpy_obj'].ready = False tcp.stream_data['htpy_obj'].temp = {} tcp.stream_data['htpy_obj'].transaction = {} return chopp return None
def handleStream(tcp): chopp = ChopProtocol('http') ((src, sport), (dst, dport)) = parse_addr(tcp) tcp.stream_data['htpy_obj']['timestamp'] = tcp.timestamp if tcp.server.count_new > 0: if tcp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s (%i)" % (src, sport, dst, dport, tcp.server.count_new)) try: tcp.stream_data['connparser'].req_data( tcp.server.data[:tcp.server.count_new]) except htpy.stop: tcp.stop() except htpy.error: chop.prnt("Stream error in htpy.") tcp.stop() tcp.discard(tcp.server.count_new) elif tcp.client.count_new > 0: if tcp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s (%i)" % (src, sport, dst, dport, tcp.client.count_new)) try: tcp.stream_data['connparser'].res_data( tcp.client.data[:tcp.client.count_new]) except htpy.stop: tcp.stop() except htpy.error: chop.prnt("Stream error in htpy.") tcp.stop() tcp.discard(tcp.client.count_new) if tcp.stream_data['htpy_obj']['ready']: trans = tcp.stream_data['htpy_obj']['transaction'] chopp.setClientData(trans['request']) chopp.setServerData(trans['response']) chopp.setTimeStamp(trans['timestamp']) chopp.setAddr(tcp.addr) chopp.flowStart = tcp.stream_data['htpy_obj']['flowStart'] tcp.stream_data['htpy_obj']['ready'] = False return chopp return None
def handleStream(tcp): chopp = ChopProtocol('http') ((src, sport), (dst, dport)) = parse_addr(tcp) tcp.stream_data['htpy_obj']['timestamp'] = tcp.timestamp if tcp.server.count_new > 0: if tcp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s (%i)" % (src, sport, dst, dport, tcp.server.count_new)) try: tcp.stream_data['connparser'].req_data(tcp.server.data[:tcp.server.count_new]) except htpy.stop: tcp.stop() except htpy.error: chop.prnt("Stream error in htpy.") tcp.stop() tcp.discard(tcp.server.count_new) elif tcp.client.count_new > 0: if tcp.module_data['options']['verbose']: chop.tsprnt("%s:%s->%s:%s (%i)" % (src, sport, dst, dport, tcp.client.count_new)) try: tcp.stream_data['connparser'].res_data(tcp.client.data[:tcp.client.count_new]) except htpy.stop: tcp.stop() except htpy.error: chop.prnt("Stream error in htpy.") tcp.stop() tcp.discard(tcp.client.count_new) if tcp.stream_data['htpy_obj']['ready']: trans = tcp.stream_data['htpy_obj']['transaction'] chopp.setClientData(trans['request']) chopp.setServerData(trans['response']) chopp.setTimeStamp(trans['timestamp']) chopp.setAddr(tcp.addr) chopp.flowStart = tcp.stream_data['htpy_obj']['flowStart'] tcp.stream_data['htpy_obj']['ready'] = False return chopp return None
def teardownProtocol(chopp): if chopp.type != 'sslim': return stream_data = chopp.stream_data #sslim returns an empty object on teardown if 'htpy_obj' not in stream_data: return hchopp = ChopProtocol('http') ((src, sport), (dst, dport)) = chopp.addr stream_data['htpy_obj'].timestamp = chopp.timestamp #There's data collected in temp if len(stream_data['htpy_obj'].temp.keys() ) > 1: #we don't care if only start is populated t = stream_data['htpy_obj'].temp if 'request' in t: if len(t['request'].keys()) == 0: try: req = stream_data['htpy_obj'].lines.get(False) except Queue.Empty: req = None else: req = t['request'] if 'tmp_hash' in t['request']: if t['request']['body_len'] > 0: t['request']['body_hash'] = t['request'][ 'tmp_hash'].hexdigest() else: t['request']['body_hash'] = "" del t['request']['tmp_hash'] if 'response' in t: resp = t['response'] if 'tmp_hash' in t['response']: if t['response']['body_len'] > 0: t['response']['body_hash'] = t['response'][ 'tmp_hash'].hexdigest() else: t['response']['body_hash'] = "" del t['response']['tmp_hash'] else: resp = None if req is not None or resp is not None: hchopp.setClientData(req) hchopp.setServerData(resp) hchopp.setTimeStamp(t['start']) hchopp.setAddr(chopp.addr) hchopp.setTeardown() hchopp.flowStart = stream_data['htpy_obj'].flowStart stream_data['htpy_obj'].ready = False stream_data['htpy_obj'].temp = {} stream_data['htpy_obj'].transaction = {} return hchopp return None
def _stream_ended_(frame, direction, tcp): hash_fn = tcp.module_data['options']['hash_fn'] transaction = { 'timestamp': tcp.stream_data['stream_cache'][frame.stream_id]['start'], 'request': { 'truncated': False, #TODO support this 'body': None, 'body_len': 0, 'body_hash': '', 'hash_fn': hash_fn, 'protocol': '2' }, 'response': { 'truncated': False, 'body': None, 'body_len': 0, 'body_hash': '', 'hash_fn': hash_fn, } } if tcp.stream_data['stream_cache'][frame.stream_id][direction][ 'stream_ended'] and tcp.stream_data['stream_cache'][ frame.stream_id][_opposite_direction_( direction)]['stream_ended']: #chop.prnt("Stream: %d" % (frame.stream_id)) for d in ['request', 'response']: headers = copy.deepcopy( tcp.stream_data['stream_cache'][frame.stream_id][d]['headers']) if d == 'request': headers[':stream_id'] = frame.stream_id method = headers.get(':method', None) path = headers.get(':path', None) if ':method' in headers: del headers[':method'] if ':path' in headers: del headers[':path'] transaction[d]['method'] = method transaction[d]['uri'] = path else: status = headers.get(':status', None) if ':status' in headers: del headers[':status'] transaction[d]['status'] = status transaction[d]['headers'] = headers #chop.prnt("\tHeaders: %s" % (headers)) if tcp.stream_data['stream_cache'][ frame.stream_id][d]['data'] is not None: content_encoding = headers.get('content-encoding', None) mimetype = headers.get('content-type', None) if mimetype is not None: mimetype = mimetype.split(';', 1)[0] content_disposition = headers.get('content-disposition', None) if content_encoding == 'gzip': try: dataStream = BytesIO(tcp.stream_data['stream_cache'][ frame.stream_id][d]['data']) gdata = gzip.GzipFile(fileobj=dataStream, mode='rb') data = gdata.read() except Exception as e: chop.prnt("Warning: Unable to gzip file") data = tcp.stream_data['stream_cache'][ frame.stream_id][d]['data'] else: data = tcp.stream_data['stream_cache'][ frame.stream_id][d]['data'] if mimetype is None: try: import magic except ImportError: pass else: try: mimetype = magic.from_buffer(data, mime=True) except Exception as e: chop.prnt( "Warning: Unable to get mime type of file: %s" % (str(e))) filename = 'noname' if d == 'response': if content_disposition is None: if transaction['request']['uri'] is not None: raw_path = transaction['request']['uri'] raw_path = raw_path.split('?', 1)[0] path_parts = os.path.split(raw_path) outPath = None while (len(path_parts) > 0): if path_parts[-1] == '': path_parts = path_parts[:-1] continue else: outPath = os.path.basename(path_parts[-1]) break if outPath is None or outPath == '': outPath = 'index' filename = sanitize_filename(outPath) else: filename = sanitize_filename(content_disposition) #chop.prnt("\tData Name: %s, Length: %d, Type: %s" % (filename, len(data), mimetype)) ((src, sport), (dst, dport)) = parse_addr(tcp) chop.savefile( "%d-%s-%d-%s-%d-%d-%s" % (tcp.timestamp, src, sport, dst, dport, frame.stream_id, filename), data) transaction[d]['body'] = data transaction[d]['body_len'] = len(data) transaction[d]['body_hash'] = __hash_function__( data).hexdigest() #chop.prnt("\n") #chop.prnt('\n') del tcp.stream_data['stream_cache'][frame.stream_id] #chop.prnt(tcp.stream_data['stream_cache'].keys()) #chop.prnt(json.dumps(transaction, indent=4)) chopp = ChopProtocol('http') chopp.setClientData(transaction['request']) chopp.setServerData(transaction['response']) chopp.setTimeStamp(transaction['timestamp']) chopp.setAddr(tcp.addr) chopp.flowStart = tcp.stream_data['flowStart'] return chopp else: return None
def teardownProtocol(chopp): if chopp.type != 'sslim': return stream_data = chopp.stream_data #sslim returns an empty object on teardown if 'htpy_obj' not in stream_data: return hchopp = ChopProtocol('http') ((src, sport), (dst, dport)) = chopp.addr stream_data['htpy_obj'].timestamp = chopp.timestamp #There's data collected in temp if len(stream_data['htpy_obj'].temp.keys()) > 1: #we don't care if only start is populated t = stream_data['htpy_obj'].temp if 'request' in t: if len(t['request'].keys()) == 0: try: req = stream_data['htpy_obj'].lines.get(False) except Queue.Empty: req = None else: req = t['request'] if 'tmp_hash' in t['request']: if t['request']['body_len'] > 0: t['request']['body_hash'] = t['request']['tmp_hash'].hexdigest() else: t['request']['body_hash'] = "" del t['request']['tmp_hash'] if 'response' in t: resp = t['response'] if 'tmp_hash' in t['response']: if t['response']['body_len'] > 0: t['response']['body_hash'] = t['response']['tmp_hash'].hexdigest() else: t['response']['body_hash'] = "" del t['response']['tmp_hash'] else: resp = None if req is not None or resp is not None: hchopp.setClientData(req) hchopp.setServerData(resp) hchopp.setTimeStamp(t['start']) hchopp.setAddr(chopp.addr) hchopp.setTeardown() hchopp.flowStart = stream_data['htpy_obj'].flowStart stream_data['htpy_obj'].ready = False stream_data['htpy_obj'].temp = {} stream_data['htpy_obj'].transaction = {} return hchopp return None