def start(self): if self.pc is not None: raise Exception('Already listening.') self.pc = PcapWrapper(self.interface, filters='tcp') try: self.pc.loop(self._on_packet) except KeyboardInterrupt: pass if self.pc.human_stats is not None: logging.info(self.pc.human_stats)
class NetworkFileListener(object): def __init__(self, interface=None, mime_types=None): self.pc = None self.on_file_complete = None self.packet_streams = {} self.local_ips = self.detect_local_ips() logging.info("Local IP Addresses: %s" % ', '.join(self.local_ips)) self.interface = interface self.mime_types = mime_types def detect_local_ips(self): """Determine all of the local ip addresses for this machine This allows us to flag traffic as inbound or outbound. """ result = set() for ifaceName in interfaces(): try: address = [i['addr'] for i in ifaddresses(ifaceName)[AF_INET]] except: pass result.add(address[0]) return tuple(result) def start(self): if self.pc is not None: raise Exception('Already listening.') self.pc = PcapWrapper(self.interface, filters='tcp') try: self.pc.loop(self._on_packet) except KeyboardInterrupt: pass if self.pc.human_stats is not None: logging.info(self.pc.human_stats) def _on_packet(self, pkt): try: self._handle_packet(pkt) except Exception as e: logging.exception(e) def _parse_tcp_packet(self, data): if len(data) == 0 or data is None: return None if data.startswith('GET') or data.startswith('POST'): return None try: return dpkt.http.Reqsponse(data) except: pass return None def _handle_packet(self, pkt): eth = dpkt.ethernet.Ethernet(pkt) ip = eth.data tcp = ip.data data = tcp.data is_outbound = ipaddr_string(ip.dst) not in self.local_ips direction = 'outbound' if is_outbound else 'inbound' connection_hash = hash_packet(eth, outbound=is_outbound) if ipaddr_string(ip.src) in self.local_ips and ipaddr_string(ip.dst) in self.local_ips: # ignore packets that exist only on this computer return # lets first check if this is a http request instead of a response if data.startswith('GET') or data.startswith('POST'): if is_outbound: # ignore inbound http request _msg = 'Detected an %s HTTP Request from %s to %s' logging.debug(_msg % (direction, ipaddr_string(ip.src), ipaddr_string(ip.dst))) self._handle_request(connection_hash, tcp) elif not is_outbound: stream = self.packet_streams.get(connection_hash) if stream is not None: self._handle_response(stream, tcp) def _handle_request(self, connection_hash, tcp_pkt): if http.has_complete_headers(tcp_pkt.data): req = http.parse_request(tcp_pkt.data) logging.debug('Request URL: %s' % req['host'] + req['path']) logging.debug('Storing stream %s.' % connection_hash) self.packet_streams[connection_hash] = TcpStream(connection_hash) def _delete_stream(self, stream): if stream.id in self.packet_streams: del self.packet_streams[stream.id] def _handle_response(self, stream, tcp_pkt): had_headers = (stream.headers is not None) stream.add_packet(tcp_pkt) if not had_headers and stream.headers is not None: # this will happen the first packet that contain http header if self.mime_types is not None: mime_type = stream.headers.get('content-type', '').split(';')[0].strip() if mime_type not in self.mime_types: logging.debug('Ignoring mime_type %s' % mime_type) self._delete_stream(stream) return if stream.is_finished: if stream.id not in self.packet_streams: # probably just a retransmission return self._delete_stream(stream) if stream.is_valid: self._on_request_complete(stream) else: _msg = "Stream was invalid at %.1f%% with %i bytes loaded" logging.error(_msg % (stream.progress * 100, stream.http_bytes_loaded)) if self.pc.human_stats is not None: logging.info(self.pc.human_stats) def _on_request_complete(self, stream): headers = stream.headers if headers is not None: mime_type = headers.get('content-type') _msg = "Successfully observed a file with %i bytes and mime-type %s" logging.info(_msg % (stream.http_content_length, stream.headers.get('content-type', ''))) f = RawFile(stream.content, mime_type) self._on_file_complete(f) def _on_file_complete(self, f): if self.on_file_complete is not None: self.on_file_complete(f)
class NetworkFileListener(object): def __init__(self, interface=None, mime_types=None): self.pc = None self.on_file_complete = None self.packet_streams = {} self.local_ips = self.detect_local_ips() logging.info("Local IP Addresses: %s" % ', '.join(self.local_ips)) self.interface = interface self.mime_types = mime_types def detect_local_ips(self): """Determine all of the local ip addresses for this machine This allows us to flag traffic as inbound or outbound. """ result = set() for ifaceName in interfaces(): try: address = [i['addr'] for i in ifaddresses(ifaceName)[AF_INET]] except: pass result.add(address[0]) return tuple(result) def start(self): if self.pc is not None: raise Exception('Already listening.') self.pc = PcapWrapper(self.interface, filters='tcp') try: self.pc.loop(self._on_packet) except KeyboardInterrupt: pass if self.pc.human_stats is not None: logging.info(self.pc.human_stats) def _on_packet(self, pkt): try: self._handle_packet(pkt) except Exception as e: logging.exception(e) def _parse_tcp_packet(self, data): if len(data) == 0 or data is None: return None if data.startswith('GET') or data.startswith('POST'): return None try: return dpkt.http.Reqsponse(data) except: pass return None def _handle_packet(self, pkt): eth = dpkt.ethernet.Ethernet(pkt) ip = eth.data tcp = ip.data data = tcp.data is_outbound = ipaddr_string(ip.dst) not in self.local_ips direction = 'outbound' if is_outbound else 'inbound' connection_hash = hash_packet(eth, outbound=is_outbound) if ipaddr_string(ip.src) in self.local_ips and ipaddr_string( ip.dst) in self.local_ips: # ignore packets that exist only on this computer return # lets first check if this is a http request instead of a response if data.startswith('GET') or data.startswith('POST'): if is_outbound: # ignore inbound http request _msg = 'Detected an %s HTTP Request from %s to %s' logging.debug( _msg % (direction, ipaddr_string(ip.src), ipaddr_string(ip.dst))) self._handle_request(connection_hash, tcp) elif not is_outbound: stream = self.packet_streams.get(connection_hash) if stream is not None: self._handle_response(stream, tcp) def _handle_request(self, connection_hash, tcp_pkt): if http.has_complete_headers(tcp_pkt.data): req = http.parse_request(tcp_pkt.data) logging.debug('Request URL: %s' % req['host'] + req['path']) logging.debug('Storing stream %s.' % connection_hash) self.packet_streams[connection_hash] = TcpStream(connection_hash) def _delete_stream(self, stream): if stream.id in self.packet_streams: del self.packet_streams[stream.id] def _handle_response(self, stream, tcp_pkt): had_headers = (stream.headers is not None) stream.add_packet(tcp_pkt) if not had_headers and stream.headers is not None: # this will happen the first packet that contain http header if self.mime_types is not None: mime_type = stream.headers.get('content-type', '').split(';')[0].strip() if mime_type not in self.mime_types: logging.debug('Ignoring mime_type %s' % mime_type) self._delete_stream(stream) return if stream.is_finished: if stream.id not in self.packet_streams: # probably just a retransmission return self._delete_stream(stream) if stream.is_valid: self._on_request_complete(stream) else: _msg = "Stream was invalid at %.1f%% with %i bytes loaded" logging.error( _msg % (stream.progress * 100, stream.http_bytes_loaded)) if self.pc.human_stats is not None: logging.info(self.pc.human_stats) def _on_request_complete(self, stream): headers = stream.headers if headers is not None: mime_type = headers.get('content-type') _msg = "Successfully observed a file with %i bytes and mime-type %s" logging.info(_msg % (stream.http_content_length, stream.headers.get('content-type', ''))) f = RawFile(stream.content, mime_type) self._on_file_complete(f) def _on_file_complete(self, f): if self.on_file_complete is not None: self.on_file_complete(f)