def start(self):
if self.pc is not None:
raise Exception('Already listening.')
self.pc = PcapWrapper(self.interface, filters='tcp')
try:
self.pc.loop(self._on_packet)
except KeyboardInterrupt:
pass
if self.pc.human_stats is not None:
logging.info(self.pc.human_stats)
def start(self):
if self.pc is not None:
raise Exception('Already listening.')
self.pc = PcapWrapper(self.interface, filters='tcp')
try:
self.pc.loop(self._on_packet)
except KeyboardInterrupt:
pass
if self.pc.human_stats is not None:
logging.info(self.pc.human_stats)
class NetworkFileListener(object):
def __init__(self, interface=None, mime_types=None):
self.pc = None
self.on_file_complete = None
self.packet_streams = {}
self.local_ips = self.detect_local_ips()
logging.info("Local IP Addresses: %s" % ', '.join(self.local_ips))
self.interface = interface
self.mime_types = mime_types
def detect_local_ips(self):
"""Determine all of the local ip addresses for this machine
This allows us to flag traffic as inbound or outbound.
"""
result = set()
for ifaceName in interfaces():
try:
address = [i['addr'] for i in ifaddresses(ifaceName)[AF_INET]]
except:
pass
result.add(address[0])
return tuple(result)
def start(self):
if self.pc is not None:
raise Exception('Already listening.')
self.pc = PcapWrapper(self.interface, filters='tcp')
try:
self.pc.loop(self._on_packet)
except KeyboardInterrupt:
pass
if self.pc.human_stats is not None:
logging.info(self.pc.human_stats)
def _on_packet(self, pkt):
try:
self._handle_packet(pkt)
except Exception as e:
logging.exception(e)
def _parse_tcp_packet(self, data):
if len(data) == 0 or data is None:
return None
if data.startswith('GET') or data.startswith('POST'):
return None
try:
return dpkt.http.Reqsponse(data)
except:
pass
return None
def _handle_packet(self, pkt):
eth = dpkt.ethernet.Ethernet(pkt)
ip = eth.data
tcp = ip.data
data = tcp.data
is_outbound = ipaddr_string(ip.dst) not in self.local_ips
direction = 'outbound' if is_outbound else 'inbound'
connection_hash = hash_packet(eth, outbound=is_outbound)
if ipaddr_string(ip.src) in self.local_ips and ipaddr_string(ip.dst) in self.local_ips:
# ignore packets that exist only on this computer
return
# lets first check if this is a http request instead of a response
if data.startswith('GET') or data.startswith('POST'):
if is_outbound:
# ignore inbound http request
_msg = 'Detected an %s HTTP Request from %s to %s'
logging.debug(_msg % (direction, ipaddr_string(ip.src), ipaddr_string(ip.dst)))
self._handle_request(connection_hash, tcp)
elif not is_outbound:
stream = self.packet_streams.get(connection_hash)
if stream is not None:
self._handle_response(stream, tcp)
def _handle_request(self, connection_hash, tcp_pkt):
if http.has_complete_headers(tcp_pkt.data):
req = http.parse_request(tcp_pkt.data)
logging.debug('Request URL: %s' % req['host'] + req['path'])
logging.debug('Storing stream %s.' % connection_hash)
self.packet_streams[connection_hash] = TcpStream(connection_hash)
def _delete_stream(self, stream):
if stream.id in self.packet_streams:
del self.packet_streams[stream.id]
def _handle_response(self, stream, tcp_pkt):
had_headers = (stream.headers is not None)
stream.add_packet(tcp_pkt)
if not had_headers and stream.headers is not None:
# this will happen the first packet that contain http header
if self.mime_types is not None:
mime_type = stream.headers.get('content-type', '').split(';')[0].strip()
if mime_type not in self.mime_types:
logging.debug('Ignoring mime_type %s' % mime_type)
self._delete_stream(stream)
return
if stream.is_finished:
if stream.id not in self.packet_streams:
# probably just a retransmission
return
self._delete_stream(stream)
if stream.is_valid:
self._on_request_complete(stream)
else:
_msg = "Stream was invalid at %.1f%% with %i bytes loaded"
logging.error(_msg % (stream.progress * 100, stream.http_bytes_loaded))
if self.pc.human_stats is not None:
logging.info(self.pc.human_stats)
def _on_request_complete(self, stream):
headers = stream.headers
if headers is not None:
mime_type = headers.get('content-type')
_msg = "Successfully observed a file with %i bytes and mime-type %s"
logging.info(_msg % (stream.http_content_length, stream.headers.get('content-type', '')))
f = RawFile(stream.content, mime_type)
self._on_file_complete(f)
def _on_file_complete(self, f):
if self.on_file_complete is not None:
self.on_file_complete(f)
class NetworkFileListener(object):
def __init__(self, interface=None, mime_types=None):
self.pc = None
self.on_file_complete = None
self.packet_streams = {}
self.local_ips = self.detect_local_ips()
logging.info("Local IP Addresses: %s" % ', '.join(self.local_ips))
self.interface = interface
self.mime_types = mime_types
def detect_local_ips(self):
"""Determine all of the local ip addresses for this machine
This allows us to flag traffic as inbound or outbound.
"""
result = set()
for ifaceName in interfaces():
try:
address = [i['addr'] for i in ifaddresses(ifaceName)[AF_INET]]
except:
pass
result.add(address[0])
return tuple(result)
def start(self):
if self.pc is not None:
raise Exception('Already listening.')
self.pc = PcapWrapper(self.interface, filters='tcp')
try:
self.pc.loop(self._on_packet)
except KeyboardInterrupt:
pass
if self.pc.human_stats is not None:
logging.info(self.pc.human_stats)
def _on_packet(self, pkt):
try:
self._handle_packet(pkt)
except Exception as e:
logging.exception(e)
def _parse_tcp_packet(self, data):
if len(data) == 0 or data is None:
return None
if data.startswith('GET') or data.startswith('POST'):
return None
try:
return dpkt.http.Reqsponse(data)
except:
pass
return None
def _handle_packet(self, pkt):
eth = dpkt.ethernet.Ethernet(pkt)
ip = eth.data
tcp = ip.data
data = tcp.data
is_outbound = ipaddr_string(ip.dst) not in self.local_ips
direction = 'outbound' if is_outbound else 'inbound'
connection_hash = hash_packet(eth, outbound=is_outbound)
if ipaddr_string(ip.src) in self.local_ips and ipaddr_string(
ip.dst) in self.local_ips:
# ignore packets that exist only on this computer
return
# lets first check if this is a http request instead of a response
if data.startswith('GET') or data.startswith('POST'):
if is_outbound:
# ignore inbound http request
_msg = 'Detected an %s HTTP Request from %s to %s'
logging.debug(
_msg %
(direction, ipaddr_string(ip.src), ipaddr_string(ip.dst)))
self._handle_request(connection_hash, tcp)
elif not is_outbound:
stream = self.packet_streams.get(connection_hash)
if stream is not None:
self._handle_response(stream, tcp)
def _handle_request(self, connection_hash, tcp_pkt):
if http.has_complete_headers(tcp_pkt.data):
req = http.parse_request(tcp_pkt.data)
logging.debug('Request URL: %s' % req['host'] + req['path'])
logging.debug('Storing stream %s.' % connection_hash)
self.packet_streams[connection_hash] = TcpStream(connection_hash)
def _delete_stream(self, stream):
if stream.id in self.packet_streams:
del self.packet_streams[stream.id]
def _handle_response(self, stream, tcp_pkt):
had_headers = (stream.headers is not None)
stream.add_packet(tcp_pkt)
if not had_headers and stream.headers is not None:
# this will happen the first packet that contain http header
if self.mime_types is not None:
mime_type = stream.headers.get('content-type',
'').split(';')[0].strip()
if mime_type not in self.mime_types:
logging.debug('Ignoring mime_type %s' % mime_type)
self._delete_stream(stream)
return
if stream.is_finished:
if stream.id not in self.packet_streams:
# probably just a retransmission
return
self._delete_stream(stream)
if stream.is_valid:
self._on_request_complete(stream)
else:
_msg = "Stream was invalid at %.1f%% with %i bytes loaded"
logging.error(
_msg % (stream.progress * 100, stream.http_bytes_loaded))
if self.pc.human_stats is not None:
logging.info(self.pc.human_stats)
def _on_request_complete(self, stream):
headers = stream.headers
if headers is not None:
mime_type = headers.get('content-type')
_msg = "Successfully observed a file with %i bytes and mime-type %s"
logging.info(_msg % (stream.http_content_length,
stream.headers.get('content-type', '')))
f = RawFile(stream.content, mime_type)
self._on_file_complete(f)
def _on_file_complete(self, f):
if self.on_file_complete is not None:
self.on_file_complete(f)
