def _process_tcp_udp_flow(self, pkt, protocol):
if protocol == 'tcp':
layer = sc.TCP
elif protocol == 'udp':
layer = sc.UDP
else:
return
# Parse packet
src_mac = pkt[sc.Ether].src
dst_mac = pkt[sc.Ether].dst
src_ip = pkt[sc.IP].src
dst_ip = pkt[sc.IP].dst
src_port = pkt[layer].sport
dst_port = pkt[layer].dport
# No broadcast
if dst_mac == 'ff:ff:ff:ff:ff:ff' or dst_ip == '255.255.255.255':
return
# Only look at flows where this host pretends to be the gateway
host_mac = self._host_state.host_mac
host_gateway_inbound = (src_mac == host_mac)
host_gateway_outbound = (dst_mac == host_mac)
if not (host_gateway_inbound or host_gateway_outbound):
return
# Extract TCP sequence and ack numbers for us to estimate flow size
# later
tcp_seq = None
tcp_ack = None
try:
tcp_layer = pkt[sc.TCP]
tcp_seq = tcp_layer.seq
if tcp_layer.ack > 0:
tcp_ack = tcp_layer.ack
except Exception:
pass
# Determine flow direction
if src_mac == host_mac:
direction = 'inbound'
device_mac = dst_mac
device_port = dst_port
remote_ip = src_ip
remote_port = src_port
elif dst_mac == host_mac:
direction = 'outbound'
device_mac = src_mac
device_port = src_port
remote_ip = dst_ip
remote_port = dst_port
else:
return
# Anonymize device mac
device_id = utils.get_device_id(device_mac, self._host_state)
# Construct flow key
flow_key = (
device_id, device_port, remote_ip, remote_port, protocol
)
# Initialize flow_stats. Note: TCP byte counts may include out-of-order
# packets and RSTs. On the other hand, TCP sequence number shows how
# many unique bytes are transmitted in a flow; this number does not
# consider the size of headers (Eth + IP + TCP = 66 bytes in my
# experiment), out-of-order packets, or RSTs.
flow_stats = {
'inbound_byte_count': 0,
'inbound_tcp_seq_min_max': (None, None),
'inbound_tcp_ack_min_max': (None, None),
'outbound_byte_count': 0,
'outbound_tcp_seq_min_max': (None, None),
'outbound_tcp_ack_min_max': (None, None),
'syn_originator': None
}
with self._host_state.lock:
flow_stats = self._host_state.pending_flow_dict \
.setdefault(flow_key, flow_stats)
# Construct flow_stats
flow_stats[direction + '_byte_count'] += len(pkt)
flow_stats[direction + '_tcp_seq_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_seq_min_max'], tcp_seq)
flow_stats[direction + '_tcp_ack_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_ack_min_max'], tcp_ack)
# Who initiated the SYN packet
syn_originator = None
try:
if pkt[sc.TCP].flags == 2:
if src_ip == remote_ip:
syn_originator = 'remote'
else:
syn_originator = 'local'
except Exception:
pass
if syn_originator and flow_stats['syn_originator'] is None:
flow_stats['syn_originator'] = syn_originator
# Extract UA and Host
if remote_port == 80 and protocol == 'tcp':
self._process_http(pkt, device_id, remote_ip)
# Extract SNI from TLS client handshake
if protocol == 'tcp':
self._process_tls(pkt, device_id)
with self._host_state.lock:
self._host_state.byte_count += len(pkt)
def _process_tcp_udp_flow(self, pkt, protocol):
if protocol == 'tcp':
layer = sc.TCP
elif protocol == 'udp':
layer = sc.UDP
else:
return
# Parse packet
src_mac = pkt[sc.Ether].src
dst_mac = pkt[sc.Ether].dst
src_ip = pkt[sc.IP].src
dst_ip = pkt[sc.IP].dst
src_port = pkt[layer].sport
dst_port = pkt[layer].dport
# No broadcast
if dst_mac == 'ff:ff:ff:ff:ff:ff' or dst_ip == '255.255.255.255':
return
# Only look at flows where this host pretends to be the gateway
host_mac = self._host_state.host_mac
# Extract TCP sequence and ack numbers for us to estimate flow size
# later
tcp_seq = None
tcp_ack = None
try:
tcp_layer = pkt[sc.TCP]
tcp_seq = tcp_layer.seq
if tcp_layer.ack > 0:
tcp_ack = tcp_layer.ack
except Exception:
pass
# Determine flow direction
if src_mac == host_mac:
direction = 'inbound'
device_mac = dst_mac
device_port = dst_port
remote_ip = src_ip
remote_port = src_port
elif dst_mac == host_mac:
direction = 'outbound'
device_mac = src_mac
device_port = src_port
remote_ip = dst_ip
remote_port = dst_port
else:
return
# Anonymize device mac
device_id = utils.get_device_id(device_mac, self._host_state)
# Get remote device_id for internal book-keeping purpose
remote_device_id = ''
remote_ip_is_inspector_host = 0 # True (1) or False (0)
remote_ip_is_gateway = 0 # True (1) or False (0)
try:
with self._host_state.lock:
real_remote_device_mac = self._host_state.ip_mac_dict[
remote_ip]
remote_device_id = utils.get_device_id(real_remote_device_mac,
self._host_state)
if remote_ip == self._host_state.host_ip:
remote_ip_is_inspector_host = 1
elif remote_ip == self._host_state.gateway_ip:
remote_ip_is_gateway = 1
except Exception:
pass
# Construct flow key
flow_key = (device_id, device_port, remote_ip, remote_port, protocol)
flow_key_str = ':'.join([str(item) for item in flow_key])
flow_ts = time.time()
# Initialize flow_stats. Note: TCP byte counts may include out-of-order
# packets and RSTs. On the other hand, TCP sequence number shows how
# many unique bytes are transmitted in a flow; this number does not
# consider the size of headers (Eth + IP + TCP = 66 bytes in my
# experiment), out-of-order packets, or RSTs.
flow_stats = {
'inbound_byte_count': 0,
'inbound_tcp_seq_min_max': (None, None),
'inbound_tcp_ack_min_max': (None, None),
'outbound_byte_count': 0,
'outbound_tcp_seq_min_max': (None, None),
'outbound_tcp_ack_min_max': (None, None),
'syn_originator': None,
'internal_remote_device_id': remote_device_id,
'internal_first_packet_originator': '',
'internal_remote_ip_is_inspector_host':
remote_ip_is_inspector_host,
'internal_remote_ip_is_gateway': remote_ip_is_gateway,
'internal_inbound_pkt_count': 0,
'internal_outbound_pkt_count': 0,
'internal_flow_ts_min': flow_ts,
'internal_flow_ts_max': flow_ts,
'internal_flow_key': flow_key_str
}
with self._host_state.lock:
flow_stats = self._host_state.pending_flow_dict \
.setdefault(flow_key, flow_stats)
# Identify who sent out the first UDP packet in this flow
if flow_stats['internal_first_packet_originator'] == '':
if direction == 'inbound':
flow_stats['internal_first_packet_originator'] = 'remote'
else:
flow_stats['internal_first_packet_originator'] = 'local'
# Construct flow_stats
flow_stats[direction + '_byte_count'] += len(pkt)
flow_stats[direction + '_tcp_seq_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_seq_min_max'], tcp_seq)
flow_stats[direction + '_tcp_ack_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_ack_min_max'], tcp_ack)
flow_stats['internal_' + direction + '_pkt_count'] += 1
flow_stats['internal_flow_ts_max'] = flow_ts
# Who initiated the SYN packet
syn_originator = None
try:
if pkt[sc.TCP].flags == 2:
if src_ip == remote_ip:
syn_originator = 'remote'
else:
syn_originator = 'local'
except Exception:
pass
if syn_originator and flow_stats['syn_originator'] is None:
flow_stats['syn_originator'] = syn_originator
# Extract UA and Host
if remote_port == 80 and protocol == 'tcp':
self._process_http(pkt, device_id, remote_ip)
# Extract SNI from TLS client handshake
if protocol == 'tcp':
self._process_tls(pkt, device_id)
with self._host_state.lock:
self._host_state.byte_count += len(pkt)