def _process_dhcp(self, pkt):
"""
Extracts the client hostname from DHCP Request packets.
"""
try:
option_dict = dict(
[t for t in pkt[sc.DHCP].options if isinstance(t, tuple)])
except Exception:
return
try:
device_hostname = option_dict.setdefault('hostname',
'').decode('utf-8')
except Exception:
device_hostname = ''
resolver_ip = option_dict.setdefault('name_server', '')
with self._host_state.lock:
if device_hostname:
# Must be a DHCP Request broadcast
if pkt[sc.Ether].dst != 'ff:ff:ff:ff:ff:ff':
return
device_mac = pkt[sc.Ether].src
device_id = utils.get_device_id(device_mac, self._host_state)
self._host_state.pending_dhcp_dict[device_id] = device_hostname
utils.log('[UPLOAD] DHCP Hostname:', device_hostname)
if resolver_ip:
# DHCP Offer broadcast
if pkt[sc.Ether].dst == 'ff:ff:ff:ff:ff:ff':
device_id = 'broadcast'
# DHCP ACK from router to device. The following block may not
# actually be called at all, because the router is likely to
# send the ACK packet directly to the device (rather than arp
# spoofed)
else:
device_ip = pkt[sc.IP].dst
try:
device_mac = self._host_state.ip_mac_dict[device_ip]
except KeyError:
return
device_id = utils.get_device_id(device_mac,
self._host_state)
self._host_state.pending_resolver_dict[device_id] = \
resolver_ip
utils.log('[UPLOAD] DHCP Resolver:', device_id, '-',
resolver_ip)
def create_user(request):
req = poem_pb.RegisterRequest()
req.ParseFromString(request.body)
clientIP = request.env.get('REMOTE_ADDR', '')
username = req.username
password = req.password
# 大小包处理
try:
if UsernameIndexing.exists(username):
raise RegisterError(_YYTEXT(u'该帐号已被注册，请重新输入'))
user = register(username, password, get_device_id(req.deviceInfo))
except RegisterError as e:
raise ApplicationError(0, e.message)
info = unpack_login_info(req.deviceInfo,
userID=user.userID,
username=user.username,
featureCode=req.featureCode,
clientIP=clientIP)
info.update({
'userID': user.userID,
'username': user.username,
'type': 'register'
})
gm_logger.info({'sessionaccess': info})
account_register.info(**info)
return poem_pb.RegisterResponse(username=user.username)
def _parse_netdisco_output(self, line):
try:
data = json.loads(line)
except ValueError:
return
if 'version' in data:
return
device_type = data['device']
for device_info in data['info']:
# Find IP
device_ip = device_info['host']
device_info['device_type'] = device_type
# Find MAC based on IP
try:
with self._host_state.lock:
device_mac = self._host_state.ip_mac_dict[device_ip]
except KeyError:
return
# Get device_id based on MAC
device_id = utils.get_device_id(device_mac, self._host_state)
# Submit for upload lter
with self._host_state.lock:
self._host_state.pending_netdisco_dict \
.setdefault(device_id, []).append(device_info)
def _init(self):
args = self._args
self._device = torch.device(
utils.get_device_id(torch.cuda.is_available()))
self._aug = utils.Augmentations(method=args.aug)
self._dataset = data.Dataset(root=args.root,
name=args.name,
num_parts=args.init_parts,
final_parts=args.final_parts,
augumentation=self._aug)
self._loader = DataLoader(
dataset=self._dataset) # [self._dataset.data]
print(f"Data Augmentation method {args.aug}")
print(f"Data: {self._dataset.data}")
hidden_layers = [int(l) for l in args.layers]
layers = [self._dataset.data.x.shape[1]] + hidden_layers
self._norm_config = utils.get_norm_configs(args.norms)
self._model = models.SelfGNN(layer_config=layers,
dropout=args.dropout,
gnn_type=args.model,
heads=args.heads,
**self._norm_config).to(self._device)
print(self._model)
self._optimizer = torch.optim.Adam(params=self._model.parameters(),
lr=args.lr)
def _arp_spoof_loop(self):
while True:
time.sleep(1)
with self._lock:
if not self._active:
return
with self._host_state.lock:
if not self._host_state.has_consent:
utils.log('[ARP Spoof] No consent; no spoofing.')
continue
# Get ARP cache
ip_mac_dict = self._host_state.get_ip_mac_dict_copy()
gateway_ip = self._host_state.gateway_ip
utils.log('[ARP Spoof] Cache:', ip_mac_dict)
utils.log('[ARP Spoof] Whitelist:',
self._host_state.device_whitelist)
# Get gateway MAC addr
try:
gateway_mac = ip_mac_dict[gateway_ip]
except KeyError:
continue
# Spoof individual devices on the network.
for (victim_ip, victim_mac) in ip_mac_dict.items():
if victim_ip == gateway_ip:
continue
# Check against whitelist.
victim_device_id = \
utils.get_device_id(victim_mac, self._host_state)
victim_mac_last_four_bytes = \
victim_mac.lower().replace(':', '')[-4:]
is_in_whitelist = (victim_mac_last_four_bytes
in self._host_state.device_whitelist
or victim_device_id
in self._host_state.device_whitelist)
if not is_in_whitelist:
utils.log('[ARP Spoof] Ignore:', victim_ip, victim_mac)
continue
utils.safe_run(self._arp_spoof,
args=(victim_mac, victim_ip, gateway_mac,
gateway_ip))
with self._lock:
if not self._active:
return
time.sleep(max(MIN_ARP_SPOOF_INTERVAL, 2.0 / len(ip_mac_dict)))
def get_device_list():
"""
Returns a list of devices; constantly changes.
"""
# Maps device_id -> {device_id, device_vendor, netdisco_name}
output_dict = {}
host_state = get_host_state()
if host_state is None:
return json.dumps(output_dict)
# Get device vendor
with host_state.lock:
for (ip, mac) in host_state.ip_mac_dict.items():
# Never include the gateway
if ip == host_state.gateway_ip:
continue
device_id = utils.get_device_id(mac, host_state)
device_vendor = oui_parser.get_vendor(utils.get_oui(mac))
output_dict.setdefault(
device_id, {
'device_id': device_id,
'device_ip': ip,
'device_vendor': device_vendor,
'netdisco_name': '',
'dhcp_name': '',
'is_inspected': device_id in host_state.device_whitelist
})
# Fill out netdisco_name
with host_state.lock:
for (device_id,
device_info_list) in host_state.pending_netdisco_dict.items():
if device_id in output_dict:
output_dict[device_id]['netdisco_name'] = device_info_list
# Reset pending dict
host_state.pending_netdisco_dict = {}
# Fill out dhcp_name
with host_state.lock:
for (device_id, dhcp_name) in host_state.pending_dhcp_dict.items():
if device_id in output_dict:
output_dict[device_id]['dhcp_name'] = dhcp_name
# Reset pending dict
host_state.pending_dhcp_dict = {}
return json.dumps(output_dict, indent=2)
def _process_dns(self, pkt):
src_mac = pkt[sc.Ether].src
dst_mac = pkt[sc.Ether].dst
src_ip = pkt[sc.IP].src
dst_ip = pkt[sc.IP].dst
# Find device ID
if pkt[sc.DNS].qr == 0:
# DNS request
if dst_mac == self._host_state.host_mac:
device_mac = src_mac
resolver_ip = dst_ip
else:
return
else:
# DNS response
if src_mac == self._host_state.host_mac:
device_mac = dst_mac
resolver_ip = src_ip
else:
return
device_id = utils.get_device_id(device_mac, self._host_state)
# Parse domain
try:
domain = pkt[sc.DNSQR].qname.lower()
except AttributeError:
return
# Remove trailing dot from domain
if domain[-1] == '.':
domain = domain[0:-1]
# Parse DNS response
ip_set = set()
if sc.DNSRR in pkt and pkt[sc.DNS].an:
for ix in range(pkt[sc.DNS].ancount):
# Extracts A-records
if pkt[sc.DNSRR][ix].type == 1:
# Extracts IPv4 addr in A-record
ip = pkt[sc.DNSRR][ix].rdata
if utils.is_ipv4_addr(ip):
ip_set.add(ip)
with self._host_state.lock:
domain = str(domain)
dns_key = (device_id, domain, resolver_ip, 0)
current_ip_set = self._host_state \
.pending_dns_dict.setdefault(dns_key, set())
ip_set = ip_set | current_ip_set
self._host_state.pending_dns_dict[dns_key] = ip_set
def get_device_list_helper():
# Maps device_id -> {device_id, device_vendor, netdisco_name}
output_dict = {}
host_state = get_host_state()
if host_state is None:
return json.dumps(output_dict)
# Get device vendor
with host_state.lock:
for (ip, mac) in host_state.ip_mac_dict.items():
# Never include the gateway
if ip == host_state.gateway_ip:
continue
device_id = utils.get_device_id(mac, host_state)
output_dict.setdefault(
device_id, {
'device_id': device_id,
'device_ip': ip,
'device_name': device_identification.get_device_name(mac),
'device_vendor':
device_identification.get_device_vendor(mac),
'netdisco_name': '',
'dhcp_name': '',
'is_inspected': device_id in host_state.device_whitelist
})
# Fill out netdisco_name
with host_state.lock:
for (device_id,
device_info_list) in host_state.pending_netdisco_dict.items():
if device_id in output_dict:
output_dict[device_id]['netdisco_name'] = device_info_list
# Reset pending dict
host_state.pending_netdisco_dict = {}
# Fill out dhcp_name
with host_state.lock:
for (device_id, dhcp_name) in host_state.pending_dhcp_dict.items():
if device_id in output_dict:
output_dict[device_id]['dhcp_name'] = dhcp_name
# Reset pending dict
host_state.pending_dhcp_dict = {}
return output_dict
def _process_syn_scan(self, pkt):
"""
Receives SYN scan response from devices.
"""
src_mac = pkt[sc.Ether].src
device_id = utils.get_device_id(src_mac, self._host_state)
device_port = pkt[sc.TCP].sport
with self._host_state.lock:
port_list = self._host_state.pending_syn_scan_dict.setdefault(
device_id, [])
if device_port not in port_list:
port_list.append(device_port)
utils.log('[SYN Scan Debug] Device {} ({}): Port {}'.format(
pkt[sc.IP].src, device_id, device_port))
def _process_dhcp(self, pkt):
"""
Extracts the client hostname from DHCP Request packets.
"""
try:
if pkt[sc.Ether].dst != 'ff:ff:ff:ff:ff:ff':
return
device_mac = pkt[sc.Ether].src
device_hostname = dict([
t for t in pkt[sc.DHCP].options if isinstance(t, tuple)
])['hostname']
except Exception:
return
device_id = utils.get_device_id(device_mac, self._host_state)
with self._host_state.lock:
self._host_state.pending_dhcp_dict[device_id] = \
device_hostname
def _run_netdisco(self):
netdis = NetworkDiscovery()
netdis.scan()
for device_type in netdis.discover():
device_info = netdis.get_info(device_type)[0]
device_ip = device_info['host']
device_info['device_type'] = device_type
# Find MAC based on IP
try:
with self._host_state.lock:
device_mac = self._host_state.ip_mac_dict[device_ip]
except KeyError:
continue
# Get device_id based on MAC
device_id = utils.get_device_id(device_mac, self._host_state)
# Submit for upload lter
with self._host_state.lock:
self._host_state.pending_netdisco_dict \
.setdefault(device_id, []).append(device_info)
def _process_tcp_udp_flow(self, pkt, protocol):
if protocol == 'tcp':
layer = sc.TCP
elif protocol == 'udp':
layer = sc.UDP
else:
return
# Parse packet
src_mac = pkt[sc.Ether].src
dst_mac = pkt[sc.Ether].dst
src_ip = pkt[sc.IP].src
dst_ip = pkt[sc.IP].dst
src_port = pkt[layer].sport
dst_port = pkt[layer].dport
# No broadcast
if dst_mac == 'ff:ff:ff:ff:ff:ff' or dst_ip == '255.255.255.255':
return
# Only look at flows where this host pretends to be the gateway
host_mac = self._host_state.host_mac
host_gateway_inbound = (src_mac == host_mac)
host_gateway_outbound = (dst_mac == host_mac)
if not (host_gateway_inbound or host_gateway_outbound):
return
# Extract TCP sequence and ack numbers for us to estimate flow size
# later
tcp_seq = None
tcp_ack = None
try:
tcp_layer = pkt[sc.TCP]
tcp_seq = tcp_layer.seq
if tcp_layer.ack > 0:
tcp_ack = tcp_layer.ack
except Exception:
pass
# Determine flow direction
if src_mac == host_mac:
direction = 'inbound'
device_mac = dst_mac
device_port = dst_port
remote_ip = src_ip
remote_port = src_port
elif dst_mac == host_mac:
direction = 'outbound'
device_mac = src_mac
device_port = src_port
remote_ip = dst_ip
remote_port = dst_port
else:
return
# Anonymize device mac
device_id = utils.get_device_id(device_mac, self._host_state)
# Construct flow key
flow_key = (
device_id, device_port, remote_ip, remote_port, protocol
)
# Initialize flow_stats. Note: TCP byte counts may include out-of-order
# packets and RSTs. On the other hand, TCP sequence number shows how
# many unique bytes are transmitted in a flow; this number does not
# consider the size of headers (Eth + IP + TCP = 66 bytes in my
# experiment), out-of-order packets, or RSTs.
flow_stats = {
'inbound_byte_count': 0,
'inbound_tcp_seq_min_max': (None, None),
'inbound_tcp_ack_min_max': (None, None),
'outbound_byte_count': 0,
'outbound_tcp_seq_min_max': (None, None),
'outbound_tcp_ack_min_max': (None, None),
'syn_originator': None
}
with self._host_state.lock:
flow_stats = self._host_state.pending_flow_dict \
.setdefault(flow_key, flow_stats)
# Construct flow_stats
flow_stats[direction + '_byte_count'] += len(pkt)
flow_stats[direction + '_tcp_seq_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_seq_min_max'], tcp_seq)
flow_stats[direction + '_tcp_ack_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_ack_min_max'], tcp_ack)
# Who initiated the SYN packet
syn_originator = None
try:
if pkt[sc.TCP].flags == 2:
if src_ip == remote_ip:
syn_originator = 'remote'
else:
syn_originator = 'local'
except Exception:
pass
if syn_originator and flow_stats['syn_originator'] is None:
flow_stats['syn_originator'] = syn_originator
# Extract UA and Host
if remote_port == 80 and protocol == 'tcp':
self._process_http(pkt, device_id, remote_ip)
# Extract SNI from TLS client handshake
if protocol == 'tcp':
self._process_tls(pkt, device_id)
with self._host_state.lock:
self._host_state.byte_count += len(pkt)
def common_login(request,
raw_username,
req,
auto_register=False,
check_password=True):
# 检查版本
try:
clientversion = int(getattr(req.deviceInfo, 'clientVersion', 1))
except ValueError:
clientversion = 1
if get_device_id(req.deviceInfo) in g_block_devices:
raise ApplicationError(0, u"该设备已被禁止登录 如有疑问请联系客服")
if get_g_version() and clientversion and get_g_version() > clientversion:
raise ApplicationError(msgTips.FAIL_MSG_LOGIN_OLDVERSION,
settings.CLIENTOLDVERSION_NOTICE)
username = raw_username
# 检查白名单
if username not in g_whitelist:
raise ApplicationError(0, get_loginlimitmsg())
# 查找用户
userID = UsernameIndexing.get_pk(username)
if userID:
user = User.get(userID)
user.load_containers()
# 检查设备锁定
if user.lock_device and get_device_id(
req.deviceInfo) != user.lock_device:
raise ApplicationError(0, u'该帐号为试玩帐号，只能在注册时的使用的设备上登录！')
else:
user = None
clientIP = request.env.get('REMOTE_ADDR', '')
password = getattr(req, 'password', '') or "dummy"
if not user and auto_register:
# register
try:
user = register(raw_username, password,
get_device_id(req.deviceInfo))
except RegisterError as e:
raise ApplicationError(0, e.message)
# 创建成功
userID = user.userID
info = unpack_login_info(req.deviceInfo,
userID=userID,
username=user.username,
featureCode=req.featureCode,
clientIP=clientIP)
info.update({
'userID': userID,
'username': user.username,
'type': 'register'
})
gm_logger.info({'sessionaccess': info})
account_register.info(**info)
if not user:
raise ApplicationError(msgTips.FAIL_MSG_LOGIN_WRONG_ACCOUNT)
# 检查密码
userID = user.userID
if check_password:
hashed = user.password
if not utils.check_password(password, hashed):
raise ApplicationError(msgTips.FAIL_MSG_INVALID_PASSWORD)
# 检查封停
now = int(time.time())
if user.blocktime and user.blocktime > now:
raise ApplicationError(0, u"该账号已被禁止登录 如有疑问请联系客服")
if user.imsi in g_block_devices:
raise ApplicationError(0, u"该账号已被禁止登录 如有疑问请联系客服")
do_login(request, userID)
user.lastserver = req.regionID
user.save()
# log
info = unpack_login_info(req.deviceInfo,
userID=userID,
username=user.username,
featureCode=req.featureCode,
clientIP=clientIP)
info.update({'userID': userID, 'username': user.username, 'type': 'login'})
gm_logger.info({'sessionaccess': info})
account_login.info(**info)
# 响应
entityIDs = user.roles.get(req.regionID, [])
return poem_pb.HTTPLoginResponse(
userID=userID,
sdk_username=user.username,
world=encode_world(
route(req.regionID, entityIDs[0] if entityIDs else None)),
verify_code=request.sid,
)
def _arp_spoof_loop(self):
prev_ip_mac_dict = None
while True:
if not self._host_state.is_inspecting():
time.sleep(2)
continue
time.sleep(1)
with self._lock:
if not self._active:
return
with self._host_state.lock:
if not self._host_state.has_consent:
utils.log('[ARP Spoof] No consent; no spoofing.')
continue
# Get ARP cache
ip_mac_dict = self._host_state.get_ip_mac_dict_copy()
gateway_ip = self._host_state.gateway_ip
if str(ip_mac_dict) != str(prev_ip_mac_dict):
prev_ip_mac_dict = ip_mac_dict
utils.log('[ARP Spoof] Cache:', ip_mac_dict)
utils.log('[ARP Spoof] Whitelist:',
self._host_state.device_whitelist)
# Get gateway MAC addr
try:
gateway_mac = ip_mac_dict[gateway_ip]
except KeyError:
continue
whitelist_ip_mac = []
# Add gateway
whitelist_ip_mac.append((gateway_ip, gateway_mac))
# Build device-to-device whitelist
for ip, mac in ip_mac_dict.items():
device_id = utils.get_device_id(mac, self._host_state)
if device_id not in self._host_state.device_whitelist:
utils.log('[ARP Spoof] Ignore:', ip, mac)
continue
whitelist_ip_mac.append((ip, mac))
# Spoof individual devices on the network.
for (victim_ip, victim_mac) in ip_mac_dict.items():
if victim_ip == gateway_ip:
continue
# Check against whitelist.
victim_device_id = \
utils.get_device_id(victim_mac, self._host_state)
if victim_device_id not in self._host_state.device_whitelist:
utils.log('[ARP Spoof] Ignore:', victim_ip, victim_mac)
continue
if utils.TEST_OUI_LIST:
victim_mac_oui = utils.get_oui(victim_mac)
if victim_mac_oui not in utils.TEST_OUI_LIST:
continue
utils.safe_run(self._arp_spoof,
args=(victim_mac, victim_ip, whitelist_ip_mac))
with self._lock:
if not self._active:
return
time.sleep(max(MIN_ARP_SPOOF_INTERVAL, 2.0 / len(ip_mac_dict)))
def _process_tcp_udp_flow(self, pkt, protocol):
if protocol == 'tcp':
layer = sc.TCP
elif protocol == 'udp':
layer = sc.UDP
else:
return
# Parse packet
src_mac = pkt[sc.Ether].src
dst_mac = pkt[sc.Ether].dst
src_ip = pkt[sc.IP].src
dst_ip = pkt[sc.IP].dst
src_port = pkt[layer].sport
dst_port = pkt[layer].dport
# No broadcast
if dst_mac == 'ff:ff:ff:ff:ff:ff' or dst_ip == '255.255.255.255':
return
# Only look at flows where this host pretends to be the gateway
host_mac = self._host_state.host_mac
# Extract TCP sequence and ack numbers for us to estimate flow size
# later
tcp_seq = None
tcp_ack = None
try:
tcp_layer = pkt[sc.TCP]
tcp_seq = tcp_layer.seq
if tcp_layer.ack > 0:
tcp_ack = tcp_layer.ack
except Exception:
pass
# Determine flow direction
if src_mac == host_mac:
direction = 'inbound'
device_mac = dst_mac
device_port = dst_port
remote_ip = src_ip
remote_port = src_port
elif dst_mac == host_mac:
direction = 'outbound'
device_mac = src_mac
device_port = src_port
remote_ip = dst_ip
remote_port = dst_port
else:
return
# Anonymize device mac
device_id = utils.get_device_id(device_mac, self._host_state)
# Get remote device_id for internal book-keeping purpose
remote_device_id = ''
remote_ip_is_inspector_host = 0 # True (1) or False (0)
remote_ip_is_gateway = 0 # True (1) or False (0)
try:
with self._host_state.lock:
real_remote_device_mac = self._host_state.ip_mac_dict[
remote_ip]
remote_device_id = utils.get_device_id(real_remote_device_mac,
self._host_state)
if remote_ip == self._host_state.host_ip:
remote_ip_is_inspector_host = 1
elif remote_ip == self._host_state.gateway_ip:
remote_ip_is_gateway = 1
except Exception:
pass
# Construct flow key
flow_key = (device_id, device_port, remote_ip, remote_port, protocol)
flow_key_str = ':'.join([str(item) for item in flow_key])
flow_ts = time.time()
# Initialize flow_stats. Note: TCP byte counts may include out-of-order
# packets and RSTs. On the other hand, TCP sequence number shows how
# many unique bytes are transmitted in a flow; this number does not
# consider the size of headers (Eth + IP + TCP = 66 bytes in my
# experiment), out-of-order packets, or RSTs.
flow_stats = {
'inbound_byte_count': 0,
'inbound_tcp_seq_min_max': (None, None),
'inbound_tcp_ack_min_max': (None, None),
'outbound_byte_count': 0,
'outbound_tcp_seq_min_max': (None, None),
'outbound_tcp_ack_min_max': (None, None),
'syn_originator': None,
'internal_remote_device_id': remote_device_id,
'internal_first_packet_originator': '',
'internal_remote_ip_is_inspector_host':
remote_ip_is_inspector_host,
'internal_remote_ip_is_gateway': remote_ip_is_gateway,
'internal_inbound_pkt_count': 0,
'internal_outbound_pkt_count': 0,
'internal_flow_ts_min': flow_ts,
'internal_flow_ts_max': flow_ts,
'internal_flow_key': flow_key_str
}
with self._host_state.lock:
flow_stats = self._host_state.pending_flow_dict \
.setdefault(flow_key, flow_stats)
# Identify who sent out the first UDP packet in this flow
if flow_stats['internal_first_packet_originator'] == '':
if direction == 'inbound':
flow_stats['internal_first_packet_originator'] = 'remote'
else:
flow_stats['internal_first_packet_originator'] = 'local'
# Construct flow_stats
flow_stats[direction + '_byte_count'] += len(pkt)
flow_stats[direction + '_tcp_seq_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_seq_min_max'], tcp_seq)
flow_stats[direction + '_tcp_ack_min_max'] = utils.get_min_max_tuple(
flow_stats[direction + '_tcp_ack_min_max'], tcp_ack)
flow_stats['internal_' + direction + '_pkt_count'] += 1
flow_stats['internal_flow_ts_max'] = flow_ts
# Who initiated the SYN packet
syn_originator = None
try:
if pkt[sc.TCP].flags == 2:
if src_ip == remote_ip:
syn_originator = 'remote'
else:
syn_originator = 'local'
except Exception:
pass
if syn_originator and flow_stats['syn_originator'] is None:
flow_stats['syn_originator'] = syn_originator
# Extract UA and Host
if remote_port == 80 and protocol == 'tcp':
self._process_http(pkt, device_id, remote_ip)
# Extract SNI from TLS client handshake
if protocol == 'tcp':
self._process_tls(pkt, device_id)
with self._host_state.lock:
self._host_state.byte_count += len(pkt)
def _prepare_upload_data(self):
"""Returns (window_duration, a dictionary of data to post)."""
window_duration = time.time() - self._last_upload_ts
# Remove all pending tasks
with self._host_state.lock:
dns_dict = self._host_state.pending_dns_dict
dhcp_dict = self._host_state.pending_dhcp_dict
resolver_dict = self._host_state.pending_resolver_dict
flow_dict = self._host_state.pending_flow_dict
ua_dict = self._host_state.pending_ua_dict
ip_mac_dict = self._host_state.ip_mac_dict
tls_dict_list = self._host_state.pending_tls_dict_list
netdisco_dict = self._host_state.pending_netdisco_dict
self._clear_host_state_pending_data()
self._last_upload_ts = time.time()
# Turn IP -> MAC dict into device_id -> (ip, device_oui) dict, ignoring
# gateway's IP.
device_dict = {}
for (ip, mac) in ip_mac_dict.iteritems():
# Never include the gateway
if ip == self._host_state.gateway_ip:
continue
device_id = utils.get_device_id(mac, self._host_state)
oui = utils.get_oui(mac)
device_dict[device_id] = (ip, oui)
# Process flow_stats
for flow_key in flow_dict:
flow_stats = flow_dict[flow_key]
# Compute unique byte count during this window using seq number
for direction in ('inbound', 'outbound'):
flow_stats[direction + '_tcp_seq_range'] = get_seq_diff(
flow_stats[direction + '_tcp_seq_min_max']
)
flow_stats[direction + '_tcp_ack_range'] = get_seq_diff(
flow_stats[direction + '_tcp_ack_min_max']
)
# We use the original byte count or the sequence number as the
# final byte count (whichever is larger), although we should
# note the caveats of using TCP seq numbers to estimate flow
# size in packet_processor.py.
flow_stats[direction + '_byte_count'] = max(
flow_stats[direction + '_byte_count'],
flow_stats[direction + '_tcp_seq_range']
)
# Fill in missing byte count (e.g., due to failure of ARP spoofing)
if flow_stats['inbound_byte_count'] == 0:
outbound_seq_diff = flow_stats['outbound_tcp_ack_range']
if outbound_seq_diff:
flow_stats['inbound_byte_count'] = outbound_seq_diff
if flow_stats['outbound_byte_count'] == 0:
inbound_seq_diff = flow_stats['inbound_tcp_ack_range']
if inbound_seq_diff:
flow_stats['outbound_byte_count'] = inbound_seq_diff
# Keep only the byte count fields
flow_dict[flow_key] = {
'inbound_byte_count': flow_stats['inbound_byte_count'],
'outbound_byte_count': flow_stats['outbound_byte_count'],
'syn_originator': flow_stats['syn_originator']
}
return (window_duration, {
'dns_dict': jsonify_dict(dns_dict),
'flow_dict': jsonify_dict(flow_dict),
'device_dict': jsonify_dict(device_dict),
'ua_dict': jsonify_dict(ua_dict),
'dhcp_dict': jsonify_dict(dhcp_dict),
'resolver_dict': jsonify_dict(resolver_dict),
'client_version': self._host_state.client_version,
'tls_dict_list': json.dumps(tls_dict_list),
'netdisco_dict': jsonify_dict(netdisco_dict),
'duration': str(window_duration),
'client_ts': str(int(time.time()))
})
def _prepare_upload_data(self):
# Remove all pending tasks
with self._host_state.lock:
dns_responses = self._host_state.pending_dns_responses
pkts = self._host_state.pending_pkts
ua_list = list(self._host_state.ua_set)
self._host_state.pending_dns_responses = []
self._host_state.pending_pkts = []
self._host_state.ua_set = set()
# Aggregate all DNS responses. Build a mapping of domain -> ip_list.
dns_dict = {}
for record in dns_responses:
ip_set = dns_dict.setdefault(record['domain'], set())
dns_dict[record['domain']] = ip_set | record['ip_set']
for domain in dns_dict:
dns_dict[domain] = list(dns_dict[domain])
# Aggregate all pkts into flows. Maps (device_id, device_oui,
# device_ip) -> (remote_ip, remote_port, direction, protocol) ->
# length.
flow_dict = {}
byte_count = 0
for pkt in pkts:
device_mac = pkt['device_mac']
device_oui = device_mac.replace(':', '').lower()[0:6]
device_id = utils.get_device_id(device_mac, self._host_state)
if device_id not in self._host_state.device_whitelist:
continue
device_key = json.dumps((device_id, device_oui, pkt['device_ip']))
device_flow_dict = flow_dict.setdefault(device_key, {})
flow_key = json.dumps((pkt['remote_ip'], pkt['remote_port'],
pkt['direction'], pkt['protocol']))
device_flow_dict.setdefault(flow_key, 0)
device_flow_dict[flow_key] += pkt['length']
byte_count += pkt['length']
# Collect arp_cache
ip_mac_dict = self._host_state.get_ip_mac_dict_copy()
arp_cache = []
for (ip, mac) in ip_mac_dict.iteritems():
arp_cache.append({
'device_ip':
ip,
'device_id':
utils.get_device_id(mac, self._host_state),
'device_oui':
mac.replace(':', '').lower()[0:6]
})
# Turn device_mac into device_id in ua_list
ua_list = [(utils.get_device_id(mac, self._host_state), ua)
for (mac, ua) in ua_list]
return (dns_dict, flow_dict, byte_count, arp_cache, ua_list)