Пример #1
0
    def _process_dhcp(self, pkt):
        """
        Extracts the client hostname from DHCP Request packets.

        """
        try:
            option_dict = dict(
                [t for t in pkt[sc.DHCP].options if isinstance(t, tuple)])

        except Exception:
            return

        try:
            device_hostname = option_dict.setdefault('hostname',
                                                     '').decode('utf-8')
        except Exception:
            device_hostname = ''
        resolver_ip = option_dict.setdefault('name_server', '')

        with self._host_state.lock:

            if device_hostname:

                # Must be a DHCP Request broadcast
                if pkt[sc.Ether].dst != 'ff:ff:ff:ff:ff:ff':
                    return

                device_mac = pkt[sc.Ether].src
                device_id = utils.get_device_id(device_mac, self._host_state)

                self._host_state.pending_dhcp_dict[device_id] = device_hostname
                utils.log('[UPLOAD] DHCP Hostname:', device_hostname)

            if resolver_ip:

                # DHCP Offer broadcast
                if pkt[sc.Ether].dst == 'ff:ff:ff:ff:ff:ff':
                    device_id = 'broadcast'

                # DHCP ACK from router to device. The following block may not
                # actually be called at all, because the router is likely to
                # send the ACK packet directly to the device (rather than arp
                # spoofed)
                else:
                    device_ip = pkt[sc.IP].dst
                    try:
                        device_mac = self._host_state.ip_mac_dict[device_ip]
                    except KeyError:
                        return
                    device_id = utils.get_device_id(device_mac,
                                                    self._host_state)

                self._host_state.pending_resolver_dict[device_id] = \
                    resolver_ip

                utils.log('[UPLOAD] DHCP Resolver:', device_id, '-',
                          resolver_ip)
Пример #2
0
def create_user(request):
    req = poem_pb.RegisterRequest()
    req.ParseFromString(request.body)
    clientIP = request.env.get('REMOTE_ADDR', '')
    username = req.username
    password = req.password
    # 大小包处理
    try:
        if UsernameIndexing.exists(username):
            raise RegisterError(_YYTEXT(u'该帐号已被注册,请重新输入'))
        user = register(username, password, get_device_id(req.deviceInfo))
    except RegisterError as e:
        raise ApplicationError(0, e.message)

    info = unpack_login_info(req.deviceInfo,
                             userID=user.userID,
                             username=user.username,
                             featureCode=req.featureCode,
                             clientIP=clientIP)

    info.update({
        'userID': user.userID,
        'username': user.username,
        'type': 'register'
    })
    gm_logger.info({'sessionaccess': info})
    account_register.info(**info)

    return poem_pb.RegisterResponse(username=user.username)
    def _parse_netdisco_output(self, line):

        try:
            data = json.loads(line)
        except ValueError:
            return

        if 'version' in data:
            return

        device_type = data['device']

        for device_info in data['info']:

            # Find IP
            device_ip = device_info['host']
            device_info['device_type'] = device_type

            # Find MAC based on IP
            try:
                with self._host_state.lock:
                    device_mac = self._host_state.ip_mac_dict[device_ip]
            except KeyError:
                return

            # Get device_id based on MAC
            device_id = utils.get_device_id(device_mac, self._host_state)

            # Submit for upload lter
            with self._host_state.lock:
                self._host_state.pending_netdisco_dict \
                    .setdefault(device_id, []).append(device_info)
Пример #4
0
    def _init(self):
        args = self._args
        self._device = torch.device(
            utils.get_device_id(torch.cuda.is_available()))
        self._aug = utils.Augmentations(method=args.aug)

        self._dataset = data.Dataset(root=args.root,
                                     name=args.name,
                                     num_parts=args.init_parts,
                                     final_parts=args.final_parts,
                                     augumentation=self._aug)
        self._loader = DataLoader(
            dataset=self._dataset)  # [self._dataset.data]
        print(f"Data Augmentation method {args.aug}")
        print(f"Data: {self._dataset.data}")
        hidden_layers = [int(l) for l in args.layers]
        layers = [self._dataset.data.x.shape[1]] + hidden_layers
        self._norm_config = utils.get_norm_configs(args.norms)
        self._model = models.SelfGNN(layer_config=layers,
                                     dropout=args.dropout,
                                     gnn_type=args.model,
                                     heads=args.heads,
                                     **self._norm_config).to(self._device)

        print(self._model)
        self._optimizer = torch.optim.Adam(params=self._model.parameters(),
                                           lr=args.lr)
Пример #5
0
    def _arp_spoof_loop(self):

        while True:

            time.sleep(1)

            with self._lock:
                if not self._active:
                    return

            with self._host_state.lock:
                if not self._host_state.has_consent:
                    utils.log('[ARP Spoof] No consent; no spoofing.')
                    continue

            # Get ARP cache
            ip_mac_dict = self._host_state.get_ip_mac_dict_copy()
            gateway_ip = self._host_state.gateway_ip

            utils.log('[ARP Spoof] Cache:', ip_mac_dict)
            utils.log('[ARP Spoof] Whitelist:',
                      self._host_state.device_whitelist)

            # Get gateway MAC addr
            try:
                gateway_mac = ip_mac_dict[gateway_ip]
            except KeyError:
                continue

            # Spoof individual devices on the network.
            for (victim_ip, victim_mac) in ip_mac_dict.items():

                if victim_ip == gateway_ip:
                    continue

                # Check against whitelist.
                victim_device_id = \
                    utils.get_device_id(victim_mac, self._host_state)
                victim_mac_last_four_bytes = \
                    victim_mac.lower().replace(':', '')[-4:]
                is_in_whitelist = (victim_mac_last_four_bytes
                                   in self._host_state.device_whitelist
                                   or victim_device_id
                                   in self._host_state.device_whitelist)
                if not is_in_whitelist:
                    utils.log('[ARP Spoof] Ignore:', victim_ip, victim_mac)
                    continue

                utils.safe_run(self._arp_spoof,
                               args=(victim_mac, victim_ip, gateway_mac,
                                     gateway_ip))

                with self._lock:
                    if not self._active:
                        return

                time.sleep(max(MIN_ARP_SPOOF_INTERVAL, 2.0 / len(ip_mac_dict)))
Пример #6
0
def get_device_list():
    """
    Returns a list of devices; constantly changes.

    """
    # Maps device_id -> {device_id, device_vendor, netdisco_name}
    output_dict = {}

    host_state = get_host_state()
    if host_state is None:
        return json.dumps(output_dict)

    # Get device vendor
    with host_state.lock:
        for (ip, mac) in host_state.ip_mac_dict.items():

            # Never include the gateway
            if ip == host_state.gateway_ip:
                continue

            device_id = utils.get_device_id(mac, host_state)
            device_vendor = oui_parser.get_vendor(utils.get_oui(mac))

            output_dict.setdefault(
                device_id, {
                    'device_id': device_id,
                    'device_ip': ip,
                    'device_vendor': device_vendor,
                    'netdisco_name': '',
                    'dhcp_name': '',
                    'is_inspected': device_id in host_state.device_whitelist
                })

    # Fill out netdisco_name

    with host_state.lock:
        for (device_id,
             device_info_list) in host_state.pending_netdisco_dict.items():
            if device_id in output_dict:
                output_dict[device_id]['netdisco_name'] = device_info_list

        # Reset pending dict
        host_state.pending_netdisco_dict = {}

    # Fill out dhcp_name

    with host_state.lock:
        for (device_id, dhcp_name) in host_state.pending_dhcp_dict.items():
            if device_id in output_dict:
                output_dict[device_id]['dhcp_name'] = dhcp_name

        # Reset pending dict
        host_state.pending_dhcp_dict = {}

    return json.dumps(output_dict, indent=2)
Пример #7
0
    def _process_dns(self, pkt):

        src_mac = pkt[sc.Ether].src
        dst_mac = pkt[sc.Ether].dst

        src_ip = pkt[sc.IP].src
        dst_ip = pkt[sc.IP].dst

        # Find device ID
        if pkt[sc.DNS].qr == 0:
            # DNS request
            if dst_mac == self._host_state.host_mac:
                device_mac = src_mac
                resolver_ip = dst_ip
            else:
                return
        else:
            # DNS response
            if src_mac == self._host_state.host_mac:
                device_mac = dst_mac
                resolver_ip = src_ip
            else:
                return

        device_id = utils.get_device_id(device_mac, self._host_state)

        # Parse domain
        try:
            domain = pkt[sc.DNSQR].qname.lower()
        except AttributeError:
            return

        # Remove trailing dot from domain
        if domain[-1] == '.':
            domain = domain[0:-1]

        # Parse DNS response
        ip_set = set()
        if sc.DNSRR in pkt and pkt[sc.DNS].an:
            for ix in range(pkt[sc.DNS].ancount):
                # Extracts A-records
                if pkt[sc.DNSRR][ix].type == 1:
                    # Extracts IPv4 addr in A-record
                    ip = pkt[sc.DNSRR][ix].rdata
                    if utils.is_ipv4_addr(ip):
                        ip_set.add(ip)

        with self._host_state.lock:
            domain = str(domain)
            dns_key = (device_id, domain, resolver_ip, 0)
            current_ip_set = self._host_state \
                .pending_dns_dict.setdefault(dns_key, set())
            ip_set = ip_set | current_ip_set
            self._host_state.pending_dns_dict[dns_key] = ip_set
Пример #8
0
def get_device_list_helper():

    # Maps device_id -> {device_id, device_vendor, netdisco_name}
    output_dict = {}

    host_state = get_host_state()
    if host_state is None:
        return json.dumps(output_dict)

    # Get device vendor
    with host_state.lock:
        for (ip, mac) in host_state.ip_mac_dict.items():

            # Never include the gateway
            if ip == host_state.gateway_ip:
                continue

            device_id = utils.get_device_id(mac, host_state)

            output_dict.setdefault(
                device_id, {
                    'device_id': device_id,
                    'device_ip': ip,
                    'device_name': device_identification.get_device_name(mac),
                    'device_vendor':
                    device_identification.get_device_vendor(mac),
                    'netdisco_name': '',
                    'dhcp_name': '',
                    'is_inspected': device_id in host_state.device_whitelist
                })

    # Fill out netdisco_name

    with host_state.lock:
        for (device_id,
             device_info_list) in host_state.pending_netdisco_dict.items():
            if device_id in output_dict:
                output_dict[device_id]['netdisco_name'] = device_info_list

        # Reset pending dict
        host_state.pending_netdisco_dict = {}

    # Fill out dhcp_name

    with host_state.lock:
        for (device_id, dhcp_name) in host_state.pending_dhcp_dict.items():
            if device_id in output_dict:
                output_dict[device_id]['dhcp_name'] = dhcp_name

        # Reset pending dict
        host_state.pending_dhcp_dict = {}

    return output_dict
Пример #9
0
    def _process_syn_scan(self, pkt):
        """
        Receives SYN scan response from devices.

        """
        src_mac = pkt[sc.Ether].src
        device_id = utils.get_device_id(src_mac, self._host_state)
        device_port = pkt[sc.TCP].sport

        with self._host_state.lock:
            port_list = self._host_state.pending_syn_scan_dict.setdefault(
                device_id, [])
            if device_port not in port_list:
                port_list.append(device_port)
                utils.log('[SYN Scan Debug] Device {} ({}): Port {}'.format(
                    pkt[sc.IP].src, device_id, device_port))
    def _process_dhcp(self, pkt):
        """
        Extracts the client hostname from DHCP Request packets.

        """
        try:
            if pkt[sc.Ether].dst != 'ff:ff:ff:ff:ff:ff':
                return

            device_mac = pkt[sc.Ether].src

            device_hostname = dict([
                t for t in pkt[sc.DHCP].options if isinstance(t, tuple)
            ])['hostname']

        except Exception:
            return

        device_id = utils.get_device_id(device_mac, self._host_state)
        with self._host_state.lock:
            self._host_state.pending_dhcp_dict[device_id] = \
                device_hostname
Пример #11
0
    def _run_netdisco(self):

        netdis = NetworkDiscovery()
        netdis.scan()

        for device_type in netdis.discover():
            device_info = netdis.get_info(device_type)[0]
            device_ip = device_info['host']
            device_info['device_type'] = device_type

            # Find MAC based on IP
            try:
                with self._host_state.lock:
                    device_mac = self._host_state.ip_mac_dict[device_ip]
            except KeyError:
                continue

            # Get device_id based on MAC
            device_id = utils.get_device_id(device_mac, self._host_state)

            # Submit for upload lter
            with self._host_state.lock:
                self._host_state.pending_netdisco_dict \
                    .setdefault(device_id, []).append(device_info)
Пример #12
0
    def _process_tcp_udp_flow(self, pkt, protocol):

        if protocol == 'tcp':
            layer = sc.TCP
        elif protocol == 'udp':
            layer = sc.UDP
        else:
            return

        # Parse packet
        src_mac = pkt[sc.Ether].src
        dst_mac = pkt[sc.Ether].dst
        src_ip = pkt[sc.IP].src
        dst_ip = pkt[sc.IP].dst
        src_port = pkt[layer].sport
        dst_port = pkt[layer].dport

        # No broadcast
        if dst_mac == 'ff:ff:ff:ff:ff:ff' or dst_ip == '255.255.255.255':
            return

        # Only look at flows where this host pretends to be the gateway
        host_mac = self._host_state.host_mac
        host_gateway_inbound = (src_mac == host_mac)
        host_gateway_outbound = (dst_mac == host_mac)
        if not (host_gateway_inbound or host_gateway_outbound):
            return

        # Extract TCP sequence and ack numbers for us to estimate flow size
        # later
        tcp_seq = None
        tcp_ack = None

        try:
            tcp_layer = pkt[sc.TCP]
            tcp_seq = tcp_layer.seq
            if tcp_layer.ack > 0:
                tcp_ack = tcp_layer.ack
        except Exception:
            pass

        # Determine flow direction
        if src_mac == host_mac:
            direction = 'inbound'
            device_mac = dst_mac
            device_port = dst_port
            remote_ip = src_ip
            remote_port = src_port
        elif dst_mac == host_mac:
            direction = 'outbound'
            device_mac = src_mac
            device_port = src_port
            remote_ip = dst_ip
            remote_port = dst_port
        else:
            return

        # Anonymize device mac
        device_id = utils.get_device_id(device_mac, self._host_state)

        # Construct flow key
        flow_key = (
            device_id, device_port, remote_ip, remote_port, protocol
        )

        # Initialize flow_stats. Note: TCP byte counts may include out-of-order
        # packets and RSTs. On the other hand, TCP sequence number shows how
        # many unique bytes are transmitted in a flow; this number does not
        # consider the size of headers (Eth + IP + TCP = 66 bytes in my
        # experiment), out-of-order packets, or RSTs.
        flow_stats = {
            'inbound_byte_count': 0,
            'inbound_tcp_seq_min_max': (None, None),
            'inbound_tcp_ack_min_max': (None, None),
            'outbound_byte_count': 0,
            'outbound_tcp_seq_min_max': (None, None),
            'outbound_tcp_ack_min_max': (None, None),
            'syn_originator': None
        }
        with self._host_state.lock:
            flow_stats = self._host_state.pending_flow_dict \
                .setdefault(flow_key, flow_stats)

        # Construct flow_stats
        flow_stats[direction + '_byte_count'] += len(pkt)
        flow_stats[direction + '_tcp_seq_min_max'] = utils.get_min_max_tuple(
            flow_stats[direction + '_tcp_seq_min_max'], tcp_seq)
        flow_stats[direction + '_tcp_ack_min_max'] = utils.get_min_max_tuple(
            flow_stats[direction + '_tcp_ack_min_max'], tcp_ack)

        # Who initiated the SYN packet
        syn_originator = None
        try:
            if pkt[sc.TCP].flags == 2:
                if src_ip == remote_ip:
                    syn_originator = 'remote'
                else:
                    syn_originator = 'local'
        except Exception:
            pass

        if syn_originator and flow_stats['syn_originator'] is None:
            flow_stats['syn_originator'] = syn_originator

        # Extract UA and Host
        if remote_port == 80 and protocol == 'tcp':
            self._process_http(pkt, device_id, remote_ip)

        # Extract SNI from TLS client handshake
        if protocol == 'tcp':
            self._process_tls(pkt, device_id)

        with self._host_state.lock:
            self._host_state.byte_count += len(pkt)
Пример #13
0
def common_login(request,
                 raw_username,
                 req,
                 auto_register=False,
                 check_password=True):
    # 检查版本
    try:
        clientversion = int(getattr(req.deviceInfo, 'clientVersion', 1))
    except ValueError:
        clientversion = 1

    if get_device_id(req.deviceInfo) in g_block_devices:
        raise ApplicationError(0, u"该设备已被禁止登录 如有疑问请联系客服")
    if get_g_version() and clientversion and get_g_version() > clientversion:
        raise ApplicationError(msgTips.FAIL_MSG_LOGIN_OLDVERSION,
                               settings.CLIENTOLDVERSION_NOTICE)

    username = raw_username

    # 检查白名单
    if username not in g_whitelist:
        raise ApplicationError(0, get_loginlimitmsg())

    # 查找用户
    userID = UsernameIndexing.get_pk(username)
    if userID:
        user = User.get(userID)
        user.load_containers()

        # 检查设备锁定
        if user.lock_device and get_device_id(
                req.deviceInfo) != user.lock_device:
            raise ApplicationError(0, u'该帐号为试玩帐号,只能在注册时的使用的设备上登录!')
    else:
        user = None

    clientIP = request.env.get('REMOTE_ADDR', '')
    password = getattr(req, 'password', '') or "dummy"

    if not user and auto_register:
        # register
        try:
            user = register(raw_username, password,
                            get_device_id(req.deviceInfo))
        except RegisterError as e:
            raise ApplicationError(0, e.message)

        # 创建成功
        userID = user.userID
        info = unpack_login_info(req.deviceInfo,
                                 userID=userID,
                                 username=user.username,
                                 featureCode=req.featureCode,
                                 clientIP=clientIP)
        info.update({
            'userID': userID,
            'username': user.username,
            'type': 'register'
        })
        gm_logger.info({'sessionaccess': info})
        account_register.info(**info)

    if not user:
        raise ApplicationError(msgTips.FAIL_MSG_LOGIN_WRONG_ACCOUNT)

    # 检查密码
    userID = user.userID
    if check_password:
        hashed = user.password
        if not utils.check_password(password, hashed):
            raise ApplicationError(msgTips.FAIL_MSG_INVALID_PASSWORD)

    # 检查封停
    now = int(time.time())
    if user.blocktime and user.blocktime > now:
        raise ApplicationError(0, u"该账号已被禁止登录 如有疑问请联系客服")
    if user.imsi in g_block_devices:
        raise ApplicationError(0, u"该账号已被禁止登录 如有疑问请联系客服")

    do_login(request, userID)
    user.lastserver = req.regionID
    user.save()

    # log
    info = unpack_login_info(req.deviceInfo,
                             userID=userID,
                             username=user.username,
                             featureCode=req.featureCode,
                             clientIP=clientIP)
    info.update({'userID': userID, 'username': user.username, 'type': 'login'})
    gm_logger.info({'sessionaccess': info})
    account_login.info(**info)

    # 响应
    entityIDs = user.roles.get(req.regionID, [])
    return poem_pb.HTTPLoginResponse(
        userID=userID,
        sdk_username=user.username,
        world=encode_world(
            route(req.regionID, entityIDs[0] if entityIDs else None)),
        verify_code=request.sid,
    )
Пример #14
0
    def _arp_spoof_loop(self):

        prev_ip_mac_dict = None

        while True:

            if not self._host_state.is_inspecting():
                time.sleep(2)
                continue

            time.sleep(1)

            with self._lock:
                if not self._active:
                    return

            with self._host_state.lock:
                if not self._host_state.has_consent:
                    utils.log('[ARP Spoof] No consent; no spoofing.')
                    continue

            # Get ARP cache
            ip_mac_dict = self._host_state.get_ip_mac_dict_copy()
            gateway_ip = self._host_state.gateway_ip

            if str(ip_mac_dict) != str(prev_ip_mac_dict):

                prev_ip_mac_dict = ip_mac_dict

                utils.log('[ARP Spoof] Cache:', ip_mac_dict)
                utils.log('[ARP Spoof] Whitelist:',
                          self._host_state.device_whitelist)

            # Get gateway MAC addr
            try:
                gateway_mac = ip_mac_dict[gateway_ip]
            except KeyError:
                continue

            whitelist_ip_mac = []
            # Add gateway
            whitelist_ip_mac.append((gateway_ip, gateway_mac))

            # Build device-to-device whitelist
            for ip, mac in ip_mac_dict.items():
                device_id = utils.get_device_id(mac, self._host_state)
                if device_id not in self._host_state.device_whitelist:
                    utils.log('[ARP Spoof] Ignore:', ip, mac)
                    continue
                whitelist_ip_mac.append((ip, mac))

            # Spoof individual devices on the network.
            for (victim_ip, victim_mac) in ip_mac_dict.items():

                if victim_ip == gateway_ip:
                    continue

                # Check against whitelist.
                victim_device_id = \
                    utils.get_device_id(victim_mac, self._host_state)
                if victim_device_id not in self._host_state.device_whitelist:
                    utils.log('[ARP Spoof] Ignore:', victim_ip, victim_mac)
                    continue

                if utils.TEST_OUI_LIST:
                    victim_mac_oui = utils.get_oui(victim_mac)
                    if victim_mac_oui not in utils.TEST_OUI_LIST:
                        continue

                utils.safe_run(self._arp_spoof,
                               args=(victim_mac, victim_ip, whitelist_ip_mac))

                with self._lock:
                    if not self._active:
                        return

                time.sleep(max(MIN_ARP_SPOOF_INTERVAL, 2.0 / len(ip_mac_dict)))
Пример #15
0
    def _process_tcp_udp_flow(self, pkt, protocol):

        if protocol == 'tcp':
            layer = sc.TCP
        elif protocol == 'udp':
            layer = sc.UDP
        else:
            return

        # Parse packet
        src_mac = pkt[sc.Ether].src
        dst_mac = pkt[sc.Ether].dst
        src_ip = pkt[sc.IP].src
        dst_ip = pkt[sc.IP].dst
        src_port = pkt[layer].sport
        dst_port = pkt[layer].dport

        # No broadcast
        if dst_mac == 'ff:ff:ff:ff:ff:ff' or dst_ip == '255.255.255.255':
            return

        # Only look at flows where this host pretends to be the gateway
        host_mac = self._host_state.host_mac

        # Extract TCP sequence and ack numbers for us to estimate flow size
        # later
        tcp_seq = None
        tcp_ack = None

        try:
            tcp_layer = pkt[sc.TCP]
            tcp_seq = tcp_layer.seq
            if tcp_layer.ack > 0:
                tcp_ack = tcp_layer.ack
        except Exception:
            pass

        # Determine flow direction
        if src_mac == host_mac:
            direction = 'inbound'
            device_mac = dst_mac
            device_port = dst_port
            remote_ip = src_ip
            remote_port = src_port
        elif dst_mac == host_mac:
            direction = 'outbound'
            device_mac = src_mac
            device_port = src_port
            remote_ip = dst_ip
            remote_port = dst_port
        else:
            return

        # Anonymize device mac
        device_id = utils.get_device_id(device_mac, self._host_state)

        # Get remote device_id for internal book-keeping purpose
        remote_device_id = ''
        remote_ip_is_inspector_host = 0  # True (1) or False (0)
        remote_ip_is_gateway = 0  # True (1) or False (0)
        try:
            with self._host_state.lock:
                real_remote_device_mac = self._host_state.ip_mac_dict[
                    remote_ip]
                remote_device_id = utils.get_device_id(real_remote_device_mac,
                                                       self._host_state)
                if remote_ip == self._host_state.host_ip:
                    remote_ip_is_inspector_host = 1
                elif remote_ip == self._host_state.gateway_ip:
                    remote_ip_is_gateway = 1
        except Exception:
            pass

        # Construct flow key
        flow_key = (device_id, device_port, remote_ip, remote_port, protocol)
        flow_key_str = ':'.join([str(item) for item in flow_key])

        flow_ts = time.time()

        # Initialize flow_stats. Note: TCP byte counts may include out-of-order
        # packets and RSTs. On the other hand, TCP sequence number shows how
        # many unique bytes are transmitted in a flow; this number does not
        # consider the size of headers (Eth + IP + TCP = 66 bytes in my
        # experiment), out-of-order packets, or RSTs.
        flow_stats = {
            'inbound_byte_count': 0,
            'inbound_tcp_seq_min_max': (None, None),
            'inbound_tcp_ack_min_max': (None, None),
            'outbound_byte_count': 0,
            'outbound_tcp_seq_min_max': (None, None),
            'outbound_tcp_ack_min_max': (None, None),
            'syn_originator': None,
            'internal_remote_device_id': remote_device_id,
            'internal_first_packet_originator': '',
            'internal_remote_ip_is_inspector_host':
            remote_ip_is_inspector_host,
            'internal_remote_ip_is_gateway': remote_ip_is_gateway,
            'internal_inbound_pkt_count': 0,
            'internal_outbound_pkt_count': 0,
            'internal_flow_ts_min': flow_ts,
            'internal_flow_ts_max': flow_ts,
            'internal_flow_key': flow_key_str
        }
        with self._host_state.lock:
            flow_stats = self._host_state.pending_flow_dict \
                .setdefault(flow_key, flow_stats)

        # Identify who sent out the first UDP packet in this flow
        if flow_stats['internal_first_packet_originator'] == '':
            if direction == 'inbound':
                flow_stats['internal_first_packet_originator'] = 'remote'
            else:
                flow_stats['internal_first_packet_originator'] = 'local'

        # Construct flow_stats
        flow_stats[direction + '_byte_count'] += len(pkt)
        flow_stats[direction + '_tcp_seq_min_max'] = utils.get_min_max_tuple(
            flow_stats[direction + '_tcp_seq_min_max'], tcp_seq)
        flow_stats[direction + '_tcp_ack_min_max'] = utils.get_min_max_tuple(
            flow_stats[direction + '_tcp_ack_min_max'], tcp_ack)
        flow_stats['internal_' + direction + '_pkt_count'] += 1
        flow_stats['internal_flow_ts_max'] = flow_ts

        # Who initiated the SYN packet
        syn_originator = None
        try:
            if pkt[sc.TCP].flags == 2:
                if src_ip == remote_ip:
                    syn_originator = 'remote'
                else:
                    syn_originator = 'local'
        except Exception:
            pass

        if syn_originator and flow_stats['syn_originator'] is None:
            flow_stats['syn_originator'] = syn_originator

        # Extract UA and Host
        if remote_port == 80 and protocol == 'tcp':
            self._process_http(pkt, device_id, remote_ip)

        # Extract SNI from TLS client handshake
        if protocol == 'tcp':
            self._process_tls(pkt, device_id)

        with self._host_state.lock:
            self._host_state.byte_count += len(pkt)
Пример #16
0
    def _prepare_upload_data(self):
        """Returns (window_duration, a dictionary of data to post)."""

        window_duration = time.time() - self._last_upload_ts

        # Remove all pending tasks
        with self._host_state.lock:

            dns_dict = self._host_state.pending_dns_dict
            dhcp_dict = self._host_state.pending_dhcp_dict
            resolver_dict = self._host_state.pending_resolver_dict
            flow_dict = self._host_state.pending_flow_dict
            ua_dict = self._host_state.pending_ua_dict
            ip_mac_dict = self._host_state.ip_mac_dict
            tls_dict_list = self._host_state.pending_tls_dict_list
            netdisco_dict = self._host_state.pending_netdisco_dict

            self._clear_host_state_pending_data()

            self._last_upload_ts = time.time()

        # Turn IP -> MAC dict into device_id -> (ip, device_oui) dict, ignoring
        # gateway's IP.
        device_dict = {}
        for (ip, mac) in ip_mac_dict.iteritems():
            # Never include the gateway
            if ip == self._host_state.gateway_ip:
                continue
            device_id = utils.get_device_id(mac, self._host_state)
            oui = utils.get_oui(mac)
            device_dict[device_id] = (ip, oui)

        # Process flow_stats
        for flow_key in flow_dict:

            flow_stats = flow_dict[flow_key]

            # Compute unique byte count during this window using seq number
            for direction in ('inbound', 'outbound'):
                flow_stats[direction + '_tcp_seq_range'] = get_seq_diff(
                    flow_stats[direction + '_tcp_seq_min_max']
                )
                flow_stats[direction + '_tcp_ack_range'] = get_seq_diff(
                    flow_stats[direction + '_tcp_ack_min_max']
                )

                # We use the original byte count or the sequence number as the
                # final byte count (whichever is larger), although we should
                # note the caveats of using TCP seq numbers to estimate flow
                # size in packet_processor.py.
                flow_stats[direction + '_byte_count'] = max(
                    flow_stats[direction + '_byte_count'],
                    flow_stats[direction + '_tcp_seq_range']
                )

            # Fill in missing byte count (e.g., due to failure of ARP spoofing)
            if flow_stats['inbound_byte_count'] == 0:
                outbound_seq_diff = flow_stats['outbound_tcp_ack_range']
                if outbound_seq_diff:
                    flow_stats['inbound_byte_count'] = outbound_seq_diff
            if flow_stats['outbound_byte_count'] == 0:
                inbound_seq_diff = flow_stats['inbound_tcp_ack_range']
                if inbound_seq_diff:
                    flow_stats['outbound_byte_count'] = inbound_seq_diff

            # Keep only the byte count fields
            flow_dict[flow_key] = {
                'inbound_byte_count': flow_stats['inbound_byte_count'],
                'outbound_byte_count': flow_stats['outbound_byte_count'],
                'syn_originator': flow_stats['syn_originator']
            }

        return (window_duration, {
            'dns_dict': jsonify_dict(dns_dict),
            'flow_dict': jsonify_dict(flow_dict),
            'device_dict': jsonify_dict(device_dict),
            'ua_dict': jsonify_dict(ua_dict),
            'dhcp_dict': jsonify_dict(dhcp_dict),
            'resolver_dict': jsonify_dict(resolver_dict),
            'client_version': self._host_state.client_version,
            'tls_dict_list': json.dumps(tls_dict_list),
            'netdisco_dict': jsonify_dict(netdisco_dict),
            'duration': str(window_duration),
            'client_ts': str(int(time.time()))
        })
    def _prepare_upload_data(self):

        # Remove all pending tasks
        with self._host_state.lock:

            dns_responses = self._host_state.pending_dns_responses
            pkts = self._host_state.pending_pkts
            ua_list = list(self._host_state.ua_set)

            self._host_state.pending_dns_responses = []
            self._host_state.pending_pkts = []
            self._host_state.ua_set = set()

        # Aggregate all DNS responses. Build a mapping of domain -> ip_list.
        dns_dict = {}
        for record in dns_responses:
            ip_set = dns_dict.setdefault(record['domain'], set())
            dns_dict[record['domain']] = ip_set | record['ip_set']
        for domain in dns_dict:
            dns_dict[domain] = list(dns_dict[domain])

        # Aggregate all pkts into flows.  Maps (device_id, device_oui,
        # device_ip) ->  (remote_ip, remote_port, direction, protocol) ->
        # length.
        flow_dict = {}
        byte_count = 0
        for pkt in pkts:

            device_mac = pkt['device_mac']
            device_oui = device_mac.replace(':', '').lower()[0:6]
            device_id = utils.get_device_id(device_mac, self._host_state)
            if device_id not in self._host_state.device_whitelist:
                continue

            device_key = json.dumps((device_id, device_oui, pkt['device_ip']))
            device_flow_dict = flow_dict.setdefault(device_key, {})

            flow_key = json.dumps((pkt['remote_ip'], pkt['remote_port'],
                                   pkt['direction'], pkt['protocol']))
            device_flow_dict.setdefault(flow_key, 0)
            device_flow_dict[flow_key] += pkt['length']

            byte_count += pkt['length']

        # Collect arp_cache
        ip_mac_dict = self._host_state.get_ip_mac_dict_copy()
        arp_cache = []
        for (ip, mac) in ip_mac_dict.iteritems():
            arp_cache.append({
                'device_ip':
                ip,
                'device_id':
                utils.get_device_id(mac, self._host_state),
                'device_oui':
                mac.replace(':', '').lower()[0:6]
            })

        # Turn device_mac into device_id in ua_list
        ua_list = [(utils.get_device_id(mac, self._host_state), ua)
                   for (mac, ua) in ua_list]

        return (dns_dict, flow_dict, byte_count, arp_cache, ua_list)