def get_from_ip(cls, ip, port, type, hostname=None, timestamp=datetime.utcnow()): try: ip_addr = IPAddress(ip) except netaddr.core.AddrFormatError: logger.debug(f"Failed to parse {ip}") return None geo = None if not ip_addr.is_private() and not ip_addr.is_reserved(): geo = cls.get_geo(ip) if not hostname: hostname = cls.get_rdns(ip) return NetworkEntity(ip, port, type, geo, hostname=hostname, timestamp=timestamp, is_enriched=True)
def is_public_ip(ip): ip_obj = IPAddress(ip) if not ip_obj.is_private() and not ip_obj.is_reserved(): return True return False
def validate_ipv4_address(cls, _, value): """ Ensures the :attr:`ip` address is valid. This checks to ensure that the value provided is: * not a hostmask * not link local (:rfc:`3927`) * not used for multicast (:rfc:`1112`) * not a netmask (:rfc:`4632`) * not reserved (:rfc:`6052`) * a private address (:rfc:`1918`) """ if value is None: return value try: address = IPAddress(value) except (AddrFormatError, ValueError) as e: raise ValueError( "%s is not a valid address format: %s" % (value, e)) if ALLOW_AGENT_LOOPBACK: loopback = lambda: False else: loopback = address.is_loopback if any([address.is_hostmask(), address.is_link_local(), loopback(), address.is_multicast(), address.is_netmask(), address.is_reserved()]): raise ValueError("%s is not a valid address type" % value) return value
def _validate_network_information(self): all_errs = [] if self.subnet_mask_size > self.MAX_SUBNET_MASK_SIZE: all_errs.append( "Subnet size too small. Subnet mask should <= {}, current value: {}" .format(self.MAX_SUBNET_MASK_SIZE, self.subnet_mask_size)) cidr_base_validator = re.compile(self.VPC_CIDR_BASE_REGEX) if not cidr_base_validator.match(self.vpc_cidr_base): all_errs.append( "Invalid VPC CIDR base {}. VPC CIDR base should match regex {}" .format(self.vpc_cidr_base, self.VPC_CIDR_BASE_REGEX)) if self.trusted_cidrs: try: for cidr in self.trusted_cidrs: [ip, mask] = cidr.split("/") if ip == "0.0.0.0": if mask != "0": all_errs.append( "Trusting traffic from everywhere should specify \"0.0.0.0/0\" as trusted CIDR." ) else: if not 0 < int(mask) <= 32: all_errs.append( "Subnet mask {} should be greater than 0 but less than 32." .format(mask)) ipaddr = IPAddress(ip) if ipaddr.is_netmask(): all_errs.append( "Trusted CIDR {} should not be a net mask". format(ip)) if ipaddr.is_hostmask(): all_errs.append( "Trusted CIDR {} should not be a host mask". format(ip)) if ipaddr.is_reserved(): all_errs.append( "Trusted CIDR {} should not be in reserved range" .format(ip)) if ipaddr.is_loopback(): all_errs.append( "Trusted CIDR {} should not be a loop back address" .format(ip)) # Currently we don't support private VPC if ipaddr.is_private(): all_errs.append( "Trusted CIDR {} should not be a private address" .format(ip)) except ValueError as ve: all_errs.append( "Cannot parse trusted CIDRs ({}). Err: {}".format( self.trusted_cidrs, ve)) else: all_errs.append( "Please provide trusted CIDRs through --trusted-cidrs flag") return all_errs
def is_valid_address(address): ''' Validate whether the address provided is routable unicast address ''' addr = IPAddress(address) if addr.is_loopback() or addr.is_reserved() or addr.is_private()\ or addr.is_link_local() or addr.is_multicast(): return False return True
def _trusted_cidr_validator(input): from netaddr import IPAddress ret = [] for cidr in input.split(" "): cidr = cidr.strip() if not cidr: # skip whitespace continue ip, mask = cidr.split("/") if int(mask) < 0 or int(mask) > 32: raise ValueError( "CIDR {} is not valid as mask {} is not in range [0-32]". format(cidr, mask)) if ip != "0.0.0.0" or mask != '0': ipaddr = IPAddress(ip) if ipaddr.is_netmask(): raise ValueError( "Trusted CIDR {} should not be a net mask".format(ip)) if ipaddr.is_hostmask(): raise ValueError( "Trusted CIDR {} should not be a host mask".format(ip)) if ipaddr.is_reserved(): raise ValueError( "Trusted CIDR {} should not be in reserved range". format(ip)) if ipaddr.is_loopback(): raise ValueError( "Trusted CIDR {} should not be a loop back address". format(ip)) # Currently we don't support private VPC if ipaddr.is_private(): raise ValueError( "Trusted CIDR {} should not be a private address". format(ip)) ret.append(cidr) return ret
def get_from_hostname(cls, hostname, type, timestamp=datetime.utcnow()): ip = cls.get_ip(hostname) geo = None try: ip_addr = IPAddress(ip) if ip_addr and not ip_addr.is_private() \ and not ip_addr.is_reserved(): geo = cls.get_geo(ip) except netaddr.core.AddrFormatError: logger.debug(f"Failed to parse {ip}") return NetworkEntity(ip, 0, type, geo, hostname=hostname, timestamp=timestamp, is_enriched=True)
def ip_check_routable(item): ip_addr = IPAddress(item) # This prevents netaddr allowing shortened ip addresses if not str(ip_addr) == item: raise AddrFormatError("IP Malformed {}".format(item)) # Check for reserved IP addresses if any([ ip_addr.is_multicast(), ip_addr.is_private(), ip_addr.is_loopback(), ip_addr.is_link_local(), ip_addr.is_reserved() ]): raise AddrFormatError("IP is reserved {}".format(item)) # Check to see if IP is IPv4 # elif ip_addr.version is not 4: # raise AddrFormatError("IP is not IPv4") return True
def validate_ip_address (address, netmask, gateway = None): """ Validate that the provided IP settings are valid. :param address: The IP address that will be assigned to the interface. :param netmask: The subnet mask for the interface. :param gateway: The default gateway to use for the system. """ ip = IPAddress (address) if (not ip.is_unicast () or ip.is_loopback () or ip.is_reserved ()): raise ValueError ("IP address is not a valid host address.") if (not IPAddress (netmask).is_netmask ()): raise ValueError ("Subnet mask is not valid.") subnet = IPNetwork (address, netmask) if gateway: route = IPNetwork (gateway, netmask) if (subnet != route): raise ValueError ("Gateway supplied is not on the specified subnet.")
def reserved_ip_check(ip_string): """determine if IP address in RFC1918 or reserved""" # IP details for invalid IP addresses invalid_ip_details = { "country": "INVALID", "location": RESERVED_IP_COORDINATES, "subdivisions": "INVALID", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for MULTICAST IP addresses multicast_ip_details = { "country": "MULTICAST", "location": RESERVED_IP_COORDINATES, "subdivisions": "MULTICAST", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for PRIVATE IP addresses private_ip_details = { "country": "PRIVATE", "location": RESERVED_IP_COORDINATES, "subdivisions": "PRIVATE", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for RESERVED IP addresses reserved_ip_details = { "country": "RESERVED", "location": RESERVED_IP_COORDINATES, "subdivisions": "RESERVED", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for NETMASK IP addresses netmask_ip_details = { "country": "NETMASK", "location": RESERVED_IP_COORDINATES, "subdivisions": "NETMASK", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for HOSTMASK IP addresses hostmask_ip_details = { "country": "HOSTMASK", "location": RESERVED_IP_COORDINATES, "subdivisions": "HOSTMASK", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for LOOPBACK IP addresses loopback_ip_details = { "country": "LOOPBACK", "location": RESERVED_IP_COORDINATES, "subdivisions": "LOOPBACK", "dch_company": "", "asn_number": "", "asn_name": "" } # Check to see if IP matches a reserved category try: ip_address = IPAddress(ip_string) except AddrFormatError: return invalid_ip_details if ip_address.is_multicast(): return multicast_ip_details elif ip_address.is_private(): return private_ip_details elif ip_address.is_reserved(): return reserved_ip_details elif ip_address.is_netmask(): return netmask_ip_details elif ip_address.is_hostmask(): return hostmask_ip_details elif ip_address.is_loopback(): return loopback_ip_details elif ip_address.is_unicast() and not ip_address.is_private(): # Boolean to be returned if IP is Public ip_reserved = False return ip_reserved else: return invalid_ip_details
def fifoReader(infile, q, exitSignal): sleeptime = 0.5 maxSleeptime = 1.0 while True: try: if exitSignal.is_set(): break line = infile.readline() if not line: time.sleep(1) continue if line == 'ENDOFFILE': break try: spl = line.split() timestamp, queriedName, clientID, ipv4 = spl except: continue else: if not '.' in queriedName: continue try: addr = IPAddress(ipv4) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): try: timestamp = int(timestamp) except ValueError: continue else: data = ((queriedName, clientID, [addr]), timestamp) queued = False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime *= 0.5 elif q.qsize() >= q._maxsize: sleeptime *= 2 if sleeptime > maxSleeptime: sleeptime = maxSleeptime else: queued = True except KeyboardInterrupt: break q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0): if not infile and not interface: # FIXME: write warning here return if infile: pc = pcap.pcapObject() try: pc.open_offline(infile) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass if interface: pc = pcap.pcapObject() try: #pc.open_live(interface, snaplen, promisc, read_timeout) pc.open_live(interface, 1600, 0, 100) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass except Exception: # most likely we got no permission to open the interface sys.stderr.write('could not open interface. insufficient ' 'permissions?\n') q.put(None) return pc.setfilter('udp', 0, 0) basets = 0 newMappings = dict() while True: if exitSignal.is_set(): break try: packet = pc.next() if not packet: if infile: # end of file break elif interface: # read timeout continue payload = packet[1] timestamp = int(packet[2]) # make sure we are dealing with IP traffic # ref: http://www.iana.org/assignments/ethernet-numbers try: eth = dpkt.ethernet.Ethernet(payload) except: continue if eth.type != 2048: continue # make sure we are dealing with UDP # ref: http://www.iana.org/assignments/protocol-numbers/ try: ip = eth.data except: continue if ip.p != 17: continue # filter on UDP assigned ports for DNS # ref: http://www.iana.org/assignments/port-numbers try: udp = ip.data except: continue if udp.sport != 53 and udp.dport != 53: continue # make the dns object out of the udp data and check for it being a RR (answer) # and for opcode QUERY (I know, counter-intuitive) try: dns = dpkt.dns.DNS(udp.data) except: continue if dns.qr != dpkt.dns.DNS_R: continue if dns.opcode != dpkt.dns.DNS_QUERY: continue if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue if len(dns.an) < 1: continue if len(dns.qd) == 0: continue aRecords = set() queriedName = dns.qd[0].name if not '.' in queriedName: continue #lastCname=queriedName for answer in dns.an: """ FIXME: this doesn't work for multiple queries in one DNS packet """ #if answer.type == dpkt.dns.DNS_CNAME: # lastCname=answer.cname if answer.type == dpkt.dns.DNS_A: ip = socket.inet_ntoa(answer.rdata) try: addr = IPAddress(ip) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): aRecords.add(addr) if thrsh: if (timestamp - basets) > thrsh: basets = timestamp newMappings.clear() newIps = checkMapping(newMappings, queriedName, aRecords) aRecords = newIps if not aRecords: continue data = ((queriedName, ip.dst, aRecords), timestamp) queued = False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime *= 0.5 elif q.qsize() >= q._maxsize: sleeptime *= 2 if sleeptime > maxSleeptime: sleeptime = maxSleeptime else: queued = True except KeyboardInterrupt: break """ send shutdown signal """ q.put(None)
def fifoReader(infile, q, exitSignal): sleeptime=0.5 maxSleeptime=1.0 while True: try: if exitSignal.is_set(): break line=infile.readline() if not line: time.sleep(1) continue if line=='ENDOFFILE': break try: spl=line.split() timestamp, queriedName, clientID, ipv4 = spl except: continue else: if not '.' in queriedName: continue try: addr=IPAddress(ipv4) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): try: timestamp=int(timestamp) except ValueError: continue else: data = ((queriedName, clientID, [addr]), timestamp) queued=False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime*=0.5 elif q.qsize() >= q._maxsize: sleeptime*=2 if sleeptime>maxSleeptime: sleeptime=maxSleeptime else: queued=True except KeyboardInterrupt: break q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0): if not infile and not interface: # FIXME: write warning here return if infile: pc=pcap.pcapObject() try: pc.open_offline(infile) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass if interface: pc=pcap.pcapObject() try: #pc.open_live(interface, snaplen, promisc, read_timeout) pc.open_live(interface, 1600, 0, 100) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass except Exception: # most likely we got no permission to open the interface sys.stderr.write('could not open interface. insufficient ' 'permissions?\n') q.put(None) return pc.setfilter('udp', 0, 0) basets=0 newMappings=dict() while True: if exitSignal.is_set(): break try: packet=pc.next() if not packet: if infile: # end of file break elif interface: # read timeout continue payload=packet[1] timestamp=int(packet[2]) # make sure we are dealing with IP traffic # ref: http://www.iana.org/assignments/ethernet-numbers try: eth = dpkt.ethernet.Ethernet(payload) except: continue if eth.type != 2048: continue # make sure we are dealing with UDP # ref: http://www.iana.org/assignments/protocol-numbers/ try: ip = eth.data except: continue if ip.p != 17: continue # filter on UDP assigned ports for DNS # ref: http://www.iana.org/assignments/port-numbers try: udp = ip.data except: continue if udp.sport != 53 and udp.dport != 53: continue # make the dns object out of the udp data and check for it being a RR (answer) # and for opcode QUERY (I know, counter-intuitive) try: dns = dpkt.dns.DNS(udp.data) except: continue if dns.qr != dpkt.dns.DNS_R: continue if dns.opcode != dpkt.dns.DNS_QUERY: continue if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue if len(dns.an) < 1: continue if len(dns.qd) == 0: continue aRecords=set() queriedName=dns.qd[0].name if not '.' in queriedName: continue #lastCname=queriedName for answer in dns.an: """ FIXME: this doesn't work for multiple queries in one DNS packet """ #if answer.type == dpkt.dns.DNS_CNAME: # lastCname=answer.cname if answer.type == dpkt.dns.DNS_A: ip=socket.inet_ntoa(answer.rdata) try: addr=IPAddress(ip) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): aRecords.add(addr) if thrsh: if (timestamp-basets) > thrsh: basets = timestamp newMappings.clear() newIps = checkMapping(newMappings, queriedName, aRecords) aRecords=newIps if not aRecords: continue data = ((queriedName, ip.dst, aRecords), timestamp) queued=False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime*=0.5 elif q.qsize() >= q._maxsize: sleeptime*=2 if sleeptime>maxSleeptime: sleeptime=maxSleeptime else: queued=True except KeyboardInterrupt: break """ send shutdown signal """ q.put(None)
def inc_stats(self, packet: dict): """ Increments stats of the current Counters :param packet: packet to process into the increment """ ip_origin = IPAddress(packet['origin']['ip']) ip_destination = IPAddress(packet['destination']['ip']) if ip_origin.is_reserved() or ip_destination.is_reserved(): return # infer direction # if both are private then is a private connection # if just one is private or known the we infer the direction internal_ips = ip_origin.is_private() + ip_destination.is_private() + ( str(ip_origin) in Counters.ipset) + (ip_destination in Counters.ipset) if internal_ips == 2: # communication between two internal addresses self.volume_internal = self.volume_internal + packet['length'] self.ip_internal.add(ip_origin) self.ip_internal.add(ip_destination) elif internal_ips == 1: # comunication with and outside entity # infer direction if ip_origin.is_private() or (ip_origin in Counters.ipset): # it's an upload self.volume_upload = self.volume_upload + packet['length'] self.ip_external.add(ip_destination) self.ip_internal.add(ip_origin) elif ip_destination.is_private() or (str(ip_destination) in Counters.ipset): # it's a download self.volume_download = self.volume_download + packet['length'] self.ip_internal.add(ip_destination) self.ip_external.add(ip_origin) else: raise RuntimeError( f"Internal IP is 1 but can't figure out which one: {str(ip_origin)} - {str(ip_destination)} {ip_origin.is_private()} {ip_destination.is_private()}" ) if packet['protocol'] != "ICMP": if packet['destination']['port'] > 1024: self.port_high = self.port_high + 1 else: self.port_low = self.port_low + 1 else: raise RuntimeError( f"Two public IPs and none is known. {str(ip_origin)} - {str(ip_destination)}" ) if packet['protocol'] == "TCP": self.tcp_syn = self.tcp_syn + packet['tcp_flags']['SYN'] self.tcp_fin = self.tcp_fin + packet['tcp_flags']['FIN'] self.tcp_rst = self.tcp_rst + packet['tcp_flags']['RST'] self.packet_count = self.packet_count + 1 # in bytes if int(packet['length']) <= 64: self.less_64b = self.less_64b + 1