def get_from_ip(cls,
ip,
port,
type,
hostname=None,
timestamp=datetime.utcnow()):
try:
ip_addr = IPAddress(ip)
except netaddr.core.AddrFormatError:
logger.debug(f"Failed to parse {ip}")
return None
geo = None
if not ip_addr.is_private() and not ip_addr.is_reserved():
geo = cls.get_geo(ip)
if not hostname:
hostname = cls.get_rdns(ip)
return NetworkEntity(ip,
port,
type,
geo,
hostname=hostname,
timestamp=timestamp,
is_enriched=True)
def is_public_ip(ip):
ip_obj = IPAddress(ip)
if not ip_obj.is_private() and not ip_obj.is_reserved():
return True
return False
def validate_ipv4_address(cls, _, value):
"""
Ensures the :attr:`ip` address is valid. This checks to ensure
that the value provided is:
* not a hostmask
* not link local (:rfc:`3927`)
* not used for multicast (:rfc:`1112`)
* not a netmask (:rfc:`4632`)
* not reserved (:rfc:`6052`)
* a private address (:rfc:`1918`)
"""
if value is None:
return value
try:
address = IPAddress(value)
except (AddrFormatError, ValueError) as e:
raise ValueError(
"%s is not a valid address format: %s" % (value, e))
if ALLOW_AGENT_LOOPBACK:
loopback = lambda: False
else:
loopback = address.is_loopback
if any([address.is_hostmask(), address.is_link_local(),
loopback(), address.is_multicast(),
address.is_netmask(), address.is_reserved()]):
raise ValueError("%s is not a valid address type" % value)
return value
def _validate_network_information(self):
all_errs = []
if self.subnet_mask_size > self.MAX_SUBNET_MASK_SIZE:
all_errs.append(
"Subnet size too small. Subnet mask should <= {}, current value: {}"
.format(self.MAX_SUBNET_MASK_SIZE, self.subnet_mask_size))
cidr_base_validator = re.compile(self.VPC_CIDR_BASE_REGEX)
if not cidr_base_validator.match(self.vpc_cidr_base):
all_errs.append(
"Invalid VPC CIDR base {}. VPC CIDR base should match regex {}"
.format(self.vpc_cidr_base, self.VPC_CIDR_BASE_REGEX))
if self.trusted_cidrs:
try:
for cidr in self.trusted_cidrs:
[ip, mask] = cidr.split("/")
if ip == "0.0.0.0":
if mask != "0":
all_errs.append(
"Trusting traffic from everywhere should specify \"0.0.0.0/0\" as trusted CIDR."
)
else:
if not 0 < int(mask) <= 32:
all_errs.append(
"Subnet mask {} should be greater than 0 but less than 32."
.format(mask))
ipaddr = IPAddress(ip)
if ipaddr.is_netmask():
all_errs.append(
"Trusted CIDR {} should not be a net mask".
format(ip))
if ipaddr.is_hostmask():
all_errs.append(
"Trusted CIDR {} should not be a host mask".
format(ip))
if ipaddr.is_reserved():
all_errs.append(
"Trusted CIDR {} should not be in reserved range"
.format(ip))
if ipaddr.is_loopback():
all_errs.append(
"Trusted CIDR {} should not be a loop back address"
.format(ip))
# Currently we don't support private VPC
if ipaddr.is_private():
all_errs.append(
"Trusted CIDR {} should not be a private address"
.format(ip))
except ValueError as ve:
all_errs.append(
"Cannot parse trusted CIDRs ({}). Err: {}".format(
self.trusted_cidrs, ve))
else:
all_errs.append(
"Please provide trusted CIDRs through --trusted-cidrs flag")
return all_errs
def is_valid_address(address):
''' Validate whether the address provided is routable unicast address '''
addr = IPAddress(address)
if addr.is_loopback() or addr.is_reserved() or addr.is_private()\
or addr.is_link_local() or addr.is_multicast():
return False
return True
def _trusted_cidr_validator(input):
from netaddr import IPAddress
ret = []
for cidr in input.split(" "):
cidr = cidr.strip()
if not cidr:
# skip whitespace
continue
ip, mask = cidr.split("/")
if int(mask) < 0 or int(mask) > 32:
raise ValueError(
"CIDR {} is not valid as mask {} is not in range [0-32]".
format(cidr, mask))
if ip != "0.0.0.0" or mask != '0':
ipaddr = IPAddress(ip)
if ipaddr.is_netmask():
raise ValueError(
"Trusted CIDR {} should not be a net mask".format(ip))
if ipaddr.is_hostmask():
raise ValueError(
"Trusted CIDR {} should not be a host mask".format(ip))
if ipaddr.is_reserved():
raise ValueError(
"Trusted CIDR {} should not be in reserved range".
format(ip))
if ipaddr.is_loopback():
raise ValueError(
"Trusted CIDR {} should not be a loop back address".
format(ip))
# Currently we don't support private VPC
if ipaddr.is_private():
raise ValueError(
"Trusted CIDR {} should not be a private address".
format(ip))
ret.append(cidr)
return ret
def get_from_hostname(cls, hostname, type, timestamp=datetime.utcnow()):
ip = cls.get_ip(hostname)
geo = None
try:
ip_addr = IPAddress(ip)
if ip_addr and not ip_addr.is_private() \
and not ip_addr.is_reserved():
geo = cls.get_geo(ip)
except netaddr.core.AddrFormatError:
logger.debug(f"Failed to parse {ip}")
return NetworkEntity(ip,
0,
type,
geo,
hostname=hostname,
timestamp=timestamp,
is_enriched=True)
def ip_check_routable(item):
ip_addr = IPAddress(item)
# This prevents netaddr allowing shortened ip addresses
if not str(ip_addr) == item:
raise AddrFormatError("IP Malformed {}".format(item))
# Check for reserved IP addresses
if any([
ip_addr.is_multicast(),
ip_addr.is_private(),
ip_addr.is_loopback(),
ip_addr.is_link_local(),
ip_addr.is_reserved()
]):
raise AddrFormatError("IP is reserved {}".format(item))
# Check to see if IP is IPv4
# elif ip_addr.version is not 4:
# raise AddrFormatError("IP is not IPv4")
return True
def validate_ip_address (address, netmask, gateway = None):
"""
Validate that the provided IP settings are valid.
:param address: The IP address that will be assigned to the interface.
:param netmask: The subnet mask for the interface.
:param gateway: The default gateway to use for the system.
"""
ip = IPAddress (address)
if (not ip.is_unicast () or ip.is_loopback () or ip.is_reserved ()):
raise ValueError ("IP address is not a valid host address.")
if (not IPAddress (netmask).is_netmask ()):
raise ValueError ("Subnet mask is not valid.")
subnet = IPNetwork (address, netmask)
if gateway:
route = IPNetwork (gateway, netmask)
if (subnet != route):
raise ValueError ("Gateway supplied is not on the specified subnet.")
def reserved_ip_check(ip_string):
"""determine if IP address in RFC1918 or reserved"""
# IP details for invalid IP addresses
invalid_ip_details = {
"country": "INVALID",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "INVALID",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for MULTICAST IP addresses
multicast_ip_details = {
"country": "MULTICAST",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "MULTICAST",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for PRIVATE IP addresses
private_ip_details = {
"country": "PRIVATE",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "PRIVATE",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for RESERVED IP addresses
reserved_ip_details = {
"country": "RESERVED",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "RESERVED",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for NETMASK IP addresses
netmask_ip_details = {
"country": "NETMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "NETMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for HOSTMASK IP addresses
hostmask_ip_details = {
"country": "HOSTMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "HOSTMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for LOOPBACK IP addresses
loopback_ip_details = {
"country": "LOOPBACK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "LOOPBACK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# Check to see if IP matches a reserved category
try:
ip_address = IPAddress(ip_string)
except AddrFormatError:
return invalid_ip_details
if ip_address.is_multicast():
return multicast_ip_details
elif ip_address.is_private():
return private_ip_details
elif ip_address.is_reserved():
return reserved_ip_details
elif ip_address.is_netmask():
return netmask_ip_details
elif ip_address.is_hostmask():
return hostmask_ip_details
elif ip_address.is_loopback():
return loopback_ip_details
elif ip_address.is_unicast() and not ip_address.is_private():
# Boolean to be returned if IP is Public
ip_reserved = False
return ip_reserved
else:
return invalid_ip_details
def fifoReader(infile, q, exitSignal):
sleeptime = 0.5
maxSleeptime = 1.0
while True:
try:
if exitSignal.is_set(): break
line = infile.readline()
if not line:
time.sleep(1)
continue
if line == 'ENDOFFILE':
break
try:
spl = line.split()
timestamp, queriedName, clientID, ipv4 = spl
except:
continue
else:
if not '.' in queriedName:
continue
try:
addr = IPAddress(ipv4)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and not addr.is_private()
and not addr.is_reserved()
and not addr.is_loopback()):
try:
timestamp = int(timestamp)
except ValueError:
continue
else:
data = ((queriedName, clientID, [addr]), timestamp)
queued = False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime *= 0.5
elif q.qsize() >= q._maxsize:
sleeptime *= 2
if sleeptime > maxSleeptime:
sleeptime = maxSleeptime
else:
queued = True
except KeyboardInterrupt:
break
q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0):
if not infile and not interface:
# FIXME: write warning here
return
if infile:
pc = pcap.pcapObject()
try:
pc.open_offline(infile)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
if interface:
pc = pcap.pcapObject()
try:
#pc.open_live(interface, snaplen, promisc, read_timeout)
pc.open_live(interface, 1600, 0, 100)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
except Exception:
# most likely we got no permission to open the interface
sys.stderr.write('could not open interface. insufficient '
'permissions?\n')
q.put(None)
return
pc.setfilter('udp', 0, 0)
basets = 0
newMappings = dict()
while True:
if exitSignal.is_set():
break
try:
packet = pc.next()
if not packet:
if infile:
# end of file
break
elif interface:
# read timeout
continue
payload = packet[1]
timestamp = int(packet[2])
# make sure we are dealing with IP traffic
# ref: http://www.iana.org/assignments/ethernet-numbers
try:
eth = dpkt.ethernet.Ethernet(payload)
except:
continue
if eth.type != 2048: continue
# make sure we are dealing with UDP
# ref: http://www.iana.org/assignments/protocol-numbers/
try:
ip = eth.data
except:
continue
if ip.p != 17: continue
# filter on UDP assigned ports for DNS
# ref: http://www.iana.org/assignments/port-numbers
try:
udp = ip.data
except:
continue
if udp.sport != 53 and udp.dport != 53: continue
# make the dns object out of the udp data and check for it being a RR (answer)
# and for opcode QUERY (I know, counter-intuitive)
try:
dns = dpkt.dns.DNS(udp.data)
except:
continue
if dns.qr != dpkt.dns.DNS_R: continue
if dns.opcode != dpkt.dns.DNS_QUERY: continue
if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue
if len(dns.an) < 1: continue
if len(dns.qd) == 0: continue
aRecords = set()
queriedName = dns.qd[0].name
if not '.' in queriedName:
continue
#lastCname=queriedName
for answer in dns.an:
"""
FIXME: this doesn't work for multiple queries in one DNS packet
"""
#if answer.type == dpkt.dns.DNS_CNAME:
# lastCname=answer.cname
if answer.type == dpkt.dns.DNS_A:
ip = socket.inet_ntoa(answer.rdata)
try:
addr = IPAddress(ip)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and not addr.is_private()
and not addr.is_reserved()
and not addr.is_loopback()):
aRecords.add(addr)
if thrsh:
if (timestamp - basets) > thrsh:
basets = timestamp
newMappings.clear()
newIps = checkMapping(newMappings, queriedName, aRecords)
aRecords = newIps
if not aRecords:
continue
data = ((queriedName, ip.dst, aRecords), timestamp)
queued = False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime *= 0.5
elif q.qsize() >= q._maxsize:
sleeptime *= 2
if sleeptime > maxSleeptime:
sleeptime = maxSleeptime
else:
queued = True
except KeyboardInterrupt:
break
"""
send shutdown signal
"""
q.put(None)
def fifoReader(infile, q, exitSignal):
sleeptime=0.5
maxSleeptime=1.0
while True:
try:
if exitSignal.is_set(): break
line=infile.readline()
if not line:
time.sleep(1)
continue
if line=='ENDOFFILE':
break
try:
spl=line.split()
timestamp, queriedName, clientID, ipv4 = spl
except:
continue
else:
if not '.' in queriedName:
continue
try:
addr=IPAddress(ipv4)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and
not addr.is_private() and
not addr.is_reserved() and
not addr.is_loopback()):
try:
timestamp=int(timestamp)
except ValueError:
continue
else:
data = ((queriedName, clientID, [addr]),
timestamp)
queued=False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime*=0.5
elif q.qsize() >= q._maxsize:
sleeptime*=2
if sleeptime>maxSleeptime:
sleeptime=maxSleeptime
else:
queued=True
except KeyboardInterrupt:
break
q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0):
if not infile and not interface:
# FIXME: write warning here
return
if infile:
pc=pcap.pcapObject()
try:
pc.open_offline(infile)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
if interface:
pc=pcap.pcapObject()
try:
#pc.open_live(interface, snaplen, promisc, read_timeout)
pc.open_live(interface, 1600, 0, 100)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
except Exception:
# most likely we got no permission to open the interface
sys.stderr.write('could not open interface. insufficient '
'permissions?\n')
q.put(None)
return
pc.setfilter('udp', 0, 0)
basets=0
newMappings=dict()
while True:
if exitSignal.is_set():
break
try:
packet=pc.next()
if not packet:
if infile:
# end of file
break
elif interface:
# read timeout
continue
payload=packet[1]
timestamp=int(packet[2])
# make sure we are dealing with IP traffic
# ref: http://www.iana.org/assignments/ethernet-numbers
try: eth = dpkt.ethernet.Ethernet(payload)
except: continue
if eth.type != 2048: continue
# make sure we are dealing with UDP
# ref: http://www.iana.org/assignments/protocol-numbers/
try: ip = eth.data
except: continue
if ip.p != 17: continue
# filter on UDP assigned ports for DNS
# ref: http://www.iana.org/assignments/port-numbers
try: udp = ip.data
except: continue
if udp.sport != 53 and udp.dport != 53: continue
# make the dns object out of the udp data and check for it being a RR (answer)
# and for opcode QUERY (I know, counter-intuitive)
try: dns = dpkt.dns.DNS(udp.data)
except: continue
if dns.qr != dpkt.dns.DNS_R: continue
if dns.opcode != dpkt.dns.DNS_QUERY: continue
if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue
if len(dns.an) < 1: continue
if len(dns.qd) == 0: continue
aRecords=set()
queriedName=dns.qd[0].name
if not '.' in queriedName:
continue
#lastCname=queriedName
for answer in dns.an:
"""
FIXME: this doesn't work for multiple queries in one DNS packet
"""
#if answer.type == dpkt.dns.DNS_CNAME:
# lastCname=answer.cname
if answer.type == dpkt.dns.DNS_A:
ip=socket.inet_ntoa(answer.rdata)
try:
addr=IPAddress(ip)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and
not addr.is_private() and
not addr.is_reserved() and
not addr.is_loopback()):
aRecords.add(addr)
if thrsh:
if (timestamp-basets) > thrsh:
basets = timestamp
newMappings.clear()
newIps = checkMapping(newMappings, queriedName, aRecords)
aRecords=newIps
if not aRecords:
continue
data = ((queriedName, ip.dst, aRecords), timestamp)
queued=False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime*=0.5
elif q.qsize() >= q._maxsize:
sleeptime*=2
if sleeptime>maxSleeptime:
sleeptime=maxSleeptime
else:
queued=True
except KeyboardInterrupt:
break
"""
send shutdown signal
"""
q.put(None)
def inc_stats(self, packet: dict):
"""
Increments stats of the current Counters
:param packet: packet to process into the increment
"""
ip_origin = IPAddress(packet['origin']['ip'])
ip_destination = IPAddress(packet['destination']['ip'])
if ip_origin.is_reserved() or ip_destination.is_reserved():
return
# infer direction
# if both are private then is a private connection
# if just one is private or known the we infer the direction
internal_ips = ip_origin.is_private() + ip_destination.is_private() + (
str(ip_origin) in Counters.ipset) + (ip_destination
in Counters.ipset)
if internal_ips == 2:
# communication between two internal addresses
self.volume_internal = self.volume_internal + packet['length']
self.ip_internal.add(ip_origin)
self.ip_internal.add(ip_destination)
elif internal_ips == 1:
# comunication with and outside entity
# infer direction
if ip_origin.is_private() or (ip_origin in Counters.ipset):
# it's an upload
self.volume_upload = self.volume_upload + packet['length']
self.ip_external.add(ip_destination)
self.ip_internal.add(ip_origin)
elif ip_destination.is_private() or (str(ip_destination)
in Counters.ipset):
# it's a download
self.volume_download = self.volume_download + packet['length']
self.ip_internal.add(ip_destination)
self.ip_external.add(ip_origin)
else:
raise RuntimeError(
f"Internal IP is 1 but can't figure out which one: {str(ip_origin)} - {str(ip_destination)} {ip_origin.is_private()} {ip_destination.is_private()}"
)
if packet['protocol'] != "ICMP":
if packet['destination']['port'] > 1024:
self.port_high = self.port_high + 1
else:
self.port_low = self.port_low + 1
else:
raise RuntimeError(
f"Two public IPs and none is known. {str(ip_origin)} - {str(ip_destination)}"
)
if packet['protocol'] == "TCP":
self.tcp_syn = self.tcp_syn + packet['tcp_flags']['SYN']
self.tcp_fin = self.tcp_fin + packet['tcp_flags']['FIN']
self.tcp_rst = self.tcp_rst + packet['tcp_flags']['RST']
self.packet_count = self.packet_count + 1
# in bytes
if int(packet['length']) <= 64:
self.less_64b = self.less_64b + 1
