def validate(url,
schemes=None,
tlds=None,
private=None,
local=None,
credentials=None):
'''
Validate and normalize an URL
:param str url: The URL to validate and normalize
:return str: The normalized URL
:raises ValidationError: when URL does not validate
'''
url = url.strip()
private = config_for(private, 'URLS_ALLOW_PRIVATE')
local = config_for(local, 'URLS_ALLOW_LOCAL')
credentials = config_for(credentials, 'URLS_ALLOW_CREDENTIALS')
schemes = config_for(schemes, 'URLS_ALLOWED_SCHEMES')
tlds = config_for(tlds, 'URLS_ALLOWED_TLDS')
match = URL_REGEX.match(url)
if not match:
error(url)
scheme = (match.group('scheme') or '').lower()
if scheme and scheme not in schemes:
error(url, 'Invalid scheme {0}'.format(scheme))
if not credentials and match.group('credentials'):
error(url, 'Credentials in URL are not allowed')
tld = match.group('tld')
if tld and tld not in tlds and idna(tld) not in tlds:
error(url, 'Invalid TLD {0}'.format(tld))
ip = match.group('ipv6') or match.group('ipv4')
if ip:
try:
ip = IPAddress(ip)
except AddrFormatError:
error(url)
if ip.is_multicast():
error(url, '{0} is a multicast IP'.format(ip))
elif not ip.is_loopback() and ip.is_hostmask() or ip.is_netmask():
error(url, '{0} is a mask IP'.format(ip))
if not local:
if ip and ip.is_loopback() or match.group('localhost'):
error(url, 'is a local URL')
if not private and ip and ip.is_private():
error(url, 'is a private URL')
return url
def auto_select_target(target, output=None):
"""Auto selection logic"""
print("Target: %s" % target)
try:
inp = IPAddress(target);
if inp.is_private() or inp.is_loopback():
print("Internal IP Detected : Skipping")
sys.exit()
else:
print("Looks like an IP, running ipOsint...\n")
ipOsint.run(target, output)
except SystemExit:
print("exiting")
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print("Looks like an EMAIL, running emailOsint...\n")
emailOsint.run(target, output)
elif get_tld(target, fix_protocol=True, fail_silently=True) is not None:
print("Looks like a DOMAIN, running domainOsint...\n")
domainOsint.run(target, output)
else:
print("Nothing Matched assuming username, running usernameOsint...\n")
usernameOsint.run(target, output)
except:
print("Unknown Error Occured")
def address(self, value):
ip = IPAddress(self.ipformat(value))
if ip.is_loopback():
raise ValidationError("You cannot use a loopback address")
if ip.is_multicast():
raise ValidationError("You cannot use a multicast address")
self._address = value
def address(self, value):
ip = IPAddress(self.ipformat(value))
if ip.is_loopback():
raise ValidationError("You cannot use a loopback address")
if ip.is_multicast():
raise ValidationError("You cannot use a multicast address")
self._address = value
def auto_select_target(target, output=None):
"""Auto selection logic"""
print "Target: %s" % target
try:
inp = IPAddress(target)
if inp.is_private() or inp.is_loopback():
print "Internal IP Detected : Skipping"
sys.exit()
else:
print "Looks like an IP, running ipOsint...\n"
ipOsint.run(target, output)
except SystemExit:
print "exiting"
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print "Looks like an EMAIL, running emailOsint...\n"
return emailOsint.run(target, output)
elif get_tld(target, fix_protocol=True,
fail_silently=True) is not None:
print "Looks like a DOMAIN, running domainOsint...\n"
# domainOsint.run(target, output)
return {
'error': True,
'message': 'We do not do domain searches yet, sorry!'
}
else:
print "Nothing Matched assuming username, running usernameOsint...\n"
# usernameOsint.run(target, output)
return {
'error': True,
'message': 'We do not do username searches yet, sorry!'
}
except:
print "Unknown Error Occured"
def _validate_network_information(self):
all_errs = []
if self.subnet_mask_size > self.MAX_SUBNET_MASK_SIZE:
all_errs.append(
"Subnet size too small. Subnet mask should <= {}, current value: {}"
.format(self.MAX_SUBNET_MASK_SIZE, self.subnet_mask_size))
cidr_base_validator = re.compile(self.VPC_CIDR_BASE_REGEX)
if not cidr_base_validator.match(self.vpc_cidr_base):
all_errs.append(
"Invalid VPC CIDR base {}. VPC CIDR base should match regex {}"
.format(self.vpc_cidr_base, self.VPC_CIDR_BASE_REGEX))
if self.trusted_cidrs:
try:
for cidr in self.trusted_cidrs:
[ip, mask] = cidr.split("/")
if ip == "0.0.0.0":
if mask != "0":
all_errs.append(
"Trusting traffic from everywhere should specify \"0.0.0.0/0\" as trusted CIDR."
)
else:
if not 0 < int(mask) <= 32:
all_errs.append(
"Subnet mask {} should be greater than 0 but less than 32."
.format(mask))
ipaddr = IPAddress(ip)
if ipaddr.is_netmask():
all_errs.append(
"Trusted CIDR {} should not be a net mask".
format(ip))
if ipaddr.is_hostmask():
all_errs.append(
"Trusted CIDR {} should not be a host mask".
format(ip))
if ipaddr.is_reserved():
all_errs.append(
"Trusted CIDR {} should not be in reserved range"
.format(ip))
if ipaddr.is_loopback():
all_errs.append(
"Trusted CIDR {} should not be a loop back address"
.format(ip))
# Currently we don't support private VPC
if ipaddr.is_private():
all_errs.append(
"Trusted CIDR {} should not be a private address"
.format(ip))
except ValueError as ve:
all_errs.append(
"Cannot parse trusted CIDRs ({}). Err: {}".format(
self.trusted_cidrs, ve))
else:
all_errs.append(
"Please provide trusted CIDRs through --trusted-cidrs flag")
return all_errs
def is_valid_address(address):
''' Validate whether the address provided is routable unicast address '''
addr = IPAddress(address)
if addr.is_loopback() or addr.is_reserved() or addr.is_private()\
or addr.is_link_local() or addr.is_multicast():
return False
return True
def get_ip_address(self, test_address=None):
"""
try to get global IP address from interface information.
if failed, just return '127.0.0.1'
:param str test_address: ip address str if test to check global ip.
normally None.
:return: global ip address if successed, or '127.0.0.1'
"""
for iface_name in netifaces.interfaces():
iface_data = netifaces.ifaddresses(iface_name)
logging.debug('Interface: %s' % (iface_name, ))
ifaces = []
if netifaces.AF_INET in iface_data:
ifaces += iface_data[netifaces.AF_INET]
if netifaces.AF_INET6 in iface_data:
ifaces += iface_data[netifaces.AF_INET6]
for iface in ifaces:
ip = iface['addr']
ip = re.sub(r'\%.+$', '', ip)
if test_address is not None:
ip = test_address
addr = IPAddress(ip)
if not addr.is_loopback() and addr.is_unicast() and\
not addr.is_private():
logging.debug('global ip %s', addr)
return ip
logging.debug('no global ip')
return '127.0.0.1'
def auto_select_target(target, output=None):
"""Auto selection logic"""
print "Target: %s" % target
try:
inp=IPAddress(target);
if inp.is_private() or inp.is_loopback():
print "Internal IP Detected : Skipping"
sys.exit()
else:
print "Looks like an IP, running ipOsint...\n"
ipOsint.run(target, output)
except SystemExit:
print "exiting"
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print "Looks like an EMAIL, running emailOsint...\n"
emailOsint.run(target, output)
elif get_tld(target, fix_protocol=True,fail_silently=True) is not None:
print "Looks like a DOMAIN, running domainOsint...\n"
domainOsint.run(target, output)
else:
print "Nothing Matched assuming username, running usernameOsint...\n"
usernameOsint.run(target, output)
except:
print "Unknown Error Occured"
def call(self, url, context):
if self.url_can_resolve(url):
try:
ip = yield self.resolver.get_host_by_name(url.domain)
ip = IPAddress(ip)
except Exception:
# context["event"].target.respond(
# u'[Error] Failed to handle URL: {}'.format(
# url.to_string()
# )
# )
self.plugin.logger.exception("Error while checking DNS")
returnValue(STOP_HANDLING)
return
if ip.is_loopback() or ip.is_private() or ip.is_link_local() \
or ip.is_multicast():
self.plugin.logger.warn(
"Prevented connection to private/internal address"
)
returnValue(STOP_HANDLING)
return
headers = {}
if url.domain in context["config"]["spoofing"]:
user_agent = context["config"]["spoofing"][url.domain]
if user_agent:
headers["User-Agent"] = user_agent
else:
headers["User-Agent"] = context["config"].get(
"default_user_agent",
"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 "
"Firefox/36.0"
)
domain_langs = context.get("config") \
.get("accept_language", {}) \
.get("domains", {})
if url.domain in domain_langs:
headers["Accept-Language"] = domain_langs.get(url.domain)
else:
headers["Accept-Language"] = context.get("config") \
.get("accept_language", {}) \
.get("default", "en")
session = self.get_session(url, context)
session.get(unicode(url), headers=headers, stream=True,
background_callback=self.background_callback) \
.addCallback(self.callback, url, context, session) \
.addErrback(self.errback, url, context, session)
returnValue(STOP_HANDLING)
def auto_select_target(self, target, output=None):
"""Auto selection logic"""
result = {}
print("Target: %s" % target)
try:
inp = IPAddress(target)
if inp.is_private() or inp.is_loopback():
print("Internal IP Detected : Skipping")
sys.exit()
else:
print("Looks like an IP, running ipOsint...\n")
data = ipOsint.run(target, output)
#print("*************************************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
except SystemExit:
print("exiting")
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print("Looks like an EMAIL, running emailOsint...\n")
data = emailOsint.run(target, output)
#print("*************************************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
elif get_tld(target, fix_protocol=True,
fail_silently=True) is not None:
print("Looks like a DOMAIN, running domainOsint...\n")
data = domainOsint.run(target, output)
#print("************************domainOsint*************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
else:
print(
"Nothing Matched assuming username, running usernameOsint...\n"
)
data = usernameOsint.run(target, output)
#print("*************************************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
except:
print("Unknown Error Occured")
return result, 201
def local_addresses():
result = []
sp = subprocess.run(['/sbin/ifconfig'], capture_output=True, text=True)
for line in sp.stdout.split('\n'):
if line.find('\tinet') > -1:
try:
ip = IPAddress(line.split()[1].split('%')[0])
except AddrFormatError:
ip = None
if ip and not ip.is_loopback() and not ip.is_link_local():
result.append(ip)
return result
def _getListenAddresses(self, port):
"""Return list of tuple (address, port) for the addresses the worker
is listening on."""
addresses = get_all_interface_source_addresses()
if addresses:
return set((addr, port) for addr in addresses)
# There are no non-loopback addresses, so return loopback
# address as a fallback.
loopback_addresses = set()
for addr in get_all_interface_addresses():
ipaddr = IPAddress(addr)
if ipaddr.is_link_local():
continue # Don't advertise link-local addresses.
if ipaddr.is_loopback():
loopback_addresses.add((addr, port))
return loopback_addresses
def is_loopback_address(hostname):
"""Determine if the given hostname appears to be a loopback address.
:param hostname: either a hostname or an IP address. No resolution is
done, but 'localhost' is considered to be loopback.
:type hostname: str
:return: True if the address is a loopback address.
"""
try:
ip = IPAddress(hostname)
except AddrFormatError:
return hostname.lower() in {"localhost", "localhost."}
return ip.is_loopback() or (ip.is_ipv4_mapped()
and ip.ipv4().is_loopback())
def metadata_file_writer(q, filename):
PLUGINS = load_metadata_plugins()
metadata_file = open(filename, "w")
ip_adresses = [0]
while True:
connection = q.get()
ip = connection[6]
ipAddress = IPAddress(hex_to_ip(ip))
if ip not in ip_adresses and not ipAddress.is_private() and not ipAddress.is_loopback():
ip_adresses.append(ip)
for p in PLUGINS:
p.set_connection(connection)
res = p.run()
if len(res):
metadata_file.write("%s, %s,%s\n" % (p.name, hex_to_ip(ip), res))
metadata_file.flush()
q.task_done()
def check_ip_address(ipaddr):
ip_attributes = []
ipaddress = IPAddress(ipaddr)
if ipaddress.is_private():
ip_attributes.append("IP Address is Private")
else:
ip_attributes.append("IP Address is public")
if ipaddress.is_unicast():
ip_attributes.append("IP Address is unicast")
elif ipaddress.is_multicast():
ip_attributes.append("IP Address is multicast")
if ipaddress.is_loopback():
ip_attributes.append("IP Address is loopback")
return "\n".join(ip_attributes)
def _trusted_cidr_validator(input):
from netaddr import IPAddress
ret = []
for cidr in input.split(" "):
cidr = cidr.strip()
if not cidr:
# skip whitespace
continue
ip, mask = cidr.split("/")
if int(mask) < 0 or int(mask) > 32:
raise ValueError(
"CIDR {} is not valid as mask {} is not in range [0-32]".
format(cidr, mask))
if ip != "0.0.0.0" or mask != '0':
ipaddr = IPAddress(ip)
if ipaddr.is_netmask():
raise ValueError(
"Trusted CIDR {} should not be a net mask".format(ip))
if ipaddr.is_hostmask():
raise ValueError(
"Trusted CIDR {} should not be a host mask".format(ip))
if ipaddr.is_reserved():
raise ValueError(
"Trusted CIDR {} should not be in reserved range".
format(ip))
if ipaddr.is_loopback():
raise ValueError(
"Trusted CIDR {} should not be a loop back address".
format(ip))
# Currently we don't support private VPC
if ipaddr.is_private():
raise ValueError(
"Trusted CIDR {} should not be a private address".
format(ip))
ret.append(cidr)
return ret
def validate_ip_address (address, netmask, gateway = None):
"""
Validate that the provided IP settings are valid.
:param address: The IP address that will be assigned to the interface.
:param netmask: The subnet mask for the interface.
:param gateway: The default gateway to use for the system.
"""
ip = IPAddress (address)
if (not ip.is_unicast () or ip.is_loopback () or ip.is_reserved ()):
raise ValueError ("IP address is not a valid host address.")
if (not IPAddress (netmask).is_netmask ()):
raise ValueError ("Subnet mask is not valid.")
subnet = IPNetwork (address, netmask)
if gateway:
route = IPNetwork (gateway, netmask)
if (subnet != route):
raise ValueError ("Gateway supplied is not on the specified subnet.")
def ip_check_routable(item):
ip_addr = IPAddress(item)
# This prevents netaddr allowing shortened ip addresses
if not str(ip_addr) == item:
raise AddrFormatError("IP Malformed {}".format(item))
# Check for reserved IP addresses
if any([
ip_addr.is_multicast(),
ip_addr.is_private(),
ip_addr.is_loopback(),
ip_addr.is_link_local(),
ip_addr.is_reserved()
]):
raise AddrFormatError("IP is reserved {}".format(item))
# Check to see if IP is IPv4
# elif ip_addr.version is not 4:
# raise AddrFormatError("IP is not IPv4")
return True
def main(argv):
output = None
desc = """
____/ /____ _ / /_ ____ _ _____ ____ / /____ (_)/ /_
/ __ // __ `// __// __ `// ___// __ \ / // __ \ / // __/
/ /_/ // /_/ // /_ / /_/ /(__ )/ /_/ // // /_/ // // /_
\__,_/ \__,_/ \__/ \__,_//____// .___//_/ \____//_/ \__/
/_/
Open Source Assistant for #OSINT
www.datasploit.info
"""
epilog = """ Connect at Social Media: @datasploit
"""
# Set all parser arguments here.
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent(desc),
epilog=epilog)
parser.add_argument("-i",
"--input",
help="Provide Input",
dest='target',
required=True)
parser.add_argument("-a",
"--active",
help="Run Active Scan attacks",
dest='active',
action="store_false")
parser.add_argument(
"-q",
"--quiet",
help="Run scans in automated manner accepting default answers",
dest='quiet',
action="store_false")
parser.add_argument("-o",
"--output",
help="Provide Destination Directory",
dest='output')
# check and ensure the config file is present otherwise create one. required for all further operations
ds_dir = os.path.dirname(os.path.realpath(__file__))
config_file_path = os.path.join(ds_dir, "config.py")
config_sample_path = os.path.join(ds_dir, "config_sample.py")
print os.path.exists(config_file_path)
if not os.path.exists(config_file_path):
print "[+] Looks like a new setup, setting up the config file."
shutil.copyfile(config_sample_path, config_file_path)
print "[+] A config file is added please follow guide at https://datasploit.github.io/datasploit/apiGeneration/ to fill API Keys for better results"
# We can think about quiting at this point.
# if no argument is provided print help and quit
if len(argv) == 0:
parser.print_help()
sys.exit()
# parse arguments in case they are provided.
x = parser.parse_args()
active = x.active
quiet = x.quiet
user_input = x.target
output = x.output
# Banner print
print textwrap.dedent(desc)
# Auto selection logic
try:
print "User Input: %s" % user_input
try:
inp = IPAddress(user_input)
if inp.is_private() or inp.is_loopback():
print "Internal IP Detected : Skipping"
sys.exit()
else:
print "Looks like an IP, running ipOsint...\n"
ipOsint.run(user_input, output)
except SystemExit:
print "exiting"
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', user_input):
print "Looks like an EMAIL, running emailOsint...\n"
emailOsint.run(user_input, output)
elif get_tld(user_input, fix_protocol=True,
fail_silently=True) is not None:
print "Looks like a DOMAIN, running domainOsint...\n"
domainOsint.run(user_input, output)
else:
print "Nothing Matched assuming username, running usernameOsint...\n"
usernameOsint.run(user_input, output)
except:
print "Unknown Error Occured"
except KeyboardInterrupt:
print "Ctrl+C called Quiting"
def callback(self, result, url, context, session):
response = result[0]
content = result[1]
self.plugin.logger.trace(
"Headers: {0}", list(response.headers)
)
self.plugin.logger.trace("HTTP code: {0}", response.status_code)
new_url = urlparse.urlparse(response.url)
if self.url_can_resolve(url):
try:
ip = yield self.resolver.get_host_by_name(new_url.hostname)
ip = IPAddress(ip)
except Exception:
# context["event"].target.respond(
# u'[Error] Failed to handle URL: {}'.format(
# url.to_string()
# )
# )
self.plugin.logger.exception("Error while checking DNS")
returnValue(STOP_HANDLING)
return
if ip.is_loopback() or ip.is_private() or ip.is_link_local() \
or ip.is_multicast():
self.plugin.logger.warn(
"Prevented connection to private/internal address"
)
returnValue(STOP_HANDLING)
return
if content is None:
self.plugin.logger.debug("No content returned")
return
soup = BeautifulSoup(content)
if soup.title and soup.title.text:
title = soup.title.text.strip()
title = re.sub("[\n\s]+", " ", title)
title = to_unicode(title)
title_limit = self.urls_plugin.config.get("max_title_length", 150)
if len(title) > title_limit:
title = title[:title_limit - 15] + u"... (truncated)"
if response.status_code == requests.codes.ok:
context["event"].target.respond(
u'"{0}" at {1}'.format(
title, new_url.hostname
)
)
else:
context["event"].target.respond(
u'[HTTP {0}] "{1}" at {2}'.format(
response.status_code,
title, new_url.hostname
)
)
else:
if response.status_code != requests.codes.ok:
context["event"].target.respond(
u'HTTP Error {0}: "{1}" at {2}'.format(
response.status_code,
STATUS_CODES.get(response.status_code, "Unknown"),
new_url.hostname
)
)
else:
self.plugin.logger.debug("No title")
self.save_session(session)
def check_ip(dns_result: str) -> bool:
addr = IPAddress(dns_result)
if ((not addr.is_unicast()) or addr.is_private()
or addr.is_loopback() or addr.is_link_local()):
return False
return True
def reserved_ip_check(ip_string):
"""determine if IP address in RFC1918 or reserved"""
# IP details for invalid IP addresses
invalid_ip_details = {
"country": "INVALID",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "INVALID",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for MULTICAST IP addresses
multicast_ip_details = {
"country": "MULTICAST",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "MULTICAST",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for PRIVATE IP addresses
private_ip_details = {
"country": "PRIVATE",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "PRIVATE",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for RESERVED IP addresses
reserved_ip_details = {
"country": "RESERVED",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "RESERVED",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for NETMASK IP addresses
netmask_ip_details = {
"country": "NETMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "NETMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for HOSTMASK IP addresses
hostmask_ip_details = {
"country": "HOSTMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "HOSTMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for LOOPBACK IP addresses
loopback_ip_details = {
"country": "LOOPBACK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "LOOPBACK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# Check to see if IP matches a reserved category
try:
ip_address = IPAddress(ip_string)
except AddrFormatError:
return invalid_ip_details
if ip_address.is_multicast():
return multicast_ip_details
elif ip_address.is_private():
return private_ip_details
elif ip_address.is_reserved():
return reserved_ip_details
elif ip_address.is_netmask():
return netmask_ip_details
elif ip_address.is_hostmask():
return hostmask_ip_details
elif ip_address.is_loopback():
return loopback_ip_details
elif ip_address.is_unicast() and not ip_address.is_private():
# Boolean to be returned if IP is Public
ip_reserved = False
return ip_reserved
else:
return invalid_ip_details
def fifoReader(infile, q, exitSignal):
sleeptime = 0.5
maxSleeptime = 1.0
while True:
try:
if exitSignal.is_set(): break
line = infile.readline()
if not line:
time.sleep(1)
continue
if line == 'ENDOFFILE':
break
try:
spl = line.split()
timestamp, queriedName, clientID, ipv4 = spl
except:
continue
else:
if not '.' in queriedName:
continue
try:
addr = IPAddress(ipv4)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and not addr.is_private()
and not addr.is_reserved()
and not addr.is_loopback()):
try:
timestamp = int(timestamp)
except ValueError:
continue
else:
data = ((queriedName, clientID, [addr]), timestamp)
queued = False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime *= 0.5
elif q.qsize() >= q._maxsize:
sleeptime *= 2
if sleeptime > maxSleeptime:
sleeptime = maxSleeptime
else:
queued = True
except KeyboardInterrupt:
break
q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0):
if not infile and not interface:
# FIXME: write warning here
return
if infile:
pc = pcap.pcapObject()
try:
pc.open_offline(infile)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
if interface:
pc = pcap.pcapObject()
try:
#pc.open_live(interface, snaplen, promisc, read_timeout)
pc.open_live(interface, 1600, 0, 100)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
except Exception:
# most likely we got no permission to open the interface
sys.stderr.write('could not open interface. insufficient '
'permissions?\n')
q.put(None)
return
pc.setfilter('udp', 0, 0)
basets = 0
newMappings = dict()
while True:
if exitSignal.is_set():
break
try:
packet = pc.next()
if not packet:
if infile:
# end of file
break
elif interface:
# read timeout
continue
payload = packet[1]
timestamp = int(packet[2])
# make sure we are dealing with IP traffic
# ref: http://www.iana.org/assignments/ethernet-numbers
try:
eth = dpkt.ethernet.Ethernet(payload)
except:
continue
if eth.type != 2048: continue
# make sure we are dealing with UDP
# ref: http://www.iana.org/assignments/protocol-numbers/
try:
ip = eth.data
except:
continue
if ip.p != 17: continue
# filter on UDP assigned ports for DNS
# ref: http://www.iana.org/assignments/port-numbers
try:
udp = ip.data
except:
continue
if udp.sport != 53 and udp.dport != 53: continue
# make the dns object out of the udp data and check for it being a RR (answer)
# and for opcode QUERY (I know, counter-intuitive)
try:
dns = dpkt.dns.DNS(udp.data)
except:
continue
if dns.qr != dpkt.dns.DNS_R: continue
if dns.opcode != dpkt.dns.DNS_QUERY: continue
if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue
if len(dns.an) < 1: continue
if len(dns.qd) == 0: continue
aRecords = set()
queriedName = dns.qd[0].name
if not '.' in queriedName:
continue
#lastCname=queriedName
for answer in dns.an:
"""
FIXME: this doesn't work for multiple queries in one DNS packet
"""
#if answer.type == dpkt.dns.DNS_CNAME:
# lastCname=answer.cname
if answer.type == dpkt.dns.DNS_A:
ip = socket.inet_ntoa(answer.rdata)
try:
addr = IPAddress(ip)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and not addr.is_private()
and not addr.is_reserved()
and not addr.is_loopback()):
aRecords.add(addr)
if thrsh:
if (timestamp - basets) > thrsh:
basets = timestamp
newMappings.clear()
newIps = checkMapping(newMappings, queriedName, aRecords)
aRecords = newIps
if not aRecords:
continue
data = ((queriedName, ip.dst, aRecords), timestamp)
queued = False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime *= 0.5
elif q.qsize() >= q._maxsize:
sleeptime *= 2
if sleeptime > maxSleeptime:
sleeptime = maxSleeptime
else:
queued = True
except KeyboardInterrupt:
break
"""
send shutdown signal
"""
q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0):
if not infile and not interface:
# FIXME: write warning here
return
if infile:
pc=pcap.pcapObject()
try:
pc.open_offline(infile)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
if interface:
pc=pcap.pcapObject()
try:
#pc.open_live(interface, snaplen, promisc, read_timeout)
pc.open_live(interface, 1600, 0, 100)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
except Exception:
# most likely we got no permission to open the interface
sys.stderr.write('could not open interface. insufficient '
'permissions?\n')
q.put(None)
return
pc.setfilter('udp', 0, 0)
basets=0
newMappings=dict()
while True:
if exitSignal.is_set():
break
try:
packet=pc.next()
if not packet:
if infile:
# end of file
break
elif interface:
# read timeout
continue
payload=packet[1]
timestamp=int(packet[2])
# make sure we are dealing with IP traffic
# ref: http://www.iana.org/assignments/ethernet-numbers
try: eth = dpkt.ethernet.Ethernet(payload)
except: continue
if eth.type != 2048: continue
# make sure we are dealing with UDP
# ref: http://www.iana.org/assignments/protocol-numbers/
try: ip = eth.data
except: continue
if ip.p != 17: continue
# filter on UDP assigned ports for DNS
# ref: http://www.iana.org/assignments/port-numbers
try: udp = ip.data
except: continue
if udp.sport != 53 and udp.dport != 53: continue
# make the dns object out of the udp data and check for it being a RR (answer)
# and for opcode QUERY (I know, counter-intuitive)
try: dns = dpkt.dns.DNS(udp.data)
except: continue
if dns.qr != dpkt.dns.DNS_R: continue
if dns.opcode != dpkt.dns.DNS_QUERY: continue
if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue
if len(dns.an) < 1: continue
if len(dns.qd) == 0: continue
aRecords=set()
queriedName=dns.qd[0].name
if not '.' in queriedName:
continue
#lastCname=queriedName
for answer in dns.an:
"""
FIXME: this doesn't work for multiple queries in one DNS packet
"""
#if answer.type == dpkt.dns.DNS_CNAME:
# lastCname=answer.cname
if answer.type == dpkt.dns.DNS_A:
ip=socket.inet_ntoa(answer.rdata)
try:
addr=IPAddress(ip)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and
not addr.is_private() and
not addr.is_reserved() and
not addr.is_loopback()):
aRecords.add(addr)
if thrsh:
if (timestamp-basets) > thrsh:
basets = timestamp
newMappings.clear()
newIps = checkMapping(newMappings, queriedName, aRecords)
aRecords=newIps
if not aRecords:
continue
data = ((queriedName, ip.dst, aRecords), timestamp)
queued=False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime*=0.5
elif q.qsize() >= q._maxsize:
sleeptime*=2
if sleeptime>maxSleeptime:
sleeptime=maxSleeptime
else:
queued=True
except KeyboardInterrupt:
break
"""
send shutdown signal
"""
q.put(None)
def fifoReader(infile, q, exitSignal):
sleeptime=0.5
maxSleeptime=1.0
while True:
try:
if exitSignal.is_set(): break
line=infile.readline()
if not line:
time.sleep(1)
continue
if line=='ENDOFFILE':
break
try:
spl=line.split()
timestamp, queriedName, clientID, ipv4 = spl
except:
continue
else:
if not '.' in queriedName:
continue
try:
addr=IPAddress(ipv4)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and
not addr.is_private() and
not addr.is_reserved() and
not addr.is_loopback()):
try:
timestamp=int(timestamp)
except ValueError:
continue
else:
data = ((queriedName, clientID, [addr]),
timestamp)
queued=False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime*=0.5
elif q.qsize() >= q._maxsize:
sleeptime*=2
if sleeptime>maxSleeptime:
sleeptime=maxSleeptime
else:
queued=True
except KeyboardInterrupt:
break
q.put(None)
