def validate(url, schemes=None, tlds=None, private=None, local=None, credentials=None): ''' Validate and normalize an URL :param str url: The URL to validate and normalize :return str: The normalized URL :raises ValidationError: when URL does not validate ''' url = url.strip() private = config_for(private, 'URLS_ALLOW_PRIVATE') local = config_for(local, 'URLS_ALLOW_LOCAL') credentials = config_for(credentials, 'URLS_ALLOW_CREDENTIALS') schemes = config_for(schemes, 'URLS_ALLOWED_SCHEMES') tlds = config_for(tlds, 'URLS_ALLOWED_TLDS') match = URL_REGEX.match(url) if not match: error(url) scheme = (match.group('scheme') or '').lower() if scheme and scheme not in schemes: error(url, 'Invalid scheme {0}'.format(scheme)) if not credentials and match.group('credentials'): error(url, 'Credentials in URL are not allowed') tld = match.group('tld') if tld and tld not in tlds and idna(tld) not in tlds: error(url, 'Invalid TLD {0}'.format(tld)) ip = match.group('ipv6') or match.group('ipv4') if ip: try: ip = IPAddress(ip) except AddrFormatError: error(url) if ip.is_multicast(): error(url, '{0} is a multicast IP'.format(ip)) elif not ip.is_loopback() and ip.is_hostmask() or ip.is_netmask(): error(url, '{0} is a mask IP'.format(ip)) if not local: if ip and ip.is_loopback() or match.group('localhost'): error(url, 'is a local URL') if not private and ip and ip.is_private(): error(url, 'is a private URL') return url
def auto_select_target(target, output=None): """Auto selection logic""" print("Target: %s" % target) try: inp = IPAddress(target); if inp.is_private() or inp.is_loopback(): print("Internal IP Detected : Skipping") sys.exit() else: print("Looks like an IP, running ipOsint...\n") ipOsint.run(target, output) except SystemExit: print("exiting") except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print("Looks like an EMAIL, running emailOsint...\n") emailOsint.run(target, output) elif get_tld(target, fix_protocol=True, fail_silently=True) is not None: print("Looks like a DOMAIN, running domainOsint...\n") domainOsint.run(target, output) else: print("Nothing Matched assuming username, running usernameOsint...\n") usernameOsint.run(target, output) except: print("Unknown Error Occured")
def address(self, value): ip = IPAddress(self.ipformat(value)) if ip.is_loopback(): raise ValidationError("You cannot use a loopback address") if ip.is_multicast(): raise ValidationError("You cannot use a multicast address") self._address = value
def auto_select_target(target, output=None): """Auto selection logic""" print "Target: %s" % target try: inp = IPAddress(target) if inp.is_private() or inp.is_loopback(): print "Internal IP Detected : Skipping" sys.exit() else: print "Looks like an IP, running ipOsint...\n" ipOsint.run(target, output) except SystemExit: print "exiting" except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print "Looks like an EMAIL, running emailOsint...\n" return emailOsint.run(target, output) elif get_tld(target, fix_protocol=True, fail_silently=True) is not None: print "Looks like a DOMAIN, running domainOsint...\n" # domainOsint.run(target, output) return { 'error': True, 'message': 'We do not do domain searches yet, sorry!' } else: print "Nothing Matched assuming username, running usernameOsint...\n" # usernameOsint.run(target, output) return { 'error': True, 'message': 'We do not do username searches yet, sorry!' } except: print "Unknown Error Occured"
def _validate_network_information(self): all_errs = [] if self.subnet_mask_size > self.MAX_SUBNET_MASK_SIZE: all_errs.append( "Subnet size too small. Subnet mask should <= {}, current value: {}" .format(self.MAX_SUBNET_MASK_SIZE, self.subnet_mask_size)) cidr_base_validator = re.compile(self.VPC_CIDR_BASE_REGEX) if not cidr_base_validator.match(self.vpc_cidr_base): all_errs.append( "Invalid VPC CIDR base {}. VPC CIDR base should match regex {}" .format(self.vpc_cidr_base, self.VPC_CIDR_BASE_REGEX)) if self.trusted_cidrs: try: for cidr in self.trusted_cidrs: [ip, mask] = cidr.split("/") if ip == "0.0.0.0": if mask != "0": all_errs.append( "Trusting traffic from everywhere should specify \"0.0.0.0/0\" as trusted CIDR." ) else: if not 0 < int(mask) <= 32: all_errs.append( "Subnet mask {} should be greater than 0 but less than 32." .format(mask)) ipaddr = IPAddress(ip) if ipaddr.is_netmask(): all_errs.append( "Trusted CIDR {} should not be a net mask". format(ip)) if ipaddr.is_hostmask(): all_errs.append( "Trusted CIDR {} should not be a host mask". format(ip)) if ipaddr.is_reserved(): all_errs.append( "Trusted CIDR {} should not be in reserved range" .format(ip)) if ipaddr.is_loopback(): all_errs.append( "Trusted CIDR {} should not be a loop back address" .format(ip)) # Currently we don't support private VPC if ipaddr.is_private(): all_errs.append( "Trusted CIDR {} should not be a private address" .format(ip)) except ValueError as ve: all_errs.append( "Cannot parse trusted CIDRs ({}). Err: {}".format( self.trusted_cidrs, ve)) else: all_errs.append( "Please provide trusted CIDRs through --trusted-cidrs flag") return all_errs
def is_valid_address(address): ''' Validate whether the address provided is routable unicast address ''' addr = IPAddress(address) if addr.is_loopback() or addr.is_reserved() or addr.is_private()\ or addr.is_link_local() or addr.is_multicast(): return False return True
def get_ip_address(self, test_address=None): """ try to get global IP address from interface information. if failed, just return '127.0.0.1' :param str test_address: ip address str if test to check global ip. normally None. :return: global ip address if successed, or '127.0.0.1' """ for iface_name in netifaces.interfaces(): iface_data = netifaces.ifaddresses(iface_name) logging.debug('Interface: %s' % (iface_name, )) ifaces = [] if netifaces.AF_INET in iface_data: ifaces += iface_data[netifaces.AF_INET] if netifaces.AF_INET6 in iface_data: ifaces += iface_data[netifaces.AF_INET6] for iface in ifaces: ip = iface['addr'] ip = re.sub(r'\%.+$', '', ip) if test_address is not None: ip = test_address addr = IPAddress(ip) if not addr.is_loopback() and addr.is_unicast() and\ not addr.is_private(): logging.debug('global ip %s', addr) return ip logging.debug('no global ip') return '127.0.0.1'
def auto_select_target(target, output=None): """Auto selection logic""" print "Target: %s" % target try: inp=IPAddress(target); if inp.is_private() or inp.is_loopback(): print "Internal IP Detected : Skipping" sys.exit() else: print "Looks like an IP, running ipOsint...\n" ipOsint.run(target, output) except SystemExit: print "exiting" except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print "Looks like an EMAIL, running emailOsint...\n" emailOsint.run(target, output) elif get_tld(target, fix_protocol=True,fail_silently=True) is not None: print "Looks like a DOMAIN, running domainOsint...\n" domainOsint.run(target, output) else: print "Nothing Matched assuming username, running usernameOsint...\n" usernameOsint.run(target, output) except: print "Unknown Error Occured"
def call(self, url, context): if self.url_can_resolve(url): try: ip = yield self.resolver.get_host_by_name(url.domain) ip = IPAddress(ip) except Exception: # context["event"].target.respond( # u'[Error] Failed to handle URL: {}'.format( # url.to_string() # ) # ) self.plugin.logger.exception("Error while checking DNS") returnValue(STOP_HANDLING) return if ip.is_loopback() or ip.is_private() or ip.is_link_local() \ or ip.is_multicast(): self.plugin.logger.warn( "Prevented connection to private/internal address" ) returnValue(STOP_HANDLING) return headers = {} if url.domain in context["config"]["spoofing"]: user_agent = context["config"]["spoofing"][url.domain] if user_agent: headers["User-Agent"] = user_agent else: headers["User-Agent"] = context["config"].get( "default_user_agent", "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 " "Firefox/36.0" ) domain_langs = context.get("config") \ .get("accept_language", {}) \ .get("domains", {}) if url.domain in domain_langs: headers["Accept-Language"] = domain_langs.get(url.domain) else: headers["Accept-Language"] = context.get("config") \ .get("accept_language", {}) \ .get("default", "en") session = self.get_session(url, context) session.get(unicode(url), headers=headers, stream=True, background_callback=self.background_callback) \ .addCallback(self.callback, url, context, session) \ .addErrback(self.errback, url, context, session) returnValue(STOP_HANDLING)
def auto_select_target(self, target, output=None): """Auto selection logic""" result = {} print("Target: %s" % target) try: inp = IPAddress(target) if inp.is_private() or inp.is_loopback(): print("Internal IP Detected : Skipping") sys.exit() else: print("Looks like an IP, running ipOsint...\n") data = ipOsint.run(target, output) #print("*************************************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result except SystemExit: print("exiting") except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print("Looks like an EMAIL, running emailOsint...\n") data = emailOsint.run(target, output) #print("*************************************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result elif get_tld(target, fix_protocol=True, fail_silently=True) is not None: print("Looks like a DOMAIN, running domainOsint...\n") data = domainOsint.run(target, output) #print("************************domainOsint*************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result else: print( "Nothing Matched assuming username, running usernameOsint...\n" ) data = usernameOsint.run(target, output) #print("*************************************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result except: print("Unknown Error Occured") return result, 201
def local_addresses(): result = [] sp = subprocess.run(['/sbin/ifconfig'], capture_output=True, text=True) for line in sp.stdout.split('\n'): if line.find('\tinet') > -1: try: ip = IPAddress(line.split()[1].split('%')[0]) except AddrFormatError: ip = None if ip and not ip.is_loopback() and not ip.is_link_local(): result.append(ip) return result
def _getListenAddresses(self, port): """Return list of tuple (address, port) for the addresses the worker is listening on.""" addresses = get_all_interface_source_addresses() if addresses: return set((addr, port) for addr in addresses) # There are no non-loopback addresses, so return loopback # address as a fallback. loopback_addresses = set() for addr in get_all_interface_addresses(): ipaddr = IPAddress(addr) if ipaddr.is_link_local(): continue # Don't advertise link-local addresses. if ipaddr.is_loopback(): loopback_addresses.add((addr, port)) return loopback_addresses
def is_loopback_address(hostname): """Determine if the given hostname appears to be a loopback address. :param hostname: either a hostname or an IP address. No resolution is done, but 'localhost' is considered to be loopback. :type hostname: str :return: True if the address is a loopback address. """ try: ip = IPAddress(hostname) except AddrFormatError: return hostname.lower() in {"localhost", "localhost."} return ip.is_loopback() or (ip.is_ipv4_mapped() and ip.ipv4().is_loopback())
def metadata_file_writer(q, filename): PLUGINS = load_metadata_plugins() metadata_file = open(filename, "w") ip_adresses = [0] while True: connection = q.get() ip = connection[6] ipAddress = IPAddress(hex_to_ip(ip)) if ip not in ip_adresses and not ipAddress.is_private() and not ipAddress.is_loopback(): ip_adresses.append(ip) for p in PLUGINS: p.set_connection(connection) res = p.run() if len(res): metadata_file.write("%s, %s,%s\n" % (p.name, hex_to_ip(ip), res)) metadata_file.flush() q.task_done()
def check_ip_address(ipaddr): ip_attributes = [] ipaddress = IPAddress(ipaddr) if ipaddress.is_private(): ip_attributes.append("IP Address is Private") else: ip_attributes.append("IP Address is public") if ipaddress.is_unicast(): ip_attributes.append("IP Address is unicast") elif ipaddress.is_multicast(): ip_attributes.append("IP Address is multicast") if ipaddress.is_loopback(): ip_attributes.append("IP Address is loopback") return "\n".join(ip_attributes)
def _trusted_cidr_validator(input): from netaddr import IPAddress ret = [] for cidr in input.split(" "): cidr = cidr.strip() if not cidr: # skip whitespace continue ip, mask = cidr.split("/") if int(mask) < 0 or int(mask) > 32: raise ValueError( "CIDR {} is not valid as mask {} is not in range [0-32]". format(cidr, mask)) if ip != "0.0.0.0" or mask != '0': ipaddr = IPAddress(ip) if ipaddr.is_netmask(): raise ValueError( "Trusted CIDR {} should not be a net mask".format(ip)) if ipaddr.is_hostmask(): raise ValueError( "Trusted CIDR {} should not be a host mask".format(ip)) if ipaddr.is_reserved(): raise ValueError( "Trusted CIDR {} should not be in reserved range". format(ip)) if ipaddr.is_loopback(): raise ValueError( "Trusted CIDR {} should not be a loop back address". format(ip)) # Currently we don't support private VPC if ipaddr.is_private(): raise ValueError( "Trusted CIDR {} should not be a private address". format(ip)) ret.append(cidr) return ret
def validate_ip_address (address, netmask, gateway = None): """ Validate that the provided IP settings are valid. :param address: The IP address that will be assigned to the interface. :param netmask: The subnet mask for the interface. :param gateway: The default gateway to use for the system. """ ip = IPAddress (address) if (not ip.is_unicast () or ip.is_loopback () or ip.is_reserved ()): raise ValueError ("IP address is not a valid host address.") if (not IPAddress (netmask).is_netmask ()): raise ValueError ("Subnet mask is not valid.") subnet = IPNetwork (address, netmask) if gateway: route = IPNetwork (gateway, netmask) if (subnet != route): raise ValueError ("Gateway supplied is not on the specified subnet.")
def ip_check_routable(item): ip_addr = IPAddress(item) # This prevents netaddr allowing shortened ip addresses if not str(ip_addr) == item: raise AddrFormatError("IP Malformed {}".format(item)) # Check for reserved IP addresses if any([ ip_addr.is_multicast(), ip_addr.is_private(), ip_addr.is_loopback(), ip_addr.is_link_local(), ip_addr.is_reserved() ]): raise AddrFormatError("IP is reserved {}".format(item)) # Check to see if IP is IPv4 # elif ip_addr.version is not 4: # raise AddrFormatError("IP is not IPv4") return True
def main(argv): output = None desc = """ ____/ /____ _ / /_ ____ _ _____ ____ / /____ (_)/ /_ / __ // __ `// __// __ `// ___// __ \ / // __ \ / // __/ / /_/ // /_/ // /_ / /_/ /(__ )/ /_/ // // /_/ // // /_ \__,_/ \__,_/ \__/ \__,_//____// .___//_/ \____//_/ \__/ /_/ Open Source Assistant for #OSINT www.datasploit.info """ epilog = """ Connect at Social Media: @datasploit """ # Set all parser arguments here. parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(desc), epilog=epilog) parser.add_argument("-i", "--input", help="Provide Input", dest='target', required=True) parser.add_argument("-a", "--active", help="Run Active Scan attacks", dest='active', action="store_false") parser.add_argument( "-q", "--quiet", help="Run scans in automated manner accepting default answers", dest='quiet', action="store_false") parser.add_argument("-o", "--output", help="Provide Destination Directory", dest='output') # check and ensure the config file is present otherwise create one. required for all further operations ds_dir = os.path.dirname(os.path.realpath(__file__)) config_file_path = os.path.join(ds_dir, "config.py") config_sample_path = os.path.join(ds_dir, "config_sample.py") print os.path.exists(config_file_path) if not os.path.exists(config_file_path): print "[+] Looks like a new setup, setting up the config file." shutil.copyfile(config_sample_path, config_file_path) print "[+] A config file is added please follow guide at https://datasploit.github.io/datasploit/apiGeneration/ to fill API Keys for better results" # We can think about quiting at this point. # if no argument is provided print help and quit if len(argv) == 0: parser.print_help() sys.exit() # parse arguments in case they are provided. x = parser.parse_args() active = x.active quiet = x.quiet user_input = x.target output = x.output # Banner print print textwrap.dedent(desc) # Auto selection logic try: print "User Input: %s" % user_input try: inp = IPAddress(user_input) if inp.is_private() or inp.is_loopback(): print "Internal IP Detected : Skipping" sys.exit() else: print "Looks like an IP, running ipOsint...\n" ipOsint.run(user_input, output) except SystemExit: print "exiting" except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', user_input): print "Looks like an EMAIL, running emailOsint...\n" emailOsint.run(user_input, output) elif get_tld(user_input, fix_protocol=True, fail_silently=True) is not None: print "Looks like a DOMAIN, running domainOsint...\n" domainOsint.run(user_input, output) else: print "Nothing Matched assuming username, running usernameOsint...\n" usernameOsint.run(user_input, output) except: print "Unknown Error Occured" except KeyboardInterrupt: print "Ctrl+C called Quiting"
def callback(self, result, url, context, session): response = result[0] content = result[1] self.plugin.logger.trace( "Headers: {0}", list(response.headers) ) self.plugin.logger.trace("HTTP code: {0}", response.status_code) new_url = urlparse.urlparse(response.url) if self.url_can_resolve(url): try: ip = yield self.resolver.get_host_by_name(new_url.hostname) ip = IPAddress(ip) except Exception: # context["event"].target.respond( # u'[Error] Failed to handle URL: {}'.format( # url.to_string() # ) # ) self.plugin.logger.exception("Error while checking DNS") returnValue(STOP_HANDLING) return if ip.is_loopback() or ip.is_private() or ip.is_link_local() \ or ip.is_multicast(): self.plugin.logger.warn( "Prevented connection to private/internal address" ) returnValue(STOP_HANDLING) return if content is None: self.plugin.logger.debug("No content returned") return soup = BeautifulSoup(content) if soup.title and soup.title.text: title = soup.title.text.strip() title = re.sub("[\n\s]+", " ", title) title = to_unicode(title) title_limit = self.urls_plugin.config.get("max_title_length", 150) if len(title) > title_limit: title = title[:title_limit - 15] + u"... (truncated)" if response.status_code == requests.codes.ok: context["event"].target.respond( u'"{0}" at {1}'.format( title, new_url.hostname ) ) else: context["event"].target.respond( u'[HTTP {0}] "{1}" at {2}'.format( response.status_code, title, new_url.hostname ) ) else: if response.status_code != requests.codes.ok: context["event"].target.respond( u'HTTP Error {0}: "{1}" at {2}'.format( response.status_code, STATUS_CODES.get(response.status_code, "Unknown"), new_url.hostname ) ) else: self.plugin.logger.debug("No title") self.save_session(session)
def check_ip(dns_result: str) -> bool: addr = IPAddress(dns_result) if ((not addr.is_unicast()) or addr.is_private() or addr.is_loopback() or addr.is_link_local()): return False return True
def reserved_ip_check(ip_string): """determine if IP address in RFC1918 or reserved""" # IP details for invalid IP addresses invalid_ip_details = { "country": "INVALID", "location": RESERVED_IP_COORDINATES, "subdivisions": "INVALID", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for MULTICAST IP addresses multicast_ip_details = { "country": "MULTICAST", "location": RESERVED_IP_COORDINATES, "subdivisions": "MULTICAST", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for PRIVATE IP addresses private_ip_details = { "country": "PRIVATE", "location": RESERVED_IP_COORDINATES, "subdivisions": "PRIVATE", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for RESERVED IP addresses reserved_ip_details = { "country": "RESERVED", "location": RESERVED_IP_COORDINATES, "subdivisions": "RESERVED", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for NETMASK IP addresses netmask_ip_details = { "country": "NETMASK", "location": RESERVED_IP_COORDINATES, "subdivisions": "NETMASK", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for HOSTMASK IP addresses hostmask_ip_details = { "country": "HOSTMASK", "location": RESERVED_IP_COORDINATES, "subdivisions": "HOSTMASK", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for LOOPBACK IP addresses loopback_ip_details = { "country": "LOOPBACK", "location": RESERVED_IP_COORDINATES, "subdivisions": "LOOPBACK", "dch_company": "", "asn_number": "", "asn_name": "" } # Check to see if IP matches a reserved category try: ip_address = IPAddress(ip_string) except AddrFormatError: return invalid_ip_details if ip_address.is_multicast(): return multicast_ip_details elif ip_address.is_private(): return private_ip_details elif ip_address.is_reserved(): return reserved_ip_details elif ip_address.is_netmask(): return netmask_ip_details elif ip_address.is_hostmask(): return hostmask_ip_details elif ip_address.is_loopback(): return loopback_ip_details elif ip_address.is_unicast() and not ip_address.is_private(): # Boolean to be returned if IP is Public ip_reserved = False return ip_reserved else: return invalid_ip_details
def fifoReader(infile, q, exitSignal): sleeptime = 0.5 maxSleeptime = 1.0 while True: try: if exitSignal.is_set(): break line = infile.readline() if not line: time.sleep(1) continue if line == 'ENDOFFILE': break try: spl = line.split() timestamp, queriedName, clientID, ipv4 = spl except: continue else: if not '.' in queriedName: continue try: addr = IPAddress(ipv4) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): try: timestamp = int(timestamp) except ValueError: continue else: data = ((queriedName, clientID, [addr]), timestamp) queued = False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime *= 0.5 elif q.qsize() >= q._maxsize: sleeptime *= 2 if sleeptime > maxSleeptime: sleeptime = maxSleeptime else: queued = True except KeyboardInterrupt: break q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0): if not infile and not interface: # FIXME: write warning here return if infile: pc = pcap.pcapObject() try: pc.open_offline(infile) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass if interface: pc = pcap.pcapObject() try: #pc.open_live(interface, snaplen, promisc, read_timeout) pc.open_live(interface, 1600, 0, 100) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass except Exception: # most likely we got no permission to open the interface sys.stderr.write('could not open interface. insufficient ' 'permissions?\n') q.put(None) return pc.setfilter('udp', 0, 0) basets = 0 newMappings = dict() while True: if exitSignal.is_set(): break try: packet = pc.next() if not packet: if infile: # end of file break elif interface: # read timeout continue payload = packet[1] timestamp = int(packet[2]) # make sure we are dealing with IP traffic # ref: http://www.iana.org/assignments/ethernet-numbers try: eth = dpkt.ethernet.Ethernet(payload) except: continue if eth.type != 2048: continue # make sure we are dealing with UDP # ref: http://www.iana.org/assignments/protocol-numbers/ try: ip = eth.data except: continue if ip.p != 17: continue # filter on UDP assigned ports for DNS # ref: http://www.iana.org/assignments/port-numbers try: udp = ip.data except: continue if udp.sport != 53 and udp.dport != 53: continue # make the dns object out of the udp data and check for it being a RR (answer) # and for opcode QUERY (I know, counter-intuitive) try: dns = dpkt.dns.DNS(udp.data) except: continue if dns.qr != dpkt.dns.DNS_R: continue if dns.opcode != dpkt.dns.DNS_QUERY: continue if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue if len(dns.an) < 1: continue if len(dns.qd) == 0: continue aRecords = set() queriedName = dns.qd[0].name if not '.' in queriedName: continue #lastCname=queriedName for answer in dns.an: """ FIXME: this doesn't work for multiple queries in one DNS packet """ #if answer.type == dpkt.dns.DNS_CNAME: # lastCname=answer.cname if answer.type == dpkt.dns.DNS_A: ip = socket.inet_ntoa(answer.rdata) try: addr = IPAddress(ip) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): aRecords.add(addr) if thrsh: if (timestamp - basets) > thrsh: basets = timestamp newMappings.clear() newIps = checkMapping(newMappings, queriedName, aRecords) aRecords = newIps if not aRecords: continue data = ((queriedName, ip.dst, aRecords), timestamp) queued = False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime *= 0.5 elif q.qsize() >= q._maxsize: sleeptime *= 2 if sleeptime > maxSleeptime: sleeptime = maxSleeptime else: queued = True except KeyboardInterrupt: break """ send shutdown signal """ q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0): if not infile and not interface: # FIXME: write warning here return if infile: pc=pcap.pcapObject() try: pc.open_offline(infile) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass if interface: pc=pcap.pcapObject() try: #pc.open_live(interface, snaplen, promisc, read_timeout) pc.open_live(interface, 1600, 0, 100) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass except Exception: # most likely we got no permission to open the interface sys.stderr.write('could not open interface. insufficient ' 'permissions?\n') q.put(None) return pc.setfilter('udp', 0, 0) basets=0 newMappings=dict() while True: if exitSignal.is_set(): break try: packet=pc.next() if not packet: if infile: # end of file break elif interface: # read timeout continue payload=packet[1] timestamp=int(packet[2]) # make sure we are dealing with IP traffic # ref: http://www.iana.org/assignments/ethernet-numbers try: eth = dpkt.ethernet.Ethernet(payload) except: continue if eth.type != 2048: continue # make sure we are dealing with UDP # ref: http://www.iana.org/assignments/protocol-numbers/ try: ip = eth.data except: continue if ip.p != 17: continue # filter on UDP assigned ports for DNS # ref: http://www.iana.org/assignments/port-numbers try: udp = ip.data except: continue if udp.sport != 53 and udp.dport != 53: continue # make the dns object out of the udp data and check for it being a RR (answer) # and for opcode QUERY (I know, counter-intuitive) try: dns = dpkt.dns.DNS(udp.data) except: continue if dns.qr != dpkt.dns.DNS_R: continue if dns.opcode != dpkt.dns.DNS_QUERY: continue if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue if len(dns.an) < 1: continue if len(dns.qd) == 0: continue aRecords=set() queriedName=dns.qd[0].name if not '.' in queriedName: continue #lastCname=queriedName for answer in dns.an: """ FIXME: this doesn't work for multiple queries in one DNS packet """ #if answer.type == dpkt.dns.DNS_CNAME: # lastCname=answer.cname if answer.type == dpkt.dns.DNS_A: ip=socket.inet_ntoa(answer.rdata) try: addr=IPAddress(ip) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): aRecords.add(addr) if thrsh: if (timestamp-basets) > thrsh: basets = timestamp newMappings.clear() newIps = checkMapping(newMappings, queriedName, aRecords) aRecords=newIps if not aRecords: continue data = ((queriedName, ip.dst, aRecords), timestamp) queued=False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime*=0.5 elif q.qsize() >= q._maxsize: sleeptime*=2 if sleeptime>maxSleeptime: sleeptime=maxSleeptime else: queued=True except KeyboardInterrupt: break """ send shutdown signal """ q.put(None)
def fifoReader(infile, q, exitSignal): sleeptime=0.5 maxSleeptime=1.0 while True: try: if exitSignal.is_set(): break line=infile.readline() if not line: time.sleep(1) continue if line=='ENDOFFILE': break try: spl=line.split() timestamp, queriedName, clientID, ipv4 = spl except: continue else: if not '.' in queriedName: continue try: addr=IPAddress(ipv4) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): try: timestamp=int(timestamp) except ValueError: continue else: data = ((queriedName, clientID, [addr]), timestamp) queued=False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime*=0.5 elif q.qsize() >= q._maxsize: sleeptime*=2 if sleeptime>maxSleeptime: sleeptime=maxSleeptime else: queued=True except KeyboardInterrupt: break q.put(None)