def get_from_ip(cls, ip, port, type, hostname=None, timestamp=datetime.utcnow()): try: ip_addr = IPAddress(ip) except netaddr.core.AddrFormatError: logger.debug(f"Failed to parse {ip}") return None geo = None if not ip_addr.is_private() and not ip_addr.is_reserved(): geo = cls.get_geo(ip) if not hostname: hostname = cls.get_rdns(ip) return NetworkEntity(ip, port, type, geo, hostname=hostname, timestamp=timestamp, is_enriched=True)
def auto_select_target(target, output=None): """Auto selection logic""" print "Target: %s" % target try: inp=IPAddress(target); if inp.is_private() or inp.is_loopback(): print "Internal IP Detected : Skipping" sys.exit() else: print "Looks like an IP, running ipOsint...\n" ipOsint.run(target, output) except SystemExit: print "exiting" except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print "Looks like an EMAIL, running emailOsint...\n" emailOsint.run(target, output) elif get_tld(target, fix_protocol=True,fail_silently=True) is not None: print "Looks like a DOMAIN, running domainOsint...\n" domainOsint.run(target, output) else: print "Nothing Matched assuming username, running usernameOsint...\n" usernameOsint.run(target, output) except: print "Unknown Error Occured"
def get_priv_info(d_iface_to_addr=None): s_net_id = None s_priv_ip = None s_priv_interface = None if d_iface_to_addr is None: d_iface_to_addr = get_iface_to_addr() networks = search('net', 'name', '*') for s_iface, d_addr in d_iface_to_addr.items(): if s_iface.startswith('lo'): continue if netifaces.AF_INET not in d_addr: continue ips = d_addr[netifaces.AF_INET] for ip in ips: o_ip = IPAddress(str(ip['addr'])) if not o_ip.is_private(): continue if ip['addr'] == '127.0.0.1': continue for net in networks: if (('netmask' in net) and (o_ip in IPNetwork(net['netmask']))): s_priv_ip = str(ip['addr']) s_priv_interface = s_iface s_net_id = net['name'] break return (s_priv_ip, s_priv_interface, s_net_id)
def auto_select_target(target, output=None): """Auto selection logic""" print "Target: %s" % target try: inp = IPAddress(target) if inp.is_private() or inp.is_loopback(): print "Internal IP Detected : Skipping" sys.exit() else: print "Looks like an IP, running ipOsint...\n" ipOsint.run(target, output) except SystemExit: print "exiting" except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print "Looks like an EMAIL, running emailOsint...\n" return emailOsint.run(target, output) elif get_tld(target, fix_protocol=True, fail_silently=True) is not None: print "Looks like a DOMAIN, running domainOsint...\n" # domainOsint.run(target, output) return { 'error': True, 'message': 'We do not do domain searches yet, sorry!' } else: print "Nothing Matched assuming username, running usernameOsint...\n" # usernameOsint.run(target, output) return { 'error': True, 'message': 'We do not do username searches yet, sorry!' } except: print "Unknown Error Occured"
def auto_select_target(target, output=None): """Auto selection logic""" print("Target: %s" % target) try: inp = IPAddress(target); if inp.is_private() or inp.is_loopback(): print("Internal IP Detected : Skipping") sys.exit() else: print("Looks like an IP, running ipOsint...\n") ipOsint.run(target, output) except SystemExit: print("exiting") except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print("Looks like an EMAIL, running emailOsint...\n") emailOsint.run(target, output) elif get_tld(target, fix_protocol=True, fail_silently=True) is not None: print("Looks like a DOMAIN, running domainOsint...\n") domainOsint.run(target, output) else: print("Nothing Matched assuming username, running usernameOsint...\n") usernameOsint.run(target, output) except: print("Unknown Error Occured")
def to_server_dict(server): public_ips = [ip["addr"] for ip in server.addresses["public"]] private_ips = [ip["addr"] for ip in server.addresses["private"]] # Pick out first public IPv4 and IPv6 address public_ipv4 = None public_ipv6 = None for ip in public_ips: try: ip_obj = IPAddress(ip) except Exception: continue if not ip_obj.is_private(): if ip_obj.version == 4: public_ipv4 = ip elif ip_obj.version == 6: public_ipv6 = ip result = { "id": server.id, "name": server.name, "status": server.status, "image_id": server.image["id"], "flavor_id": server.flavor["id"], "public_ips": public_ips, "private_ips": private_ips, "public_ipv4": public_ipv4, "public_ipv6": public_ipv6, "key_name": server.key_name, "metadata": server.metadata, } return result
def get_ip_address(self, test_address=None): """ try to get global IP address from interface information. if failed, just return '127.0.0.1' :param str test_address: ip address str if test to check global ip. normally None. :return: global ip address if successed, or '127.0.0.1' """ for iface_name in netifaces.interfaces(): iface_data = netifaces.ifaddresses(iface_name) logging.debug('Interface: %s' % (iface_name, )) ifaces = [] if netifaces.AF_INET in iface_data: ifaces += iface_data[netifaces.AF_INET] if netifaces.AF_INET6 in iface_data: ifaces += iface_data[netifaces.AF_INET6] for iface in ifaces: ip = iface['addr'] ip = re.sub(r'\%.+$', '', ip) if test_address is not None: ip = test_address addr = IPAddress(ip) if not addr.is_loopback() and addr.is_unicast() and\ not addr.is_private(): logging.debug('global ip %s', addr) return ip logging.debug('no global ip') return '127.0.0.1'
def is_valid_address(address): ''' Validate whether the address provided is routable unicast address ''' addr = IPAddress(address) if addr.is_loopback() or addr.is_reserved() or addr.is_private()\ or addr.is_link_local() or addr.is_multicast(): return False return True
def _validate_network_information(self): all_errs = [] if self.subnet_mask_size > self.MAX_SUBNET_MASK_SIZE: all_errs.append( "Subnet size too small. Subnet mask should <= {}, current value: {}" .format(self.MAX_SUBNET_MASK_SIZE, self.subnet_mask_size)) cidr_base_validator = re.compile(self.VPC_CIDR_BASE_REGEX) if not cidr_base_validator.match(self.vpc_cidr_base): all_errs.append( "Invalid VPC CIDR base {}. VPC CIDR base should match regex {}" .format(self.vpc_cidr_base, self.VPC_CIDR_BASE_REGEX)) if self.trusted_cidrs: try: for cidr in self.trusted_cidrs: [ip, mask] = cidr.split("/") if ip == "0.0.0.0": if mask != "0": all_errs.append( "Trusting traffic from everywhere should specify \"0.0.0.0/0\" as trusted CIDR." ) else: if not 0 < int(mask) <= 32: all_errs.append( "Subnet mask {} should be greater than 0 but less than 32." .format(mask)) ipaddr = IPAddress(ip) if ipaddr.is_netmask(): all_errs.append( "Trusted CIDR {} should not be a net mask". format(ip)) if ipaddr.is_hostmask(): all_errs.append( "Trusted CIDR {} should not be a host mask". format(ip)) if ipaddr.is_reserved(): all_errs.append( "Trusted CIDR {} should not be in reserved range" .format(ip)) if ipaddr.is_loopback(): all_errs.append( "Trusted CIDR {} should not be a loop back address" .format(ip)) # Currently we don't support private VPC if ipaddr.is_private(): all_errs.append( "Trusted CIDR {} should not be a private address" .format(ip)) except ValueError as ve: all_errs.append( "Cannot parse trusted CIDRs ({}). Err: {}".format( self.trusted_cidrs, ve)) else: all_errs.append( "Please provide trusted CIDRs through --trusted-cidrs flag") return all_errs
def _is_local_network(self, address): url = urlparse.urlparse(address) try: net = IPAddress(url) return net.is_private() except (AddrFormatError, TypeError): return False
def valid_ip(self, ip_name): """ this method checks the validation of provided ip i.e whether, 1. ip contains 4 octets or not 2. each octet of ip in range of 256 or not 3. provided ip is public or not :param ip_name:provided ip. ex: 8.8.8.8 :return: either true or false """ try: ip_name = IPAddress(ip_name) except AddrFormatError: print(f"invalid IPaddress.provide data as ips") return False else: if ip_name.version in [4, 6]: if ip_name.is_private(): print( f'provided {ip_name} is private. don\'t use private ips for reputation...' ) return False else: return True else: return True
def getNetwork(self, ip_address): '''return the site and network names for an IP address we assume the site is the first level subgroup (group = CLIENT.SiteName) for public adresses, we try a reverse DNS and a WHOIS lookups''' ip = IPAddress(ip_address) for _, row in self.networks.iterrows(): if ip in IPNetwork(row['cidr']): return (row['group'].split('.')[1], row['name']) # if the IP is not private and hasn't been found in network hierarchy # try a reverse DNS lookup if ip.is_private(): return ('', '') else: try: res = gethostbyaddr(ip_address) return ('', res[0]) except herror: # Reverse DNS failed, let's do a WHOIS lookup try: res = IPWhois(ip).lookup_whois() return (res['asn_country_code'], res['asn_description'][:30]) except: return ('', '')
def is_public_ip(ip): ip_obj = IPAddress(ip) if not ip_obj.is_private() and not ip_obj.is_reserved(): return True return False
def to_server_dict(server): public_ips = [ip['addr'] for ip in server.addresses['public']] private_ips = [ip['addr'] for ip in server.addresses['private']] # Pick out first public IPv4 and IPv6 address public_ipv4 = None public_ipv6 = None for ip in public_ips: try: ip_obj = IPAddress(ip) except Exception: continue if not ip_obj.is_private(): if ip_obj.version == 4: public_ipv4 = ip elif ip_obj.version == 6: public_ipv6 = ip result = { 'id': server.id, 'name': server.name, 'status': server.status, 'image_id': server.image['id'], 'flavor_id': server.flavor['id'], 'public_ips': public_ips, 'private_ips': private_ips, 'public_ipv4': public_ipv4, 'public_ipv6': public_ipv6, 'key_name': server.key_name, 'metadata': server.metadata } return result
def is_public_ip(ip_address): try: ip_addr = IPAddress(str(ip_address)) # We also need to filter multicast ip which is a private ip. For example, # multicast ip '239.192.0.1', is a private ip. return ip_addr.is_unicast() and not ip_addr.is_private() except: # pylint: disable=bare-except return False
def call(self, url, context): if self.url_can_resolve(url): try: ip = yield self.resolver.get_host_by_name(url.domain) ip = IPAddress(ip) except Exception: # context["event"].target.respond( # u'[Error] Failed to handle URL: {}'.format( # url.to_string() # ) # ) self.plugin.logger.exception("Error while checking DNS") returnValue(STOP_HANDLING) return if ip.is_loopback() or ip.is_private() or ip.is_link_local() \ or ip.is_multicast(): self.plugin.logger.warn( "Prevented connection to private/internal address" ) returnValue(STOP_HANDLING) return headers = {} if url.domain in context["config"]["spoofing"]: user_agent = context["config"]["spoofing"][url.domain] if user_agent: headers["User-Agent"] = user_agent else: headers["User-Agent"] = context["config"].get( "default_user_agent", "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 " "Firefox/36.0" ) domain_langs = context.get("config") \ .get("accept_language", {}) \ .get("domains", {}) if url.domain in domain_langs: headers["Accept-Language"] = domain_langs.get(url.domain) else: headers["Accept-Language"] = context.get("config") \ .get("accept_language", {}) \ .get("default", "en") session = self.get_session(url, context) session.get(unicode(url), headers=headers, stream=True, background_callback=self.background_callback) \ .addCallback(self.callback, url, context, session) \ .addErrback(self.errback, url, context, session) returnValue(STOP_HANDLING)
def auto_select_target(self, target, output=None): """Auto selection logic""" result = {} print("Target: %s" % target) try: inp = IPAddress(target) if inp.is_private() or inp.is_loopback(): print("Internal IP Detected : Skipping") sys.exit() else: print("Looks like an IP, running ipOsint...\n") data = ipOsint.run(target, output) #print("*************************************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result except SystemExit: print("exiting") except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', target): print("Looks like an EMAIL, running emailOsint...\n") data = emailOsint.run(target, output) #print("*************************************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result elif get_tld(target, fix_protocol=True, fail_silently=True) is not None: print("Looks like a DOMAIN, running domainOsint...\n") data = domainOsint.run(target, output) #print("************************domainOsint*************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result else: print( "Nothing Matched assuming username, running usernameOsint...\n" ) data = usernameOsint.run(target, output) #print("*************************************************************************") #print(data) #print("*************************************************************************") result = {"target": target, "data": data} results.append(result) return result except: print("Unknown Error Occured") return result, 201
def validate(url, schemes=None, tlds=None, private=None, local=None, credentials=None): ''' Validate and normalize an URL :param str url: The URL to validate and normalize :return str: The normalized URL :raises ValidationError: when URL does not validate ''' url = url.strip() private = config_for(private, 'URLS_ALLOW_PRIVATE') local = config_for(local, 'URLS_ALLOW_LOCAL') credentials = config_for(credentials, 'URLS_ALLOW_CREDENTIALS') schemes = config_for(schemes, 'URLS_ALLOWED_SCHEMES') tlds = config_for(tlds, 'URLS_ALLOWED_TLDS') match = URL_REGEX.match(url) if not match: error(url) scheme = (match.group('scheme') or '').lower() if scheme and scheme not in schemes: error(url, 'Invalid scheme {0}'.format(scheme)) if not credentials and match.group('credentials'): error(url, 'Credentials in URL are not allowed') tld = match.group('tld') if tld and tld not in tlds and idna(tld) not in tlds: error(url, 'Invalid TLD {0}'.format(tld)) ip = match.group('ipv6') or match.group('ipv4') if ip: try: ip = IPAddress(ip) except AddrFormatError: error(url) if ip.is_multicast(): error(url, '{0} is a multicast IP'.format(ip)) elif not ip.is_loopback() and ip.is_hostmask() or ip.is_netmask(): error(url, '{0} is a mask IP'.format(ip)) if not local: if ip and ip.is_loopback() or match.group('localhost'): error(url, 'is a local URL') if not private and ip and ip.is_private(): error(url, 'is a private URL') return url
def metadata_file_writer(q, filename): PLUGINS = load_metadata_plugins() metadata_file = open(filename, "w") ip_adresses = [0] while True: connection = q.get() ip = connection[6] ipAddress = IPAddress(hex_to_ip(ip)) if ip not in ip_adresses and not ipAddress.is_private() and not ipAddress.is_loopback(): ip_adresses.append(ip) for p in PLUGINS: p.set_connection(connection) res = p.run() if len(res): metadata_file.write("%s, %s,%s\n" % (p.name, hex_to_ip(ip), res)) metadata_file.flush() q.task_done()
def check_ip_address(ipaddr): ip_attributes = [] ipaddress = IPAddress(ipaddr) if ipaddress.is_private(): ip_attributes.append("IP Address is Private") else: ip_attributes.append("IP Address is public") if ipaddress.is_unicast(): ip_attributes.append("IP Address is unicast") elif ipaddress.is_multicast(): ip_attributes.append("IP Address is multicast") if ipaddress.is_loopback(): ip_attributes.append("IP Address is loopback") return "\n".join(ip_attributes)
def _trusted_cidr_validator(input): from netaddr import IPAddress ret = [] for cidr in input.split(" "): cidr = cidr.strip() if not cidr: # skip whitespace continue ip, mask = cidr.split("/") if int(mask) < 0 or int(mask) > 32: raise ValueError( "CIDR {} is not valid as mask {} is not in range [0-32]". format(cidr, mask)) if ip != "0.0.0.0" or mask != '0': ipaddr = IPAddress(ip) if ipaddr.is_netmask(): raise ValueError( "Trusted CIDR {} should not be a net mask".format(ip)) if ipaddr.is_hostmask(): raise ValueError( "Trusted CIDR {} should not be a host mask".format(ip)) if ipaddr.is_reserved(): raise ValueError( "Trusted CIDR {} should not be in reserved range". format(ip)) if ipaddr.is_loopback(): raise ValueError( "Trusted CIDR {} should not be a loop back address". format(ip)) # Currently we don't support private VPC if ipaddr.is_private(): raise ValueError( "Trusted CIDR {} should not be a private address". format(ip)) ret.append(cidr) return ret
def get_from_hostname(cls, hostname, type, timestamp=datetime.utcnow()): ip = cls.get_ip(hostname) geo = None try: ip_addr = IPAddress(ip) if ip_addr and not ip_addr.is_private() \ and not ip_addr.is_reserved(): geo = cls.get_geo(ip) except netaddr.core.AddrFormatError: logger.debug(f"Failed to parse {ip}") return NetworkEntity(ip, 0, type, geo, hostname=hostname, timestamp=timestamp, is_enriched=True)
def ip_check_routable(item): ip_addr = IPAddress(item) # This prevents netaddr allowing shortened ip addresses if not str(ip_addr) == item: raise AddrFormatError("IP Malformed {}".format(item)) # Check for reserved IP addresses if any([ ip_addr.is_multicast(), ip_addr.is_private(), ip_addr.is_loopback(), ip_addr.is_link_local(), ip_addr.is_reserved() ]): raise AddrFormatError("IP is reserved {}".format(item)) # Check to see if IP is IPv4 # elif ip_addr.version is not 4: # raise AddrFormatError("IP is not IPv4") return True
def callback(self, result, url, context, session): response = result[0] content = result[1] self.plugin.logger.trace( "Headers: {0}", list(response.headers) ) self.plugin.logger.trace("HTTP code: {0}", response.status_code) new_url = urlparse.urlparse(response.url) if self.url_can_resolve(url): try: ip = yield self.resolver.get_host_by_name(new_url.hostname) ip = IPAddress(ip) except Exception: # context["event"].target.respond( # u'[Error] Failed to handle URL: {}'.format( # url.to_string() # ) # ) self.plugin.logger.exception("Error while checking DNS") returnValue(STOP_HANDLING) return if ip.is_loopback() or ip.is_private() or ip.is_link_local() \ or ip.is_multicast(): self.plugin.logger.warn( "Prevented connection to private/internal address" ) returnValue(STOP_HANDLING) return if content is None: self.plugin.logger.debug("No content returned") return soup = BeautifulSoup(content) if soup.title and soup.title.text: title = soup.title.text.strip() title = re.sub("[\n\s]+", " ", title) title = to_unicode(title) title_limit = self.urls_plugin.config.get("max_title_length", 150) if len(title) > title_limit: title = title[:title_limit - 15] + u"... (truncated)" if response.status_code == requests.codes.ok: context["event"].target.respond( u'"{0}" at {1}'.format( title, new_url.hostname ) ) else: context["event"].target.respond( u'[HTTP {0}] "{1}" at {2}'.format( response.status_code, title, new_url.hostname ) ) else: if response.status_code != requests.codes.ok: context["event"].target.respond( u'HTTP Error {0}: "{1}" at {2}'.format( response.status_code, STATUS_CODES.get(response.status_code, "Unknown"), new_url.hostname ) ) else: self.plugin.logger.debug("No title") self.save_session(session)
def shodan_lookup_ipaddr(title, ipaddr, asset_id, host_id, refresh=False): shodan_api = shodan.Shodan(orca_helpers.get_shodan_key()) orca_dbconn = OrcaDbConnector() orca_dbconn.init_host_table_name(title) orca_dbconn.init_shodan_table_name(title) orca_dbconn.init_vuln_table_name(title) orca_dir = os.path.expanduser(ORCA_CONFIG_DIR + "ipasn_db.dat") asndb = pyasn.pyasn(orca_dir) na_ipaddr = IPAddress(ipaddr) if not na_ipaddr.is_private(): if (not orca_dbconn.is_ipaddr_in_db(ipaddr)) or refresh: shodan_response = get_shodan_data(shodan_api, ipaddr) if shodan_response is not None: hostname = [] org = '' country = '' asn = '' last_update = '' ports = [] banners = [] cpes = {} modules = [] if len(shodan_response['hostnames']) > 0: hostname = shodan_response['hostnames'] for port in shodan_response['ports']: ports.append(int(port)) for res in shodan_response['data']: modules.append(res['_shodan']['module']) for banner in shodan_response['data']: if 'data' in banner: banners.append(banner['data']) for i in range(0, len(shodan_response['data']) - 1): if 'cpe' in shodan_response['data'][i]: if 'cpe' not in cpes: cpes['cpe'] = [{ shodan_response['data'][i]['_shodan']['module']: shodan_response['data'][i]['cpe'] }] else: cpes['cpe'].extend([{ shodan_response['data'][i]['_shodan']['module']: shodan_response['data'][i]['cpe'] }]) if 'org' in shodan_response: org = shodan_response['org'] if 'last_update' in shodan_response: last_update = shodan_response['last_update'] if '.' in last_update: last_update = last_update.rsplit('.')[0] if 'asn' in shodan_response: asn = shodan_response['asn'] if 'country_code3' in shodan_response: country = shodan_response['country_code3'] cidr = lookup_cidr_prefix(asndb, ipaddr) # tqdm.write("ipaddr {} hostname {} timestamp {} last_update {} module {} ports {} cpe {} netname {} asn {} country {} cidr {}".format(blessed_t.yellow(ipaddr), blessed_t.cyan( # orca_helpers.list_to_string(hostname)), blessed_t.cyan(datetime.now().isoformat(' ', 'seconds')), blessed_t.cyan(last_update), blessed_t.cyan(','.join(modules)), blessed_t.cyan(",".join("{0}".format(n) for n in ports)), blessed_t.cyan( # orca_helpers.cpe_to_string(cpes)), blessed_t.cyan(org), blessed_t.cyan(asn), blessed_t.cyan(country), blessed_t.cyan(cidr))) # tqdm.write( "\nIP Address - {ipaddr} [{host_value}]" "\n{ports_text:>12}: {ports_value}" "\n{modules_text:>12}: {modules_value}" "\n{asn_text:>12}: {asn_value}" "\n{netblock_text:>12}: {netblock_value}" "\n{org_text:>12}: {org_value}" "\n{country_text:>12}: {country_value}" "\n{cpe_text:>12}: {cpe_value}".format( ipaddr=blessed_t.yellow(ipaddr), host_text="Hostname", host_value=orca_helpers.list_to_string(hostname), ports_text="Ports", ports_value=",".join("{0}".format(n) for n in ports), modules_text="Modules", modules_value=','.join(modules), asn_text="AS Number", asn_value=asn, netblock_text="Netblock", netblock_value=cidr, org_text="Organization", org_value=org, country_text="Country", country_value=country, cpe_text="CPE", cpe_value=orca_helpers.cpe_to_string(cpes))) orca_dbconn.add_entry_to_db( ipaddr, date.fromtimestamp(time.time()), datetime.strptime(last_update, "%Y-%m-%dT%H:%M:%S"), modules, ports, banners, json.dumps(cpes), hostname, org, cidr, asn, country, asset_id, host_id) for i in range(0, len(shodan_response['data']) - 1): if 'vulns' in shodan_response['data'][i]: for cve, data in shodan_response['data'][i][ 'vulns'].items(): if 'cpe' in shodan_response['data'][i]: # tqdm.write("Adding vuln {} IP address {} hostname {} module {} CPE {} CVSS {} Summary {}".format(blessed_t.red(cve), blessed_t.yellow(ipaddr), blessed_t.cyan( # orca_helpers.list_to_string(hostname)), blessed_t.cyan(shodan_response['data'][i]['_shodan']['module']), blessed_t.cyan(','.join(shodan_response['data'][i]['cpe'])), blessed_t.red(str(data['cvss'])), blessed_t.cyan(data['summary']))) tqdm.write( "\t{cve:>20} [CVSS: {cvss:3} - Module: {module}]: \"{summary}\"" .format(cve=cve, cvss=str(data['cvss']), module=shodan_response['data'][i] ['_shodan']['module'], summary=textwrap.shorten( data['summary'], width=128, placeholder="..."))) orca_dbconn.add_vuln_to_db( orca_dbconn.get_hostid_from_ipaddr(ipaddr), ipaddr, hostname, shodan_response['data'] [i]['_shodan']['module'], shodan_response['data'][i]['cpe'], cve, data['cvss'], data['verified'], data['summary']) else: #tqdm.write("{:16s} not found in SHODAN".format(ipaddr)) pass else: tqdm.write("{:16s}| IP addr in DB".format(ipaddr)) else: tqdm.write("{:16s}| Skipping private IP".format(ipaddr))
def isPublic(ipAddr): myIpAddr = IPAddress(ipAddr) # unicast and not private return myIpAddr.is_unicast() and not myIpAddr.is_private()
class IpAddress(DatabaseObject): ''' Wraps the netaddr IPAddress class ''' uuid = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid4())) box_id = Column(Integer, ForeignKey('box.id'), nullable=False) _address = Column(String(40), unique=True) _ip_address = None @classmethod def all(cls): ''' Returns a list of all objects in the database ''' return dbsession.query(cls).all() @classmethod def by_id(cls, _id): ''' Returns a the object with id of _id ''' return dbsession.query(cls).filter_by(id=_id).first() @classmethod def by_uuid(cls, _uuid): ''' Return and object based on a _uuid ''' return dbsession.query(cls).filter_by(uuid=_uuid).first() @classmethod def by_address(cls, addr): ''' Return and object based on an address ''' return dbsession.query(cls).filter_by(address=addr).first() @property def address(self): if self._ip_address is None: self._ip_address = IPAddress(self._address) return self._ip_address.format() @address.setter def address(self, value): ip = IPAddress(value) if ip.is_loopback(): raise ValueError("You cannot use a loopback address") if ip.is_multicast(): raise ValueError("You cannot use a multicast address") self._address = value @property def version(self): if self._ip_address is None: self._ip_address = IPAddress(self._address) return self._ip_address.version @property def is_private(self): if self._ip_address is None: self._ip_address = IPAddress(self._address) return self._ip_address.is_private() def to_xml(self, parent): ip_elem = ET.SubElement(parent, "ip") ip_elem.set("version", str(self.version)) ET.SubElement(ip_elem, "address").text = self.address def __repr__(self): return "<IpAddress - %s>" % self.address def __str__(self): return self._address def __eq__(self, other): return self._address == other._address def __ne__(self, other): return not self == other
def test_is_public(): ip = IPAddress('62.125.24.5') assert ip.is_unicast() and not ip.is_private()
def validate_ip(self, ip): ip = IPAddress(ip) if ip.is_unicast() and not ip.is_private(): return True else: return False
def _is_public(self, ip): ip = IPAddress(ip) return ip.is_unicast() and not ip.is_private()
def reserved_ip_check(ip_string): """determine if IP address in RFC1918 or reserved""" # IP details for invalid IP addresses invalid_ip_details = { "country": "INVALID", "location": RESERVED_IP_COORDINATES, "subdivisions": "INVALID", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for MULTICAST IP addresses multicast_ip_details = { "country": "MULTICAST", "location": RESERVED_IP_COORDINATES, "subdivisions": "MULTICAST", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for PRIVATE IP addresses private_ip_details = { "country": "PRIVATE", "location": RESERVED_IP_COORDINATES, "subdivisions": "PRIVATE", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for RESERVED IP addresses reserved_ip_details = { "country": "RESERVED", "location": RESERVED_IP_COORDINATES, "subdivisions": "RESERVED", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for NETMASK IP addresses netmask_ip_details = { "country": "NETMASK", "location": RESERVED_IP_COORDINATES, "subdivisions": "NETMASK", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for HOSTMASK IP addresses hostmask_ip_details = { "country": "HOSTMASK", "location": RESERVED_IP_COORDINATES, "subdivisions": "HOSTMASK", "dch_company": "", "asn_number": "", "asn_name": "" } # IP details for LOOPBACK IP addresses loopback_ip_details = { "country": "LOOPBACK", "location": RESERVED_IP_COORDINATES, "subdivisions": "LOOPBACK", "dch_company": "", "asn_number": "", "asn_name": "" } # Check to see if IP matches a reserved category try: ip_address = IPAddress(ip_string) except AddrFormatError: return invalid_ip_details if ip_address.is_multicast(): return multicast_ip_details elif ip_address.is_private(): return private_ip_details elif ip_address.is_reserved(): return reserved_ip_details elif ip_address.is_netmask(): return netmask_ip_details elif ip_address.is_hostmask(): return hostmask_ip_details elif ip_address.is_loopback(): return loopback_ip_details elif ip_address.is_unicast() and not ip_address.is_private(): # Boolean to be returned if IP is Public ip_reserved = False return ip_reserved else: return invalid_ip_details
def fifoReader(infile, q, exitSignal): sleeptime = 0.5 maxSleeptime = 1.0 while True: try: if exitSignal.is_set(): break line = infile.readline() if not line: time.sleep(1) continue if line == 'ENDOFFILE': break try: spl = line.split() timestamp, queriedName, clientID, ipv4 = spl except: continue else: if not '.' in queriedName: continue try: addr = IPAddress(ipv4) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): try: timestamp = int(timestamp) except ValueError: continue else: data = ((queriedName, clientID, [addr]), timestamp) queued = False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime *= 0.5 elif q.qsize() >= q._maxsize: sleeptime *= 2 if sleeptime > maxSleeptime: sleeptime = maxSleeptime else: queued = True except KeyboardInterrupt: break q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0): if not infile and not interface: # FIXME: write warning here return if infile: pc = pcap.pcapObject() try: pc.open_offline(infile) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass if interface: pc = pcap.pcapObject() try: #pc.open_live(interface, snaplen, promisc, read_timeout) pc.open_live(interface, 1600, 0, 100) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass except Exception: # most likely we got no permission to open the interface sys.stderr.write('could not open interface. insufficient ' 'permissions?\n') q.put(None) return pc.setfilter('udp', 0, 0) basets = 0 newMappings = dict() while True: if exitSignal.is_set(): break try: packet = pc.next() if not packet: if infile: # end of file break elif interface: # read timeout continue payload = packet[1] timestamp = int(packet[2]) # make sure we are dealing with IP traffic # ref: http://www.iana.org/assignments/ethernet-numbers try: eth = dpkt.ethernet.Ethernet(payload) except: continue if eth.type != 2048: continue # make sure we are dealing with UDP # ref: http://www.iana.org/assignments/protocol-numbers/ try: ip = eth.data except: continue if ip.p != 17: continue # filter on UDP assigned ports for DNS # ref: http://www.iana.org/assignments/port-numbers try: udp = ip.data except: continue if udp.sport != 53 and udp.dport != 53: continue # make the dns object out of the udp data and check for it being a RR (answer) # and for opcode QUERY (I know, counter-intuitive) try: dns = dpkt.dns.DNS(udp.data) except: continue if dns.qr != dpkt.dns.DNS_R: continue if dns.opcode != dpkt.dns.DNS_QUERY: continue if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue if len(dns.an) < 1: continue if len(dns.qd) == 0: continue aRecords = set() queriedName = dns.qd[0].name if not '.' in queriedName: continue #lastCname=queriedName for answer in dns.an: """ FIXME: this doesn't work for multiple queries in one DNS packet """ #if answer.type == dpkt.dns.DNS_CNAME: # lastCname=answer.cname if answer.type == dpkt.dns.DNS_A: ip = socket.inet_ntoa(answer.rdata) try: addr = IPAddress(ip) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): aRecords.add(addr) if thrsh: if (timestamp - basets) > thrsh: basets = timestamp newMappings.clear() newIps = checkMapping(newMappings, queriedName, aRecords) aRecords = newIps if not aRecords: continue data = ((queriedName, ip.dst, aRecords), timestamp) queued = False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime *= 0.5 elif q.qsize() >= q._maxsize: sleeptime *= 2 if sleeptime > maxSleeptime: sleeptime = maxSleeptime else: queued = True except KeyboardInterrupt: break """ send shutdown signal """ q.put(None)
def check_ip(addr): """Verify that the server/client IP is valid""" addr = IPAddress(addr) if addr.version != 4 or not addr.is_private(): raise ValidationError("Only private IPv4 networks are supported.")
def fifoReader(infile, q, exitSignal): sleeptime=0.5 maxSleeptime=1.0 while True: try: if exitSignal.is_set(): break line=infile.readline() if not line: time.sleep(1) continue if line=='ENDOFFILE': break try: spl=line.split() timestamp, queriedName, clientID, ipv4 = spl except: continue else: if not '.' in queriedName: continue try: addr=IPAddress(ipv4) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): try: timestamp=int(timestamp) except ValueError: continue else: data = ((queriedName, clientID, [addr]), timestamp) queued=False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime*=0.5 elif q.qsize() >= q._maxsize: sleeptime*=2 if sleeptime>maxSleeptime: sleeptime=maxSleeptime else: queued=True except KeyboardInterrupt: break q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0): if not infile and not interface: # FIXME: write warning here return if infile: pc=pcap.pcapObject() try: pc.open_offline(infile) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass if interface: pc=pcap.pcapObject() try: #pc.open_live(interface, snaplen, promisc, read_timeout) pc.open_live(interface, 1600, 0, 100) except IOError: #log("could not open pcap interface "+str(input_interface)+"\n") pass except Exception: # most likely we got no permission to open the interface sys.stderr.write('could not open interface. insufficient ' 'permissions?\n') q.put(None) return pc.setfilter('udp', 0, 0) basets=0 newMappings=dict() while True: if exitSignal.is_set(): break try: packet=pc.next() if not packet: if infile: # end of file break elif interface: # read timeout continue payload=packet[1] timestamp=int(packet[2]) # make sure we are dealing with IP traffic # ref: http://www.iana.org/assignments/ethernet-numbers try: eth = dpkt.ethernet.Ethernet(payload) except: continue if eth.type != 2048: continue # make sure we are dealing with UDP # ref: http://www.iana.org/assignments/protocol-numbers/ try: ip = eth.data except: continue if ip.p != 17: continue # filter on UDP assigned ports for DNS # ref: http://www.iana.org/assignments/port-numbers try: udp = ip.data except: continue if udp.sport != 53 and udp.dport != 53: continue # make the dns object out of the udp data and check for it being a RR (answer) # and for opcode QUERY (I know, counter-intuitive) try: dns = dpkt.dns.DNS(udp.data) except: continue if dns.qr != dpkt.dns.DNS_R: continue if dns.opcode != dpkt.dns.DNS_QUERY: continue if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue if len(dns.an) < 1: continue if len(dns.qd) == 0: continue aRecords=set() queriedName=dns.qd[0].name if not '.' in queriedName: continue #lastCname=queriedName for answer in dns.an: """ FIXME: this doesn't work for multiple queries in one DNS packet """ #if answer.type == dpkt.dns.DNS_CNAME: # lastCname=answer.cname if answer.type == dpkt.dns.DNS_A: ip=socket.inet_ntoa(answer.rdata) try: addr=IPAddress(ip) except netaddr.AddrFormatError: continue else: if (addr.is_unicast() and not addr.is_private() and not addr.is_reserved() and not addr.is_loopback()): aRecords.add(addr) if thrsh: if (timestamp-basets) > thrsh: basets = timestamp newMappings.clear() newIps = checkMapping(newMappings, queriedName, aRecords) aRecords=newIps if not aRecords: continue data = ((queriedName, ip.dst, aRecords), timestamp) queued=False while not queued: try: q.put_nowait(data) except Queue.Full: # we saturated the queue, let's give the reading # process some time to empty it again, where we don't # try to put something in the queue and thereby lock it # continuously time.sleep(sleeptime) if q.empty(): sleeptime*=0.5 elif q.qsize() >= q._maxsize: sleeptime*=2 if sleeptime>maxSleeptime: sleeptime=maxSleeptime else: queued=True except KeyboardInterrupt: break """ send shutdown signal """ q.put(None)
class IpAddress(DatabaseObject): """ Wraps the netaddr IPAddress class """ uuid = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid4())) box_id = Column(Integer, ForeignKey("box.id"), nullable=False) _address = Column(String(80)) _ip_address = None visible = Column(Boolean, default=True) @classmethod def all(cls): """ Returns a list of all objects in the database """ return dbsession.query(cls).all() @classmethod def by_id(cls, _id): """ Returns a the object with id of _id """ return dbsession.query(cls).filter_by(id=_id).first() @classmethod def by_uuid(cls, _uuid): """ Return and object based on a _uuid """ return dbsession.query(cls).filter_by(uuid=_uuid).first() @classmethod def by_address(cls, address): """ Return and object based on an address """ return dbsession.query(cls).filter_by(_address=address).first() @classmethod def ipformat(self, value): ipformat = value if ipformat: if ipformat.count(":") == 1: # ip v4 with port ipformat = ipformat.split(":")[0] elif "]:" in ipformat: # ip v6 with port ipformat = ipformat.split("]:")[0] # ip v6 enclosing ipformat = ipformat.replace("[", "").replace("]", "") if "/" in ipformat: # remove any file info ipformat = ipformat.split("/")[0] return ipformat @property def address(self): return self._address @address.setter def address(self, value): try: ip = IPAddress(self.ipformat(value)) except: raise ValidationError("Invalid IP Address") if ip.is_loopback(): raise ValidationError("You cannot use a loopback address") if ip.is_multicast(): raise ValidationError("You cannot use a multicast address") self._address = value @property def version(self): if self._ip_address is None: self._ip_address = IPAddress(self.ipformat(self._address)) return self._ip_address.version @property def is_private(self): if self._ip_address is None: self._ip_address = IPAddress(self.ipformat(self._address)) return self._ip_address.is_private() def to_xml(self, parent): ip_elem = ET.SubElement(parent, "ip") ip_elem.set("version", str(self.version)) ET.SubElement(ip_elem, "address").text = self.address def __repr__(self): return "<IpAddress - %s>" % self.address def __str__(self): return self._address def __eq__(self, other): return self._address == other._address def __ne__(self, other): return not self == other
class IpAddress(DatabaseObject): ''' Wraps the netaddr IPAddress class ''' uuid = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid4()) ) box_id = Column(Integer, ForeignKey('box.id'), nullable=False) _address = Column(String(80)) _ip_address = None visable = Column(Boolean, default=True) @classmethod def all(cls): ''' Returns a list of all objects in the database ''' return dbsession.query(cls).all() @classmethod def by_id(cls, _id): ''' Returns a the object with id of _id ''' return dbsession.query(cls).filter_by(id=_id).first() @classmethod def by_uuid(cls, _uuid): ''' Return and object based on a _uuid ''' return dbsession.query(cls).filter_by(uuid=_uuid).first() @classmethod def by_address(cls, address): ''' Return and object based on an address ''' return dbsession.query(cls).filter_by(_address=address).first() @classmethod def ipformat(self, value): ipformat = value if ipformat: if ipformat.count(":") == 1: #ip v4 with port ipformat = ipformat.split(":")[0] elif "]:" in ipformat: #ip v6 with port ipformat = ipformat.split("]:")[0] #ip v6 enclosing ipformat = ipformat.replace("[", "").replace("]", "") if "/" in ipformat: #remove any file info ipformat = ipformat.split("/")[0] return ipformat @property def address(self): return self._address @address.setter def address(self, value): ip = IPAddress(self.ipformat(value)) if ip.is_loopback(): raise ValidationError("You cannot use a loopback address") if ip.is_multicast(): raise ValidationError("You cannot use a multicast address") self._address = value @property def version(self): if self._ip_address is None: self._ip_address = IPAddress(self.ipformat(self._address)) return self._ip_address.version @property def is_private(self): if self._ip_address is None: self._ip_address = IPAddress(self.ipformat(self._address)) return self._ip_address.is_private() def to_xml(self, parent): ip_elem = ET.SubElement(parent, "ip") ip_elem.set("version", str(self.version)) ET.SubElement(ip_elem, "address").text = self.address def __repr__(self): return "<IpAddress - %s>" % self.address def __str__(self): return self._address def __eq__(self, other): return self._address == other._address def __ne__(self, other): return not self == other
def main(argv): output = None desc = """ ____/ /____ _ / /_ ____ _ _____ ____ / /____ (_)/ /_ / __ // __ `// __// __ `// ___// __ \ / // __ \ / // __/ / /_/ // /_/ // /_ / /_/ /(__ )/ /_/ // // /_/ // // /_ \__,_/ \__,_/ \__/ \__,_//____// .___//_/ \____//_/ \__/ /_/ Open Source Assistant for #OSINT www.datasploit.info """ epilog = """ Connect at Social Media: @datasploit """ # Set all parser arguments here. parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(desc), epilog=epilog) parser.add_argument("-i", "--input", help="Provide Input", dest='target', required=True) parser.add_argument("-a", "--active", help="Run Active Scan attacks", dest='active', action="store_false") parser.add_argument( "-q", "--quiet", help="Run scans in automated manner accepting default answers", dest='quiet', action="store_false") parser.add_argument("-o", "--output", help="Provide Destination Directory", dest='output') # check and ensure the config file is present otherwise create one. required for all further operations ds_dir = os.path.dirname(os.path.realpath(__file__)) config_file_path = os.path.join(ds_dir, "config.py") config_sample_path = os.path.join(ds_dir, "config_sample.py") print os.path.exists(config_file_path) if not os.path.exists(config_file_path): print "[+] Looks like a new setup, setting up the config file." shutil.copyfile(config_sample_path, config_file_path) print "[+] A config file is added please follow guide at https://datasploit.github.io/datasploit/apiGeneration/ to fill API Keys for better results" # We can think about quiting at this point. # if no argument is provided print help and quit if len(argv) == 0: parser.print_help() sys.exit() # parse arguments in case they are provided. x = parser.parse_args() active = x.active quiet = x.quiet user_input = x.target output = x.output # Banner print print textwrap.dedent(desc) # Auto selection logic try: print "User Input: %s" % user_input try: inp = IPAddress(user_input) if inp.is_private() or inp.is_loopback(): print "Internal IP Detected : Skipping" sys.exit() else: print "Looks like an IP, running ipOsint...\n" ipOsint.run(user_input, output) except SystemExit: print "exiting" except AddrFormatError: if re.match('[^@]+@[^@]+\.[^@]+', user_input): print "Looks like an EMAIL, running emailOsint...\n" emailOsint.run(user_input, output) elif get_tld(user_input, fix_protocol=True, fail_silently=True) is not None: print "Looks like a DOMAIN, running domainOsint...\n" domainOsint.run(user_input, output) else: print "Nothing Matched assuming username, running usernameOsint...\n" usernameOsint.run(user_input, output) except: print "Unknown Error Occured" except KeyboardInterrupt: print "Ctrl+C called Quiting"
def has_public_ip(self): ip = IPAddress(self.ip) return ip.is_unicast() and not ip.is_private()