def get_from_ip(cls,
ip,
port,
type,
hostname=None,
timestamp=datetime.utcnow()):
try:
ip_addr = IPAddress(ip)
except netaddr.core.AddrFormatError:
logger.debug(f"Failed to parse {ip}")
return None
geo = None
if not ip_addr.is_private() and not ip_addr.is_reserved():
geo = cls.get_geo(ip)
if not hostname:
hostname = cls.get_rdns(ip)
return NetworkEntity(ip,
port,
type,
geo,
hostname=hostname,
timestamp=timestamp,
is_enriched=True)
def auto_select_target(target, output=None):
"""Auto selection logic"""
print "Target: %s" % target
try:
inp=IPAddress(target);
if inp.is_private() or inp.is_loopback():
print "Internal IP Detected : Skipping"
sys.exit()
else:
print "Looks like an IP, running ipOsint...\n"
ipOsint.run(target, output)
except SystemExit:
print "exiting"
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print "Looks like an EMAIL, running emailOsint...\n"
emailOsint.run(target, output)
elif get_tld(target, fix_protocol=True,fail_silently=True) is not None:
print "Looks like a DOMAIN, running domainOsint...\n"
domainOsint.run(target, output)
else:
print "Nothing Matched assuming username, running usernameOsint...\n"
usernameOsint.run(target, output)
except:
print "Unknown Error Occured"
def get_priv_info(d_iface_to_addr=None):
s_net_id = None
s_priv_ip = None
s_priv_interface = None
if d_iface_to_addr is None:
d_iface_to_addr = get_iface_to_addr()
networks = search('net', 'name', '*')
for s_iface, d_addr in d_iface_to_addr.items():
if s_iface.startswith('lo'):
continue
if netifaces.AF_INET not in d_addr:
continue
ips = d_addr[netifaces.AF_INET]
for ip in ips:
o_ip = IPAddress(str(ip['addr']))
if not o_ip.is_private():
continue
if ip['addr'] == '127.0.0.1':
continue
for net in networks:
if (('netmask' in net) and
(o_ip in IPNetwork(net['netmask']))):
s_priv_ip = str(ip['addr'])
s_priv_interface = s_iface
s_net_id = net['name']
break
return (s_priv_ip, s_priv_interface, s_net_id)
def auto_select_target(target, output=None):
"""Auto selection logic"""
print "Target: %s" % target
try:
inp = IPAddress(target)
if inp.is_private() or inp.is_loopback():
print "Internal IP Detected : Skipping"
sys.exit()
else:
print "Looks like an IP, running ipOsint...\n"
ipOsint.run(target, output)
except SystemExit:
print "exiting"
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print "Looks like an EMAIL, running emailOsint...\n"
return emailOsint.run(target, output)
elif get_tld(target, fix_protocol=True,
fail_silently=True) is not None:
print "Looks like a DOMAIN, running domainOsint...\n"
# domainOsint.run(target, output)
return {
'error': True,
'message': 'We do not do domain searches yet, sorry!'
}
else:
print "Nothing Matched assuming username, running usernameOsint...\n"
# usernameOsint.run(target, output)
return {
'error': True,
'message': 'We do not do username searches yet, sorry!'
}
except:
print "Unknown Error Occured"
def auto_select_target(target, output=None):
"""Auto selection logic"""
print("Target: %s" % target)
try:
inp = IPAddress(target);
if inp.is_private() or inp.is_loopback():
print("Internal IP Detected : Skipping")
sys.exit()
else:
print("Looks like an IP, running ipOsint...\n")
ipOsint.run(target, output)
except SystemExit:
print("exiting")
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print("Looks like an EMAIL, running emailOsint...\n")
emailOsint.run(target, output)
elif get_tld(target, fix_protocol=True, fail_silently=True) is not None:
print("Looks like a DOMAIN, running domainOsint...\n")
domainOsint.run(target, output)
else:
print("Nothing Matched assuming username, running usernameOsint...\n")
usernameOsint.run(target, output)
except:
print("Unknown Error Occured")
def to_server_dict(server):
public_ips = [ip["addr"] for ip in server.addresses["public"]]
private_ips = [ip["addr"] for ip in server.addresses["private"]]
# Pick out first public IPv4 and IPv6 address
public_ipv4 = None
public_ipv6 = None
for ip in public_ips:
try:
ip_obj = IPAddress(ip)
except Exception:
continue
if not ip_obj.is_private():
if ip_obj.version == 4:
public_ipv4 = ip
elif ip_obj.version == 6:
public_ipv6 = ip
result = {
"id": server.id,
"name": server.name,
"status": server.status,
"image_id": server.image["id"],
"flavor_id": server.flavor["id"],
"public_ips": public_ips,
"private_ips": private_ips,
"public_ipv4": public_ipv4,
"public_ipv6": public_ipv6,
"key_name": server.key_name,
"metadata": server.metadata,
}
return result
def get_ip_address(self, test_address=None):
"""
try to get global IP address from interface information.
if failed, just return '127.0.0.1'
:param str test_address: ip address str if test to check global ip.
normally None.
:return: global ip address if successed, or '127.0.0.1'
"""
for iface_name in netifaces.interfaces():
iface_data = netifaces.ifaddresses(iface_name)
logging.debug('Interface: %s' % (iface_name, ))
ifaces = []
if netifaces.AF_INET in iface_data:
ifaces += iface_data[netifaces.AF_INET]
if netifaces.AF_INET6 in iface_data:
ifaces += iface_data[netifaces.AF_INET6]
for iface in ifaces:
ip = iface['addr']
ip = re.sub(r'\%.+$', '', ip)
if test_address is not None:
ip = test_address
addr = IPAddress(ip)
if not addr.is_loopback() and addr.is_unicast() and\
not addr.is_private():
logging.debug('global ip %s', addr)
return ip
logging.debug('no global ip')
return '127.0.0.1'
def is_valid_address(address):
''' Validate whether the address provided is routable unicast address '''
addr = IPAddress(address)
if addr.is_loopback() or addr.is_reserved() or addr.is_private()\
or addr.is_link_local() or addr.is_multicast():
return False
return True
def _validate_network_information(self):
all_errs = []
if self.subnet_mask_size > self.MAX_SUBNET_MASK_SIZE:
all_errs.append(
"Subnet size too small. Subnet mask should <= {}, current value: {}"
.format(self.MAX_SUBNET_MASK_SIZE, self.subnet_mask_size))
cidr_base_validator = re.compile(self.VPC_CIDR_BASE_REGEX)
if not cidr_base_validator.match(self.vpc_cidr_base):
all_errs.append(
"Invalid VPC CIDR base {}. VPC CIDR base should match regex {}"
.format(self.vpc_cidr_base, self.VPC_CIDR_BASE_REGEX))
if self.trusted_cidrs:
try:
for cidr in self.trusted_cidrs:
[ip, mask] = cidr.split("/")
if ip == "0.0.0.0":
if mask != "0":
all_errs.append(
"Trusting traffic from everywhere should specify \"0.0.0.0/0\" as trusted CIDR."
)
else:
if not 0 < int(mask) <= 32:
all_errs.append(
"Subnet mask {} should be greater than 0 but less than 32."
.format(mask))
ipaddr = IPAddress(ip)
if ipaddr.is_netmask():
all_errs.append(
"Trusted CIDR {} should not be a net mask".
format(ip))
if ipaddr.is_hostmask():
all_errs.append(
"Trusted CIDR {} should not be a host mask".
format(ip))
if ipaddr.is_reserved():
all_errs.append(
"Trusted CIDR {} should not be in reserved range"
.format(ip))
if ipaddr.is_loopback():
all_errs.append(
"Trusted CIDR {} should not be a loop back address"
.format(ip))
# Currently we don't support private VPC
if ipaddr.is_private():
all_errs.append(
"Trusted CIDR {} should not be a private address"
.format(ip))
except ValueError as ve:
all_errs.append(
"Cannot parse trusted CIDRs ({}). Err: {}".format(
self.trusted_cidrs, ve))
else:
all_errs.append(
"Please provide trusted CIDRs through --trusted-cidrs flag")
return all_errs
def _is_local_network(self, address):
url = urlparse.urlparse(address)
try:
net = IPAddress(url)
return net.is_private()
except (AddrFormatError, TypeError):
return False
def valid_ip(self, ip_name):
"""
this method checks the validation of provided ip i.e whether,
1. ip contains 4 octets or not
2. each octet of ip in range of 256 or not
3. provided ip is public or not
:param ip_name:provided ip. ex: 8.8.8.8
:return: either true or false
"""
try:
ip_name = IPAddress(ip_name)
except AddrFormatError:
print(f"invalid IPaddress.provide data as ips")
return False
else:
if ip_name.version in [4, 6]:
if ip_name.is_private():
print(
f'provided {ip_name} is private. don\'t use private ips for reputation...'
)
return False
else:
return True
else:
return True
def getNetwork(self, ip_address):
'''return the site and network names for an IP address
we assume the site is the first level subgroup (group = CLIENT.SiteName)
for public adresses, we try a reverse DNS and a WHOIS lookups'''
ip = IPAddress(ip_address)
for _, row in self.networks.iterrows():
if ip in IPNetwork(row['cidr']):
return (row['group'].split('.')[1], row['name'])
# if the IP is not private and hasn't been found in network hierarchy
# try a reverse DNS lookup
if ip.is_private():
return ('', '')
else:
try:
res = gethostbyaddr(ip_address)
return ('', res[0])
except herror:
# Reverse DNS failed, let's do a WHOIS lookup
try:
res = IPWhois(ip).lookup_whois()
return (res['asn_country_code'], res['asn_description'][:30])
except:
return ('', '')
def is_public_ip(ip):
ip_obj = IPAddress(ip)
if not ip_obj.is_private() and not ip_obj.is_reserved():
return True
return False
def to_server_dict(server):
public_ips = [ip['addr'] for ip in server.addresses['public']]
private_ips = [ip['addr'] for ip in server.addresses['private']]
# Pick out first public IPv4 and IPv6 address
public_ipv4 = None
public_ipv6 = None
for ip in public_ips:
try:
ip_obj = IPAddress(ip)
except Exception:
continue
if not ip_obj.is_private():
if ip_obj.version == 4:
public_ipv4 = ip
elif ip_obj.version == 6:
public_ipv6 = ip
result = {
'id': server.id,
'name': server.name,
'status': server.status,
'image_id': server.image['id'],
'flavor_id': server.flavor['id'],
'public_ips': public_ips,
'private_ips': private_ips,
'public_ipv4': public_ipv4,
'public_ipv6': public_ipv6,
'key_name': server.key_name,
'metadata': server.metadata
}
return result
def is_public_ip(ip_address):
try:
ip_addr = IPAddress(str(ip_address))
# We also need to filter multicast ip which is a private ip. For example,
# multicast ip '239.192.0.1', is a private ip.
return ip_addr.is_unicast() and not ip_addr.is_private()
except: # pylint: disable=bare-except
return False
def call(self, url, context):
if self.url_can_resolve(url):
try:
ip = yield self.resolver.get_host_by_name(url.domain)
ip = IPAddress(ip)
except Exception:
# context["event"].target.respond(
# u'[Error] Failed to handle URL: {}'.format(
# url.to_string()
# )
# )
self.plugin.logger.exception("Error while checking DNS")
returnValue(STOP_HANDLING)
return
if ip.is_loopback() or ip.is_private() or ip.is_link_local() \
or ip.is_multicast():
self.plugin.logger.warn(
"Prevented connection to private/internal address"
)
returnValue(STOP_HANDLING)
return
headers = {}
if url.domain in context["config"]["spoofing"]:
user_agent = context["config"]["spoofing"][url.domain]
if user_agent:
headers["User-Agent"] = user_agent
else:
headers["User-Agent"] = context["config"].get(
"default_user_agent",
"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 "
"Firefox/36.0"
)
domain_langs = context.get("config") \
.get("accept_language", {}) \
.get("domains", {})
if url.domain in domain_langs:
headers["Accept-Language"] = domain_langs.get(url.domain)
else:
headers["Accept-Language"] = context.get("config") \
.get("accept_language", {}) \
.get("default", "en")
session = self.get_session(url, context)
session.get(unicode(url), headers=headers, stream=True,
background_callback=self.background_callback) \
.addCallback(self.callback, url, context, session) \
.addErrback(self.errback, url, context, session)
returnValue(STOP_HANDLING)
def auto_select_target(self, target, output=None):
"""Auto selection logic"""
result = {}
print("Target: %s" % target)
try:
inp = IPAddress(target)
if inp.is_private() or inp.is_loopback():
print("Internal IP Detected : Skipping")
sys.exit()
else:
print("Looks like an IP, running ipOsint...\n")
data = ipOsint.run(target, output)
#print("*************************************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
except SystemExit:
print("exiting")
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', target):
print("Looks like an EMAIL, running emailOsint...\n")
data = emailOsint.run(target, output)
#print("*************************************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
elif get_tld(target, fix_protocol=True,
fail_silently=True) is not None:
print("Looks like a DOMAIN, running domainOsint...\n")
data = domainOsint.run(target, output)
#print("************************domainOsint*************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
else:
print(
"Nothing Matched assuming username, running usernameOsint...\n"
)
data = usernameOsint.run(target, output)
#print("*************************************************************************")
#print(data)
#print("*************************************************************************")
result = {"target": target, "data": data}
results.append(result)
return result
except:
print("Unknown Error Occured")
return result, 201
def validate(url,
schemes=None,
tlds=None,
private=None,
local=None,
credentials=None):
'''
Validate and normalize an URL
:param str url: The URL to validate and normalize
:return str: The normalized URL
:raises ValidationError: when URL does not validate
'''
url = url.strip()
private = config_for(private, 'URLS_ALLOW_PRIVATE')
local = config_for(local, 'URLS_ALLOW_LOCAL')
credentials = config_for(credentials, 'URLS_ALLOW_CREDENTIALS')
schemes = config_for(schemes, 'URLS_ALLOWED_SCHEMES')
tlds = config_for(tlds, 'URLS_ALLOWED_TLDS')
match = URL_REGEX.match(url)
if not match:
error(url)
scheme = (match.group('scheme') or '').lower()
if scheme and scheme not in schemes:
error(url, 'Invalid scheme {0}'.format(scheme))
if not credentials and match.group('credentials'):
error(url, 'Credentials in URL are not allowed')
tld = match.group('tld')
if tld and tld not in tlds and idna(tld) not in tlds:
error(url, 'Invalid TLD {0}'.format(tld))
ip = match.group('ipv6') or match.group('ipv4')
if ip:
try:
ip = IPAddress(ip)
except AddrFormatError:
error(url)
if ip.is_multicast():
error(url, '{0} is a multicast IP'.format(ip))
elif not ip.is_loopback() and ip.is_hostmask() or ip.is_netmask():
error(url, '{0} is a mask IP'.format(ip))
if not local:
if ip and ip.is_loopback() or match.group('localhost'):
error(url, 'is a local URL')
if not private and ip and ip.is_private():
error(url, 'is a private URL')
return url
def metadata_file_writer(q, filename):
PLUGINS = load_metadata_plugins()
metadata_file = open(filename, "w")
ip_adresses = [0]
while True:
connection = q.get()
ip = connection[6]
ipAddress = IPAddress(hex_to_ip(ip))
if ip not in ip_adresses and not ipAddress.is_private() and not ipAddress.is_loopback():
ip_adresses.append(ip)
for p in PLUGINS:
p.set_connection(connection)
res = p.run()
if len(res):
metadata_file.write("%s, %s,%s\n" % (p.name, hex_to_ip(ip), res))
metadata_file.flush()
q.task_done()
def check_ip_address(ipaddr):
ip_attributes = []
ipaddress = IPAddress(ipaddr)
if ipaddress.is_private():
ip_attributes.append("IP Address is Private")
else:
ip_attributes.append("IP Address is public")
if ipaddress.is_unicast():
ip_attributes.append("IP Address is unicast")
elif ipaddress.is_multicast():
ip_attributes.append("IP Address is multicast")
if ipaddress.is_loopback():
ip_attributes.append("IP Address is loopback")
return "\n".join(ip_attributes)
def _trusted_cidr_validator(input):
from netaddr import IPAddress
ret = []
for cidr in input.split(" "):
cidr = cidr.strip()
if not cidr:
# skip whitespace
continue
ip, mask = cidr.split("/")
if int(mask) < 0 or int(mask) > 32:
raise ValueError(
"CIDR {} is not valid as mask {} is not in range [0-32]".
format(cidr, mask))
if ip != "0.0.0.0" or mask != '0':
ipaddr = IPAddress(ip)
if ipaddr.is_netmask():
raise ValueError(
"Trusted CIDR {} should not be a net mask".format(ip))
if ipaddr.is_hostmask():
raise ValueError(
"Trusted CIDR {} should not be a host mask".format(ip))
if ipaddr.is_reserved():
raise ValueError(
"Trusted CIDR {} should not be in reserved range".
format(ip))
if ipaddr.is_loopback():
raise ValueError(
"Trusted CIDR {} should not be a loop back address".
format(ip))
# Currently we don't support private VPC
if ipaddr.is_private():
raise ValueError(
"Trusted CIDR {} should not be a private address".
format(ip))
ret.append(cidr)
return ret
def get_from_hostname(cls, hostname, type, timestamp=datetime.utcnow()):
ip = cls.get_ip(hostname)
geo = None
try:
ip_addr = IPAddress(ip)
if ip_addr and not ip_addr.is_private() \
and not ip_addr.is_reserved():
geo = cls.get_geo(ip)
except netaddr.core.AddrFormatError:
logger.debug(f"Failed to parse {ip}")
return NetworkEntity(ip,
0,
type,
geo,
hostname=hostname,
timestamp=timestamp,
is_enriched=True)
def ip_check_routable(item):
ip_addr = IPAddress(item)
# This prevents netaddr allowing shortened ip addresses
if not str(ip_addr) == item:
raise AddrFormatError("IP Malformed {}".format(item))
# Check for reserved IP addresses
if any([
ip_addr.is_multicast(),
ip_addr.is_private(),
ip_addr.is_loopback(),
ip_addr.is_link_local(),
ip_addr.is_reserved()
]):
raise AddrFormatError("IP is reserved {}".format(item))
# Check to see if IP is IPv4
# elif ip_addr.version is not 4:
# raise AddrFormatError("IP is not IPv4")
return True
def callback(self, result, url, context, session):
response = result[0]
content = result[1]
self.plugin.logger.trace(
"Headers: {0}", list(response.headers)
)
self.plugin.logger.trace("HTTP code: {0}", response.status_code)
new_url = urlparse.urlparse(response.url)
if self.url_can_resolve(url):
try:
ip = yield self.resolver.get_host_by_name(new_url.hostname)
ip = IPAddress(ip)
except Exception:
# context["event"].target.respond(
# u'[Error] Failed to handle URL: {}'.format(
# url.to_string()
# )
# )
self.plugin.logger.exception("Error while checking DNS")
returnValue(STOP_HANDLING)
return
if ip.is_loopback() or ip.is_private() or ip.is_link_local() \
or ip.is_multicast():
self.plugin.logger.warn(
"Prevented connection to private/internal address"
)
returnValue(STOP_HANDLING)
return
if content is None:
self.plugin.logger.debug("No content returned")
return
soup = BeautifulSoup(content)
if soup.title and soup.title.text:
title = soup.title.text.strip()
title = re.sub("[\n\s]+", " ", title)
title = to_unicode(title)
title_limit = self.urls_plugin.config.get("max_title_length", 150)
if len(title) > title_limit:
title = title[:title_limit - 15] + u"... (truncated)"
if response.status_code == requests.codes.ok:
context["event"].target.respond(
u'"{0}" at {1}'.format(
title, new_url.hostname
)
)
else:
context["event"].target.respond(
u'[HTTP {0}] "{1}" at {2}'.format(
response.status_code,
title, new_url.hostname
)
)
else:
if response.status_code != requests.codes.ok:
context["event"].target.respond(
u'HTTP Error {0}: "{1}" at {2}'.format(
response.status_code,
STATUS_CODES.get(response.status_code, "Unknown"),
new_url.hostname
)
)
else:
self.plugin.logger.debug("No title")
self.save_session(session)
def shodan_lookup_ipaddr(title, ipaddr, asset_id, host_id, refresh=False):
shodan_api = shodan.Shodan(orca_helpers.get_shodan_key())
orca_dbconn = OrcaDbConnector()
orca_dbconn.init_host_table_name(title)
orca_dbconn.init_shodan_table_name(title)
orca_dbconn.init_vuln_table_name(title)
orca_dir = os.path.expanduser(ORCA_CONFIG_DIR + "ipasn_db.dat")
asndb = pyasn.pyasn(orca_dir)
na_ipaddr = IPAddress(ipaddr)
if not na_ipaddr.is_private():
if (not orca_dbconn.is_ipaddr_in_db(ipaddr)) or refresh:
shodan_response = get_shodan_data(shodan_api, ipaddr)
if shodan_response is not None:
hostname = []
org = ''
country = ''
asn = ''
last_update = ''
ports = []
banners = []
cpes = {}
modules = []
if len(shodan_response['hostnames']) > 0:
hostname = shodan_response['hostnames']
for port in shodan_response['ports']:
ports.append(int(port))
for res in shodan_response['data']:
modules.append(res['_shodan']['module'])
for banner in shodan_response['data']:
if 'data' in banner:
banners.append(banner['data'])
for i in range(0, len(shodan_response['data']) - 1):
if 'cpe' in shodan_response['data'][i]:
if 'cpe' not in cpes:
cpes['cpe'] = [{
shodan_response['data'][i]['_shodan']['module']:
shodan_response['data'][i]['cpe']
}]
else:
cpes['cpe'].extend([{
shodan_response['data'][i]['_shodan']['module']:
shodan_response['data'][i]['cpe']
}])
if 'org' in shodan_response:
org = shodan_response['org']
if 'last_update' in shodan_response:
last_update = shodan_response['last_update']
if '.' in last_update:
last_update = last_update.rsplit('.')[0]
if 'asn' in shodan_response:
asn = shodan_response['asn']
if 'country_code3' in shodan_response:
country = shodan_response['country_code3']
cidr = lookup_cidr_prefix(asndb, ipaddr)
# tqdm.write("ipaddr {} hostname {} timestamp {} last_update {} module {} ports {} cpe {} netname {} asn {} country {} cidr {}".format(blessed_t.yellow(ipaddr), blessed_t.cyan(
# orca_helpers.list_to_string(hostname)), blessed_t.cyan(datetime.now().isoformat(' ', 'seconds')), blessed_t.cyan(last_update), blessed_t.cyan(','.join(modules)), blessed_t.cyan(",".join("{0}".format(n) for n in ports)), blessed_t.cyan(
# orca_helpers.cpe_to_string(cpes)), blessed_t.cyan(org), blessed_t.cyan(asn), blessed_t.cyan(country), blessed_t.cyan(cidr)))
#
tqdm.write(
"\nIP Address - {ipaddr} [{host_value}]"
"\n{ports_text:>12}: {ports_value}"
"\n{modules_text:>12}: {modules_value}"
"\n{asn_text:>12}: {asn_value}"
"\n{netblock_text:>12}: {netblock_value}"
"\n{org_text:>12}: {org_value}"
"\n{country_text:>12}: {country_value}"
"\n{cpe_text:>12}: {cpe_value}".format(
ipaddr=blessed_t.yellow(ipaddr),
host_text="Hostname",
host_value=orca_helpers.list_to_string(hostname),
ports_text="Ports",
ports_value=",".join("{0}".format(n) for n in ports),
modules_text="Modules",
modules_value=','.join(modules),
asn_text="AS Number",
asn_value=asn,
netblock_text="Netblock",
netblock_value=cidr,
org_text="Organization",
org_value=org,
country_text="Country",
country_value=country,
cpe_text="CPE",
cpe_value=orca_helpers.cpe_to_string(cpes)))
orca_dbconn.add_entry_to_db(
ipaddr, date.fromtimestamp(time.time()),
datetime.strptime(last_update, "%Y-%m-%dT%H:%M:%S"),
modules, ports, banners, json.dumps(cpes), hostname, org,
cidr, asn, country, asset_id, host_id)
for i in range(0, len(shodan_response['data']) - 1):
if 'vulns' in shodan_response['data'][i]:
for cve, data in shodan_response['data'][i][
'vulns'].items():
if 'cpe' in shodan_response['data'][i]:
# tqdm.write("Adding vuln {} IP address {} hostname {} module {} CPE {} CVSS {} Summary {}".format(blessed_t.red(cve), blessed_t.yellow(ipaddr), blessed_t.cyan(
# orca_helpers.list_to_string(hostname)), blessed_t.cyan(shodan_response['data'][i]['_shodan']['module']), blessed_t.cyan(','.join(shodan_response['data'][i]['cpe'])), blessed_t.red(str(data['cvss'])), blessed_t.cyan(data['summary'])))
tqdm.write(
"\t{cve:>20} [CVSS: {cvss:3} - Module: {module}]: \"{summary}\""
.format(cve=cve,
cvss=str(data['cvss']),
module=shodan_response['data'][i]
['_shodan']['module'],
summary=textwrap.shorten(
data['summary'],
width=128,
placeholder="...")))
orca_dbconn.add_vuln_to_db(
orca_dbconn.get_hostid_from_ipaddr(ipaddr),
ipaddr, hostname, shodan_response['data']
[i]['_shodan']['module'],
shodan_response['data'][i]['cpe'], cve,
data['cvss'], data['verified'],
data['summary'])
else:
#tqdm.write("{:16s} not found in SHODAN".format(ipaddr))
pass
else:
tqdm.write("{:16s}| IP addr in DB".format(ipaddr))
else:
tqdm.write("{:16s}| Skipping private IP".format(ipaddr))
def isPublic(ipAddr):
myIpAddr = IPAddress(ipAddr)
# unicast and not private
return myIpAddr.is_unicast() and not myIpAddr.is_private()
class IpAddress(DatabaseObject):
''' Wraps the netaddr IPAddress class '''
uuid = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid4()))
box_id = Column(Integer, ForeignKey('box.id'), nullable=False)
_address = Column(String(40), unique=True)
_ip_address = None
@classmethod
def all(cls):
''' Returns a list of all objects in the database '''
return dbsession.query(cls).all()
@classmethod
def by_id(cls, _id):
''' Returns a the object with id of _id '''
return dbsession.query(cls).filter_by(id=_id).first()
@classmethod
def by_uuid(cls, _uuid):
''' Return and object based on a _uuid '''
return dbsession.query(cls).filter_by(uuid=_uuid).first()
@classmethod
def by_address(cls, addr):
''' Return and object based on an address '''
return dbsession.query(cls).filter_by(address=addr).first()
@property
def address(self):
if self._ip_address is None:
self._ip_address = IPAddress(self._address)
return self._ip_address.format()
@address.setter
def address(self, value):
ip = IPAddress(value)
if ip.is_loopback():
raise ValueError("You cannot use a loopback address")
if ip.is_multicast():
raise ValueError("You cannot use a multicast address")
self._address = value
@property
def version(self):
if self._ip_address is None:
self._ip_address = IPAddress(self._address)
return self._ip_address.version
@property
def is_private(self):
if self._ip_address is None:
self._ip_address = IPAddress(self._address)
return self._ip_address.is_private()
def to_xml(self, parent):
ip_elem = ET.SubElement(parent, "ip")
ip_elem.set("version", str(self.version))
ET.SubElement(ip_elem, "address").text = self.address
def __repr__(self):
return "<IpAddress - %s>" % self.address
def __str__(self):
return self._address
def __eq__(self, other):
return self._address == other._address
def __ne__(self, other):
return not self == other
def test_is_public():
ip = IPAddress('62.125.24.5')
assert ip.is_unicast() and not ip.is_private()
def validate_ip(self, ip):
ip = IPAddress(ip)
if ip.is_unicast() and not ip.is_private():
return True
else:
return False
def _is_public(self, ip):
ip = IPAddress(ip)
return ip.is_unicast() and not ip.is_private()
def reserved_ip_check(ip_string):
"""determine if IP address in RFC1918 or reserved"""
# IP details for invalid IP addresses
invalid_ip_details = {
"country": "INVALID",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "INVALID",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for MULTICAST IP addresses
multicast_ip_details = {
"country": "MULTICAST",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "MULTICAST",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for PRIVATE IP addresses
private_ip_details = {
"country": "PRIVATE",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "PRIVATE",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for RESERVED IP addresses
reserved_ip_details = {
"country": "RESERVED",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "RESERVED",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for NETMASK IP addresses
netmask_ip_details = {
"country": "NETMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "NETMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for HOSTMASK IP addresses
hostmask_ip_details = {
"country": "HOSTMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "HOSTMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for LOOPBACK IP addresses
loopback_ip_details = {
"country": "LOOPBACK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "LOOPBACK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# Check to see if IP matches a reserved category
try:
ip_address = IPAddress(ip_string)
except AddrFormatError:
return invalid_ip_details
if ip_address.is_multicast():
return multicast_ip_details
elif ip_address.is_private():
return private_ip_details
elif ip_address.is_reserved():
return reserved_ip_details
elif ip_address.is_netmask():
return netmask_ip_details
elif ip_address.is_hostmask():
return hostmask_ip_details
elif ip_address.is_loopback():
return loopback_ip_details
elif ip_address.is_unicast() and not ip_address.is_private():
# Boolean to be returned if IP is Public
ip_reserved = False
return ip_reserved
else:
return invalid_ip_details
def fifoReader(infile, q, exitSignal):
sleeptime = 0.5
maxSleeptime = 1.0
while True:
try:
if exitSignal.is_set(): break
line = infile.readline()
if not line:
time.sleep(1)
continue
if line == 'ENDOFFILE':
break
try:
spl = line.split()
timestamp, queriedName, clientID, ipv4 = spl
except:
continue
else:
if not '.' in queriedName:
continue
try:
addr = IPAddress(ipv4)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and not addr.is_private()
and not addr.is_reserved()
and not addr.is_loopback()):
try:
timestamp = int(timestamp)
except ValueError:
continue
else:
data = ((queriedName, clientID, [addr]), timestamp)
queued = False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime *= 0.5
elif q.qsize() >= q._maxsize:
sleeptime *= 2
if sleeptime > maxSleeptime:
sleeptime = maxSleeptime
else:
queued = True
except KeyboardInterrupt:
break
q.put(None)
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0):
if not infile and not interface:
# FIXME: write warning here
return
if infile:
pc = pcap.pcapObject()
try:
pc.open_offline(infile)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
if interface:
pc = pcap.pcapObject()
try:
#pc.open_live(interface, snaplen, promisc, read_timeout)
pc.open_live(interface, 1600, 0, 100)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
except Exception:
# most likely we got no permission to open the interface
sys.stderr.write('could not open interface. insufficient '
'permissions?\n')
q.put(None)
return
pc.setfilter('udp', 0, 0)
basets = 0
newMappings = dict()
while True:
if exitSignal.is_set():
break
try:
packet = pc.next()
if not packet:
if infile:
# end of file
break
elif interface:
# read timeout
continue
payload = packet[1]
timestamp = int(packet[2])
# make sure we are dealing with IP traffic
# ref: http://www.iana.org/assignments/ethernet-numbers
try:
eth = dpkt.ethernet.Ethernet(payload)
except:
continue
if eth.type != 2048: continue
# make sure we are dealing with UDP
# ref: http://www.iana.org/assignments/protocol-numbers/
try:
ip = eth.data
except:
continue
if ip.p != 17: continue
# filter on UDP assigned ports for DNS
# ref: http://www.iana.org/assignments/port-numbers
try:
udp = ip.data
except:
continue
if udp.sport != 53 and udp.dport != 53: continue
# make the dns object out of the udp data and check for it being a RR (answer)
# and for opcode QUERY (I know, counter-intuitive)
try:
dns = dpkt.dns.DNS(udp.data)
except:
continue
if dns.qr != dpkt.dns.DNS_R: continue
if dns.opcode != dpkt.dns.DNS_QUERY: continue
if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue
if len(dns.an) < 1: continue
if len(dns.qd) == 0: continue
aRecords = set()
queriedName = dns.qd[0].name
if not '.' in queriedName:
continue
#lastCname=queriedName
for answer in dns.an:
"""
FIXME: this doesn't work for multiple queries in one DNS packet
"""
#if answer.type == dpkt.dns.DNS_CNAME:
# lastCname=answer.cname
if answer.type == dpkt.dns.DNS_A:
ip = socket.inet_ntoa(answer.rdata)
try:
addr = IPAddress(ip)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and not addr.is_private()
and not addr.is_reserved()
and not addr.is_loopback()):
aRecords.add(addr)
if thrsh:
if (timestamp - basets) > thrsh:
basets = timestamp
newMappings.clear()
newIps = checkMapping(newMappings, queriedName, aRecords)
aRecords = newIps
if not aRecords:
continue
data = ((queriedName, ip.dst, aRecords), timestamp)
queued = False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime *= 0.5
elif q.qsize() >= q._maxsize:
sleeptime *= 2
if sleeptime > maxSleeptime:
sleeptime = maxSleeptime
else:
queued = True
except KeyboardInterrupt:
break
"""
send shutdown signal
"""
q.put(None)
def check_ip(addr):
"""Verify that the server/client IP is valid"""
addr = IPAddress(addr)
if addr.version != 4 or not addr.is_private():
raise ValidationError("Only private IPv4 networks are supported.")
def fifoReader(infile, q, exitSignal):
sleeptime=0.5
maxSleeptime=1.0
while True:
try:
if exitSignal.is_set(): break
line=infile.readline()
if not line:
time.sleep(1)
continue
if line=='ENDOFFILE':
break
try:
spl=line.split()
timestamp, queriedName, clientID, ipv4 = spl
except:
continue
else:
if not '.' in queriedName:
continue
try:
addr=IPAddress(ipv4)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and
not addr.is_private() and
not addr.is_reserved() and
not addr.is_loopback()):
try:
timestamp=int(timestamp)
except ValueError:
continue
else:
data = ((queriedName, clientID, [addr]),
timestamp)
queued=False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime*=0.5
elif q.qsize() >= q._maxsize:
sleeptime*=2
if sleeptime>maxSleeptime:
sleeptime=maxSleeptime
else:
queued=True
except KeyboardInterrupt:
break
q.put(None)
def _is_public(self, ip):
ip = IPAddress(ip)
return ip.is_unicast() and not ip.is_private()
def pcapReader(q, exitSignal, infile=None, interface=None, thrsh=0):
if not infile and not interface:
# FIXME: write warning here
return
if infile:
pc=pcap.pcapObject()
try:
pc.open_offline(infile)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
if interface:
pc=pcap.pcapObject()
try:
#pc.open_live(interface, snaplen, promisc, read_timeout)
pc.open_live(interface, 1600, 0, 100)
except IOError:
#log("could not open pcap interface "+str(input_interface)+"\n")
pass
except Exception:
# most likely we got no permission to open the interface
sys.stderr.write('could not open interface. insufficient '
'permissions?\n')
q.put(None)
return
pc.setfilter('udp', 0, 0)
basets=0
newMappings=dict()
while True:
if exitSignal.is_set():
break
try:
packet=pc.next()
if not packet:
if infile:
# end of file
break
elif interface:
# read timeout
continue
payload=packet[1]
timestamp=int(packet[2])
# make sure we are dealing with IP traffic
# ref: http://www.iana.org/assignments/ethernet-numbers
try: eth = dpkt.ethernet.Ethernet(payload)
except: continue
if eth.type != 2048: continue
# make sure we are dealing with UDP
# ref: http://www.iana.org/assignments/protocol-numbers/
try: ip = eth.data
except: continue
if ip.p != 17: continue
# filter on UDP assigned ports for DNS
# ref: http://www.iana.org/assignments/port-numbers
try: udp = ip.data
except: continue
if udp.sport != 53 and udp.dport != 53: continue
# make the dns object out of the udp data and check for it being a RR (answer)
# and for opcode QUERY (I know, counter-intuitive)
try: dns = dpkt.dns.DNS(udp.data)
except: continue
if dns.qr != dpkt.dns.DNS_R: continue
if dns.opcode != dpkt.dns.DNS_QUERY: continue
if dns.rcode != dpkt.dns.DNS_RCODE_NOERR: continue
if len(dns.an) < 1: continue
if len(dns.qd) == 0: continue
aRecords=set()
queriedName=dns.qd[0].name
if not '.' in queriedName:
continue
#lastCname=queriedName
for answer in dns.an:
"""
FIXME: this doesn't work for multiple queries in one DNS packet
"""
#if answer.type == dpkt.dns.DNS_CNAME:
# lastCname=answer.cname
if answer.type == dpkt.dns.DNS_A:
ip=socket.inet_ntoa(answer.rdata)
try:
addr=IPAddress(ip)
except netaddr.AddrFormatError:
continue
else:
if (addr.is_unicast() and
not addr.is_private() and
not addr.is_reserved() and
not addr.is_loopback()):
aRecords.add(addr)
if thrsh:
if (timestamp-basets) > thrsh:
basets = timestamp
newMappings.clear()
newIps = checkMapping(newMappings, queriedName, aRecords)
aRecords=newIps
if not aRecords:
continue
data = ((queriedName, ip.dst, aRecords), timestamp)
queued=False
while not queued:
try:
q.put_nowait(data)
except Queue.Full:
# we saturated the queue, let's give the reading
# process some time to empty it again, where we don't
# try to put something in the queue and thereby lock it
# continuously
time.sleep(sleeptime)
if q.empty():
sleeptime*=0.5
elif q.qsize() >= q._maxsize:
sleeptime*=2
if sleeptime>maxSleeptime:
sleeptime=maxSleeptime
else:
queued=True
except KeyboardInterrupt:
break
"""
send shutdown signal
"""
q.put(None)
class IpAddress(DatabaseObject):
""" Wraps the netaddr IPAddress class """
uuid = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid4()))
box_id = Column(Integer, ForeignKey("box.id"), nullable=False)
_address = Column(String(80))
_ip_address = None
visible = Column(Boolean, default=True)
@classmethod
def all(cls):
""" Returns a list of all objects in the database """
return dbsession.query(cls).all()
@classmethod
def by_id(cls, _id):
""" Returns a the object with id of _id """
return dbsession.query(cls).filter_by(id=_id).first()
@classmethod
def by_uuid(cls, _uuid):
""" Return and object based on a _uuid """
return dbsession.query(cls).filter_by(uuid=_uuid).first()
@classmethod
def by_address(cls, address):
""" Return and object based on an address """
return dbsession.query(cls).filter_by(_address=address).first()
@classmethod
def ipformat(self, value):
ipformat = value
if ipformat:
if ipformat.count(":") == 1:
# ip v4 with port
ipformat = ipformat.split(":")[0]
elif "]:" in ipformat:
# ip v6 with port
ipformat = ipformat.split("]:")[0]
# ip v6 enclosing
ipformat = ipformat.replace("[", "").replace("]", "")
if "/" in ipformat:
# remove any file info
ipformat = ipformat.split("/")[0]
return ipformat
@property
def address(self):
return self._address
@address.setter
def address(self, value):
try:
ip = IPAddress(self.ipformat(value))
except:
raise ValidationError("Invalid IP Address")
if ip.is_loopback():
raise ValidationError("You cannot use a loopback address")
if ip.is_multicast():
raise ValidationError("You cannot use a multicast address")
self._address = value
@property
def version(self):
if self._ip_address is None:
self._ip_address = IPAddress(self.ipformat(self._address))
return self._ip_address.version
@property
def is_private(self):
if self._ip_address is None:
self._ip_address = IPAddress(self.ipformat(self._address))
return self._ip_address.is_private()
def to_xml(self, parent):
ip_elem = ET.SubElement(parent, "ip")
ip_elem.set("version", str(self.version))
ET.SubElement(ip_elem, "address").text = self.address
def __repr__(self):
return "<IpAddress - %s>" % self.address
def __str__(self):
return self._address
def __eq__(self, other):
return self._address == other._address
def __ne__(self, other):
return not self == other
class IpAddress(DatabaseObject):
''' Wraps the netaddr IPAddress class '''
uuid = Column(String(36),
unique=True,
nullable=False,
default=lambda: str(uuid4())
)
box_id = Column(Integer, ForeignKey('box.id'), nullable=False)
_address = Column(String(80))
_ip_address = None
visable = Column(Boolean, default=True)
@classmethod
def all(cls):
''' Returns a list of all objects in the database '''
return dbsession.query(cls).all()
@classmethod
def by_id(cls, _id):
''' Returns a the object with id of _id '''
return dbsession.query(cls).filter_by(id=_id).first()
@classmethod
def by_uuid(cls, _uuid):
''' Return and object based on a _uuid '''
return dbsession.query(cls).filter_by(uuid=_uuid).first()
@classmethod
def by_address(cls, address):
''' Return and object based on an address '''
return dbsession.query(cls).filter_by(_address=address).first()
@classmethod
def ipformat(self, value):
ipformat = value
if ipformat:
if ipformat.count(":") == 1:
#ip v4 with port
ipformat = ipformat.split(":")[0]
elif "]:" in ipformat:
#ip v6 with port
ipformat = ipformat.split("]:")[0]
#ip v6 enclosing
ipformat = ipformat.replace("[", "").replace("]", "")
if "/" in ipformat:
#remove any file info
ipformat = ipformat.split("/")[0]
return ipformat
@property
def address(self):
return self._address
@address.setter
def address(self, value):
ip = IPAddress(self.ipformat(value))
if ip.is_loopback():
raise ValidationError("You cannot use a loopback address")
if ip.is_multicast():
raise ValidationError("You cannot use a multicast address")
self._address = value
@property
def version(self):
if self._ip_address is None:
self._ip_address = IPAddress(self.ipformat(self._address))
return self._ip_address.version
@property
def is_private(self):
if self._ip_address is None:
self._ip_address = IPAddress(self.ipformat(self._address))
return self._ip_address.is_private()
def to_xml(self, parent):
ip_elem = ET.SubElement(parent, "ip")
ip_elem.set("version", str(self.version))
ET.SubElement(ip_elem, "address").text = self.address
def __repr__(self):
return "<IpAddress - %s>" % self.address
def __str__(self):
return self._address
def __eq__(self, other):
return self._address == other._address
def __ne__(self, other):
return not self == other
def main(argv):
output = None
desc = """
____/ /____ _ / /_ ____ _ _____ ____ / /____ (_)/ /_
/ __ // __ `// __// __ `// ___// __ \ / // __ \ / // __/
/ /_/ // /_/ // /_ / /_/ /(__ )/ /_/ // // /_/ // // /_
\__,_/ \__,_/ \__/ \__,_//____// .___//_/ \____//_/ \__/
/_/
Open Source Assistant for #OSINT
www.datasploit.info
"""
epilog = """ Connect at Social Media: @datasploit
"""
# Set all parser arguments here.
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent(desc),
epilog=epilog)
parser.add_argument("-i",
"--input",
help="Provide Input",
dest='target',
required=True)
parser.add_argument("-a",
"--active",
help="Run Active Scan attacks",
dest='active',
action="store_false")
parser.add_argument(
"-q",
"--quiet",
help="Run scans in automated manner accepting default answers",
dest='quiet',
action="store_false")
parser.add_argument("-o",
"--output",
help="Provide Destination Directory",
dest='output')
# check and ensure the config file is present otherwise create one. required for all further operations
ds_dir = os.path.dirname(os.path.realpath(__file__))
config_file_path = os.path.join(ds_dir, "config.py")
config_sample_path = os.path.join(ds_dir, "config_sample.py")
print os.path.exists(config_file_path)
if not os.path.exists(config_file_path):
print "[+] Looks like a new setup, setting up the config file."
shutil.copyfile(config_sample_path, config_file_path)
print "[+] A config file is added please follow guide at https://datasploit.github.io/datasploit/apiGeneration/ to fill API Keys for better results"
# We can think about quiting at this point.
# if no argument is provided print help and quit
if len(argv) == 0:
parser.print_help()
sys.exit()
# parse arguments in case they are provided.
x = parser.parse_args()
active = x.active
quiet = x.quiet
user_input = x.target
output = x.output
# Banner print
print textwrap.dedent(desc)
# Auto selection logic
try:
print "User Input: %s" % user_input
try:
inp = IPAddress(user_input)
if inp.is_private() or inp.is_loopback():
print "Internal IP Detected : Skipping"
sys.exit()
else:
print "Looks like an IP, running ipOsint...\n"
ipOsint.run(user_input, output)
except SystemExit:
print "exiting"
except AddrFormatError:
if re.match('[^@]+@[^@]+\.[^@]+', user_input):
print "Looks like an EMAIL, running emailOsint...\n"
emailOsint.run(user_input, output)
elif get_tld(user_input, fix_protocol=True,
fail_silently=True) is not None:
print "Looks like a DOMAIN, running domainOsint...\n"
domainOsint.run(user_input, output)
else:
print "Nothing Matched assuming username, running usernameOsint...\n"
usernameOsint.run(user_input, output)
except:
print "Unknown Error Occured"
except KeyboardInterrupt:
print "Ctrl+C called Quiting"
def has_public_ip(self):
ip = IPAddress(self.ip)
return ip.is_unicast() and not ip.is_private()
