def recv_info(self, info):
# This is where we'll collect the data we'll return.
results = []
# Get the IP address or domain name.
# Skip unsupported targets.
if info.data_subtype == IP.data_subtype:
if info.version != 4:
return
target = info.address
parsed = netaddr.IPAddress(target)
if parsed.is_loopback() or \
parsed.is_private() or \
parsed.is_link_local():
return
elif info.data_subtype == Domain.data_subtype:
target = info.hostname
if "." not in target:
return
else:
assert False, type(info)
# Query the freegeoip.net service.
# FIXME: the service supports SSL, but we need up to date certificates.
Logger.log_more_verbose("Querying freegeoip.net for: " + target)
resp = requests.get("http://freegeoip.net/json/" + target)
if not resp.content:
raise RuntimeError(
"Freegeoip.net webservice is not available,"
" possible network error?"
)
kwargs = json_decode(resp.content)
# Remove the IP address from the response.
address = kwargs.pop("ip")
# Create a Geolocation object.
geoip = Geolocation(**kwargs)
geoip.add_resource(info)
results.append(geoip)
# Log the location.
try:
Logger.log_verbose("%s is in %s" % (target, geoip))
except Exception, e:
fmt = traceback.format_exc()
Logger.log_error("Error: %s" % str(e))
Logger.log_error_more_verbose(fmt)
class ShodanPlugin(TestingPlugin):
"""
This plugin tries to perform passive reconnaissance on a target using
the Shodan web API.
"""
#--------------------------------------------------------------------------
def check_params(self):
# Make sure we have an API key.
self.get_api_key()
#--------------------------------------------------------------------------
def get_accepted_types(self):
return [IP]
#--------------------------------------------------------------------------
def get_api_key(self):
key = Config.plugin_args.get("apikey", None)
if not key:
key = Config.plugin_config.get("apikey", None)
if not key:
raise ValueError(
"Missing API key! Get one at:"
" http://www.shodanhq.com/api_doc")
return key
#--------------------------------------------------------------------------
def run(self, info):
# This is where we'll collect the data we'll return.
results = []
# Skip unsupported IP addresses.
if info.version != 4:
return
ip = info.address
parsed = netaddr.IPAddress(ip)
if parsed.is_loopback() or \
parsed.is_private() or \
parsed.is_link_local():
return
# Query Shodan for this host.
try:
key = self.get_api_key()
api = WebAPI(key)
shodan = api.host(ip)
except Exception, e:
tb = traceback.format_exc()
Logger.log_error("Error querying Shodan for host %s: %s" % (ip, str(e)))
Logger.log_error_more_verbose(tb)
return
# Make sure we got the same IP address we asked for.
if ip != shodan.get("ip", ip):
Logger.log_error(
"Shodan gave us a different IP address... weird!")
Logger.log_error_verbose(
"Old IP: %s - New IP: %s" % (ip, shodan["ip"]))
ip = to_utf8( shodan["ip"] )
info = IP(ip)
results.append(info)
# Extract all hostnames and link them to this IP address.
# Note: sometimes Shodan sends IP addresses here! (?)
seen_host = {}
for hostname in shodan.get("hostnames", []):
if hostname == ip:
continue
if hostname in seen_host:
domain = seen_host[hostname]
else:
try:
try:
host = IP(hostname)
except ValueError:
host = Domain(hostname)
except Exception:
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
seen_host[hostname] = host
results.append(host)
domain = host
domain.add_resource(info)
# Get the OS fingerprint, if available.
os = to_utf8( shodan.get("os") )
if os:
Logger.log("Host %s is running %s" % (ip, os))
pass # XXX TODO we'll need to reverse lookup the CPE
# Get the GPS data, if available.
# Complete any missing data using the default values.
try:
latitude = float( shodan["latitude"] )
longitude = float( shodan["longitude"] )
except Exception:
latitude = None
longitude = None
if latitude is not None and longitude is not None:
area_code = shodan.get("area_code")
if not area_code:
area_code = None
else:
area_code = str(area_code)
country_code = shodan.get("country_code")
if not country_code:
country_code = shodan.get("country_code3")
if not country_code:
country_code = None
else:
country_code = str(country_code)
else:
country_code = str(country_code)
country_name = shodan.get("country_name")
if not country_name:
country_name = None
city = shodan.get("city")
if not city:
city = None
dma_code = shodan.get("dma_code")
if not dma_code:
dma_code = None
else:
dma_code = str(dma_code)
postal_code = shodan.get("postal_code")
if not postal_code:
postal_code = None
else:
postal_code = str(postal_code)
region_name = shodan.get("region_name")
if not region_name:
region_name = None
geoip = Geolocation(
latitude, longitude,
country_code = country_code,
country_name = country_name,
region_name = region_name,
city = city,
zipcode = postal_code,
metro_code = dma_code,
area_code = area_code,
)
results.append(geoip)
geoip.add_resource(info)
# Go through every result and pick only the latest ones.
latest = {}
for data in shodan.get("data", []):
if (
not "banner" in data or
not "ip" in data or
not "port" in data or
not "timestamp" in data
):
Logger.log_error("Malformed results from Shodan?")
from pprint import pformat
Logger.log_error_more_verbose(pformat(data))
continue
key = (
data["ip"],
data["port"],
data["banner"],
)
try:
timestamp = reversed( # DD.MM.YYYY -> (YYYY, MM, DD)
map(int, data["timestamp"].split(".", 2)))
except Exception:
continue
if key not in latest or timestamp > latest[key][0]:
latest[key] = (timestamp, data)
# Process the latest results.
seen_isp_or_org = set()
seen_html = set()
for _, data in latest.values():
# Extract all domains, but don't link them.
for hostname in data.get("domains", []):
if hostname not in seen_host:
try:
domain = Domain(hostname)
except Exception:
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
continue
seen_host[hostname] = domain
results.append(domain)
# We don't have any use for this information yet,
# but log it so at least the user can see it.
isp = to_utf8( data.get("isp") )
org = to_utf8( data.get("org") )
if org and org not in seen_isp_or_org:
seen_isp_or_org.add(org)
Logger.log_verbose(
"Host %s belongs to: %s"
% (ip, org)
)
if isp and (not org or isp != org) and isp not in seen_isp_or_org:
seen_isp_or_org.add(isp)
Logger.log_verbose(
"IP address %s is provided by ISP: %s"
% (ip, isp)
)
# Get the HTML content, if available.
raw_html = to_utf8( data.get("html") )
if raw_html:
hash_raw_html = hash(raw_html)
if hash_raw_html not in seen_html:
seen_html.add(hash_raw_html)
try:
html = HTML(raw_html)
except Exception:
html = None
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
if html:
html.add_resource(info)
results.append(html)
# Get the banner, if available.
raw_banner = to_utf8( data.get("banner") )
try:
port = int( data.get("port", "0") )
except Exception:
port = 0
if raw_banner and port:
try:
banner = Banner(info, raw_banner, port)
except Exception:
banner = None
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
if banner:
results.append(banner)
# Was this host located somewhere else in the past?
for data in reversed(shodan.get("data", [])):
try:
timestamp = reversed( # DD.MM.YYYY -> (YYYY, MM, DD)
map(int, data["timestamp"].split(".", 2)))
old_location = data.get("location")
if old_location:
old_latitude = old_location.get("latitude", latitude)
old_longitude = old_location.get("longitude", longitude)
if (
old_latitude is not None and
old_longitude is not None and
(old_latitude != latitude or old_longitude != longitude)
):
# Get the geoip information.
area_code = old_location.get("area_code")
if not area_code:
area_code = None
country_code = old_location.get("country_code")
if not country_code:
country_code = old_location.get("country_code3")
if not country_code:
country_code = None
country_name = old_location.get("country_name")
if not country_name:
country_name = None
city = old_location.get("city")
if not city:
city = None
postal_code = old_location.get("postal_code")
if not postal_code:
postal_code = None
region_name = old_location.get("region_name")
if not region_name:
region_name = None
geoip = Geolocation(
latitude, longitude,
country_code = country_code,
country_name = country_name,
region_name = region_name,
city = city,
zipcode = postal_code,
area_code = area_code,
)
# If this is the first time we geolocate this IP,
# use this information as it if were up to date.
if latitude is None or longitude is None:
latitude = old_latitude
longitude = old_longitude
results.append(geoip)
geoip.add_resource(info)
# Otherwise, just log the event.
else:
discard_data(geoip)
where = str(geoip)
when = datetime.date(*timestamp)
msg = "Host %s used to be located at %s on %s."
msg %= (ip, where, when.strftime("%B %d, %Y"))
Logger.log_verbose(msg)
except Exception:
tb = traceback.format_exc()
Logger.log_error_more_verbose(tb)
# Return the results.
return results
def recv_info(self, info):
# This is where we'll collect the data we'll return.
results = []
# Augment geolocation data obtained through other means.
# (For example: image metadata)
if info.is_instance(Geolocation):
if not info.street_addr:
street_addr = self.query_google(info.latitude, info.longitude)
if street_addr:
info.street_addr = street_addr
#
# TODO: parse the street address
#
Logger.log("(%s, %s) is in %s" % \
(info.latitude, info.longitude, street_addr))
return
# Extract IPs and domains from traceroute results and geolocate them.
if info.is_instance(Traceroute):
hops = []
for hop in info.hops:
if hop is not None:
if hop.address:
hops.append( IP(hop.address) )
elif hop.hostname:
hops.append( Domain(hop.hostname) )
results.extend(hops)
for res in hops:
r = self.recv_info(res)
if r:
results.extend(r)
return results
# Get the IP address or domain name.
# Skip unsupported targets.
if info.is_instance(IP):
if info.version != 4:
return
target = info.address
parsed = netaddr.IPAddress(target)
if parsed.is_loopback() or \
parsed.is_private() or \
parsed.is_link_local():
return
elif info.is_instance(Domain):
target = info.hostname
if "." not in target:
return
else:
assert False, type(info)
# Query the freegeoip.net service.
kwargs = self.query_freegeoip(target)
if not kwargs:
return
# Remove the IP address from the response.
address = kwargs.pop("ip")
# Query the Google Geocoder.
street_addr = self.query_google(
kwargs["latitude"], kwargs["longitude"])
if street_addr:
kwargs["street_addr"] = street_addr
# Create a Geolocation object.
geoip = Geolocation(**kwargs)
geoip.add_resource(info)
results.append(geoip)
# Log the location.
try:
Logger.log_verbose("%s is in %s" % (target, geoip))
except Exception, e:
fmt = traceback.format_exc()
Logger.log_error("Error: %s" % str(e))
Logger.log_error_more_verbose(fmt)
def run(self, info):
# This is where we'll collect the data we'll return.
results = []
# Augment geolocation data obtained through other means.
# (For example: image metadata)
if info.is_instance(Geolocation):
if not info.street_addr:
street_addr = self.query_google(info.latitude, info.longitude)
if street_addr:
info.street_addr = street_addr
#
# TODO: parse the street address
#
Logger.log("Location (%s, %s) is in %s" % \
(info.latitude, info.longitude, street_addr))
return
# Extract IPs from traceroute results and geolocate them.
if info.is_instance(Traceroute):
addr_to_ip = {}
for hop in info.hops:
if hop is not None:
if hop.address and hop.address not in addr_to_ip:
addr_to_ip[hop.address] = IP(hop.address)
results.extend(addr_to_ip.itervalues())
coords_to_geoip = {}
for res in addr_to_ip.itervalues():
r = self.run(res)
if r:
for x in r:
if not x.is_instance(Geolocation):
results.append(x)
else:
key = (x.latitude, x.longitude)
if key not in coords_to_geoip:
coords_to_geoip[key] = x
results.append(x)
else:
coords_to_geoip[key].merge(x)
return results
# Geolocate IP addresses using Freegeoip.
if info.is_instance(IP):
# Skip unsupported targets.
if info.version != 4:
return
ip = info.address
parsed = netaddr.IPAddress(ip)
if parsed.is_loopback() or \
parsed.is_private() or \
parsed.is_link_local():
return
# Query the freegeoip.net service.
kwargs = self.query_freegeoip(ip)
if not kwargs:
return
# Translate the arguments for Geolocation().
kwargs.pop("ip")
# Geolocate BSSIDs using Skyhook.
elif info.is_instance(BSSID) or info.is_instance(MAC):
skyhook = self.query_skyhook(info.address)
if not skyhook:
return
# Translate the arguments for Geolocation().
kwargs = {
"latitude": skyhook["latitude"],
"longitude": skyhook["longitude"],
"accuracy": skyhook["hpe"],
"country_name": skyhook["country"],
"country_code": skyhook["country_code"],
"region_code": skyhook["state_code"],
"region_name": skyhook["state"],
}
# Fail for other data types.
else:
assert False, "Internal error! Unexpected type: %r" % type(info)
# Query the Google Geocoder to get the street address.
street_addr = self.query_google(kwargs["latitude"],
kwargs["longitude"])
if street_addr:
kwargs["street_addr"] = street_addr
# Create a Geolocation object.
geoip = Geolocation(**kwargs)
geoip.add_resource(info)
results.append(geoip)
# Log the location.
try:
Logger.log_verbose("%s %s is located in %s" %
(info.display_name, info.address, geoip))
except Exception, e:
fmt = traceback.format_exc()
Logger.log_error("Error: %s" % str(e))
Logger.log_error_more_verbose(fmt)
def run(self, info):
# This is where we'll collect the data we'll return.
results = []
# Augment geolocation data obtained through other means.
# (For example: image metadata)
if info.is_instance(Geolocation):
if not info.street_addr:
street_addr = self.query_google(info.latitude, info.longitude)
if street_addr:
info.street_addr = street_addr
#
# TODO: parse the street address
#
Logger.log("Location (%s, %s) is in %s" % \
(info.latitude, info.longitude, street_addr))
return
# Extract IPs from traceroute results and geolocate them.
if info.is_instance(Traceroute):
addr_to_ip = {}
for hop in info.hops:
if hop is not None:
if hop.address and hop.address not in addr_to_ip:
addr_to_ip[hop.address] = IP(hop.address)
results.extend( addr_to_ip.itervalues() )
coords_to_geoip = {}
for res in addr_to_ip.itervalues():
r = self.run(res)
if r:
for x in r:
if not x.is_instance(Geolocation):
results.append(x)
else:
key = (x.latitude, x.longitude)
if key not in coords_to_geoip:
coords_to_geoip[key] = x
results.append(x)
else:
coords_to_geoip[key].merge(x)
return results
# Geolocate IP addresses using Freegeoip.
if info.is_instance(IP):
# Skip unsupported targets.
if info.version != 4:
return
ip = info.address
parsed = netaddr.IPAddress(ip)
if parsed.is_loopback() or \
parsed.is_private() or \
parsed.is_link_local():
return
# Query the freegeoip.net service.
kwargs = self.query_freegeoip(ip)
if not kwargs:
return
# Translate the arguments for Geolocation().
kwargs.pop("ip")
# Geolocate BSSIDs using Skyhook.
elif info.is_instance(BSSID) or info.is_instance(MAC):
skyhook = self.query_skyhook(info.address)
if not skyhook:
return
# Translate the arguments for Geolocation().
kwargs = {
"latitude": skyhook["latitude"],
"longitude": skyhook["longitude"],
"accuracy": skyhook["hpe"],
"country_name": skyhook["country"],
"country_code": skyhook["country_code"],
"region_code": skyhook["state_code"],
"region_name": skyhook["state"],
}
# Fail for other data types.
else:
assert False, "Internal error! Unexpected type: %r" % type(info)
# Query the Google Geocoder to get the street address.
street_addr = self.query_google(
kwargs["latitude"], kwargs["longitude"])
if street_addr:
kwargs["street_addr"] = street_addr
# Create a Geolocation object.
geoip = Geolocation(**kwargs)
geoip.add_resource(info)
results.append(geoip)
# Log the location.
try:
Logger.log_verbose(
"%s %s is located in %s"
% (info.display_name, info.address, geoip))
except Exception, e:
fmt = traceback.format_exc()
Logger.log_error("Error: %s" % str(e))
Logger.log_error_more_verbose(fmt)
