def recv_info(self, info): # This is where we'll collect the data we'll return. results = [] # Get the IP address or domain name. # Skip unsupported targets. if info.data_subtype == IP.data_subtype: if info.version != 4: return target = info.address parsed = netaddr.IPAddress(target) if parsed.is_loopback() or \ parsed.is_private() or \ parsed.is_link_local(): return elif info.data_subtype == Domain.data_subtype: target = info.hostname if "." not in target: return else: assert False, type(info) # Query the freegeoip.net service. # FIXME: the service supports SSL, but we need up to date certificates. Logger.log_more_verbose("Querying freegeoip.net for: " + target) resp = requests.get("http://freegeoip.net/json/" + target) if not resp.content: raise RuntimeError( "Freegeoip.net webservice is not available," " possible network error?" ) kwargs = json_decode(resp.content) # Remove the IP address from the response. address = kwargs.pop("ip") # Create a Geolocation object. geoip = Geolocation(**kwargs) geoip.add_resource(info) results.append(geoip) # Log the location. try: Logger.log_verbose("%s is in %s" % (target, geoip)) except Exception, e: fmt = traceback.format_exc() Logger.log_error("Error: %s" % str(e)) Logger.log_error_more_verbose(fmt)
class ShodanPlugin(TestingPlugin): """ This plugin tries to perform passive reconnaissance on a target using the Shodan web API. """ #-------------------------------------------------------------------------- def check_params(self): # Make sure we have an API key. self.get_api_key() #-------------------------------------------------------------------------- def get_accepted_types(self): return [IP] #-------------------------------------------------------------------------- def get_api_key(self): key = Config.plugin_args.get("apikey", None) if not key: key = Config.plugin_config.get("apikey", None) if not key: raise ValueError( "Missing API key! Get one at:" " http://www.shodanhq.com/api_doc") return key #-------------------------------------------------------------------------- def run(self, info): # This is where we'll collect the data we'll return. results = [] # Skip unsupported IP addresses. if info.version != 4: return ip = info.address parsed = netaddr.IPAddress(ip) if parsed.is_loopback() or \ parsed.is_private() or \ parsed.is_link_local(): return # Query Shodan for this host. try: key = self.get_api_key() api = WebAPI(key) shodan = api.host(ip) except Exception, e: tb = traceback.format_exc() Logger.log_error("Error querying Shodan for host %s: %s" % (ip, str(e))) Logger.log_error_more_verbose(tb) return # Make sure we got the same IP address we asked for. if ip != shodan.get("ip", ip): Logger.log_error( "Shodan gave us a different IP address... weird!") Logger.log_error_verbose( "Old IP: %s - New IP: %s" % (ip, shodan["ip"])) ip = to_utf8( shodan["ip"] ) info = IP(ip) results.append(info) # Extract all hostnames and link them to this IP address. # Note: sometimes Shodan sends IP addresses here! (?) seen_host = {} for hostname in shodan.get("hostnames", []): if hostname == ip: continue if hostname in seen_host: domain = seen_host[hostname] else: try: try: host = IP(hostname) except ValueError: host = Domain(hostname) except Exception: tb = traceback.format_exc() Logger.log_error_more_verbose(tb) seen_host[hostname] = host results.append(host) domain = host domain.add_resource(info) # Get the OS fingerprint, if available. os = to_utf8( shodan.get("os") ) if os: Logger.log("Host %s is running %s" % (ip, os)) pass # XXX TODO we'll need to reverse lookup the CPE # Get the GPS data, if available. # Complete any missing data using the default values. try: latitude = float( shodan["latitude"] ) longitude = float( shodan["longitude"] ) except Exception: latitude = None longitude = None if latitude is not None and longitude is not None: area_code = shodan.get("area_code") if not area_code: area_code = None else: area_code = str(area_code) country_code = shodan.get("country_code") if not country_code: country_code = shodan.get("country_code3") if not country_code: country_code = None else: country_code = str(country_code) else: country_code = str(country_code) country_name = shodan.get("country_name") if not country_name: country_name = None city = shodan.get("city") if not city: city = None dma_code = shodan.get("dma_code") if not dma_code: dma_code = None else: dma_code = str(dma_code) postal_code = shodan.get("postal_code") if not postal_code: postal_code = None else: postal_code = str(postal_code) region_name = shodan.get("region_name") if not region_name: region_name = None geoip = Geolocation( latitude, longitude, country_code = country_code, country_name = country_name, region_name = region_name, city = city, zipcode = postal_code, metro_code = dma_code, area_code = area_code, ) results.append(geoip) geoip.add_resource(info) # Go through every result and pick only the latest ones. latest = {} for data in shodan.get("data", []): if ( not "banner" in data or not "ip" in data or not "port" in data or not "timestamp" in data ): Logger.log_error("Malformed results from Shodan?") from pprint import pformat Logger.log_error_more_verbose(pformat(data)) continue key = ( data["ip"], data["port"], data["banner"], ) try: timestamp = reversed( # DD.MM.YYYY -> (YYYY, MM, DD) map(int, data["timestamp"].split(".", 2))) except Exception: continue if key not in latest or timestamp > latest[key][0]: latest[key] = (timestamp, data) # Process the latest results. seen_isp_or_org = set() seen_html = set() for _, data in latest.values(): # Extract all domains, but don't link them. for hostname in data.get("domains", []): if hostname not in seen_host: try: domain = Domain(hostname) except Exception: tb = traceback.format_exc() Logger.log_error_more_verbose(tb) continue seen_host[hostname] = domain results.append(domain) # We don't have any use for this information yet, # but log it so at least the user can see it. isp = to_utf8( data.get("isp") ) org = to_utf8( data.get("org") ) if org and org not in seen_isp_or_org: seen_isp_or_org.add(org) Logger.log_verbose( "Host %s belongs to: %s" % (ip, org) ) if isp and (not org or isp != org) and isp not in seen_isp_or_org: seen_isp_or_org.add(isp) Logger.log_verbose( "IP address %s is provided by ISP: %s" % (ip, isp) ) # Get the HTML content, if available. raw_html = to_utf8( data.get("html") ) if raw_html: hash_raw_html = hash(raw_html) if hash_raw_html not in seen_html: seen_html.add(hash_raw_html) try: html = HTML(raw_html) except Exception: html = None tb = traceback.format_exc() Logger.log_error_more_verbose(tb) if html: html.add_resource(info) results.append(html) # Get the banner, if available. raw_banner = to_utf8( data.get("banner") ) try: port = int( data.get("port", "0") ) except Exception: port = 0 if raw_banner and port: try: banner = Banner(info, raw_banner, port) except Exception: banner = None tb = traceback.format_exc() Logger.log_error_more_verbose(tb) if banner: results.append(banner) # Was this host located somewhere else in the past? for data in reversed(shodan.get("data", [])): try: timestamp = reversed( # DD.MM.YYYY -> (YYYY, MM, DD) map(int, data["timestamp"].split(".", 2))) old_location = data.get("location") if old_location: old_latitude = old_location.get("latitude", latitude) old_longitude = old_location.get("longitude", longitude) if ( old_latitude is not None and old_longitude is not None and (old_latitude != latitude or old_longitude != longitude) ): # Get the geoip information. area_code = old_location.get("area_code") if not area_code: area_code = None country_code = old_location.get("country_code") if not country_code: country_code = old_location.get("country_code3") if not country_code: country_code = None country_name = old_location.get("country_name") if not country_name: country_name = None city = old_location.get("city") if not city: city = None postal_code = old_location.get("postal_code") if not postal_code: postal_code = None region_name = old_location.get("region_name") if not region_name: region_name = None geoip = Geolocation( latitude, longitude, country_code = country_code, country_name = country_name, region_name = region_name, city = city, zipcode = postal_code, area_code = area_code, ) # If this is the first time we geolocate this IP, # use this information as it if were up to date. if latitude is None or longitude is None: latitude = old_latitude longitude = old_longitude results.append(geoip) geoip.add_resource(info) # Otherwise, just log the event. else: discard_data(geoip) where = str(geoip) when = datetime.date(*timestamp) msg = "Host %s used to be located at %s on %s." msg %= (ip, where, when.strftime("%B %d, %Y")) Logger.log_verbose(msg) except Exception: tb = traceback.format_exc() Logger.log_error_more_verbose(tb) # Return the results. return results
def recv_info(self, info): # This is where we'll collect the data we'll return. results = [] # Augment geolocation data obtained through other means. # (For example: image metadata) if info.is_instance(Geolocation): if not info.street_addr: street_addr = self.query_google(info.latitude, info.longitude) if street_addr: info.street_addr = street_addr # # TODO: parse the street address # Logger.log("(%s, %s) is in %s" % \ (info.latitude, info.longitude, street_addr)) return # Extract IPs and domains from traceroute results and geolocate them. if info.is_instance(Traceroute): hops = [] for hop in info.hops: if hop is not None: if hop.address: hops.append( IP(hop.address) ) elif hop.hostname: hops.append( Domain(hop.hostname) ) results.extend(hops) for res in hops: r = self.recv_info(res) if r: results.extend(r) return results # Get the IP address or domain name. # Skip unsupported targets. if info.is_instance(IP): if info.version != 4: return target = info.address parsed = netaddr.IPAddress(target) if parsed.is_loopback() or \ parsed.is_private() or \ parsed.is_link_local(): return elif info.is_instance(Domain): target = info.hostname if "." not in target: return else: assert False, type(info) # Query the freegeoip.net service. kwargs = self.query_freegeoip(target) if not kwargs: return # Remove the IP address from the response. address = kwargs.pop("ip") # Query the Google Geocoder. street_addr = self.query_google( kwargs["latitude"], kwargs["longitude"]) if street_addr: kwargs["street_addr"] = street_addr # Create a Geolocation object. geoip = Geolocation(**kwargs) geoip.add_resource(info) results.append(geoip) # Log the location. try: Logger.log_verbose("%s is in %s" % (target, geoip)) except Exception, e: fmt = traceback.format_exc() Logger.log_error("Error: %s" % str(e)) Logger.log_error_more_verbose(fmt)
def run(self, info): # This is where we'll collect the data we'll return. results = [] # Augment geolocation data obtained through other means. # (For example: image metadata) if info.is_instance(Geolocation): if not info.street_addr: street_addr = self.query_google(info.latitude, info.longitude) if street_addr: info.street_addr = street_addr # # TODO: parse the street address # Logger.log("Location (%s, %s) is in %s" % \ (info.latitude, info.longitude, street_addr)) return # Extract IPs from traceroute results and geolocate them. if info.is_instance(Traceroute): addr_to_ip = {} for hop in info.hops: if hop is not None: if hop.address and hop.address not in addr_to_ip: addr_to_ip[hop.address] = IP(hop.address) results.extend(addr_to_ip.itervalues()) coords_to_geoip = {} for res in addr_to_ip.itervalues(): r = self.run(res) if r: for x in r: if not x.is_instance(Geolocation): results.append(x) else: key = (x.latitude, x.longitude) if key not in coords_to_geoip: coords_to_geoip[key] = x results.append(x) else: coords_to_geoip[key].merge(x) return results # Geolocate IP addresses using Freegeoip. if info.is_instance(IP): # Skip unsupported targets. if info.version != 4: return ip = info.address parsed = netaddr.IPAddress(ip) if parsed.is_loopback() or \ parsed.is_private() or \ parsed.is_link_local(): return # Query the freegeoip.net service. kwargs = self.query_freegeoip(ip) if not kwargs: return # Translate the arguments for Geolocation(). kwargs.pop("ip") # Geolocate BSSIDs using Skyhook. elif info.is_instance(BSSID) or info.is_instance(MAC): skyhook = self.query_skyhook(info.address) if not skyhook: return # Translate the arguments for Geolocation(). kwargs = { "latitude": skyhook["latitude"], "longitude": skyhook["longitude"], "accuracy": skyhook["hpe"], "country_name": skyhook["country"], "country_code": skyhook["country_code"], "region_code": skyhook["state_code"], "region_name": skyhook["state"], } # Fail for other data types. else: assert False, "Internal error! Unexpected type: %r" % type(info) # Query the Google Geocoder to get the street address. street_addr = self.query_google(kwargs["latitude"], kwargs["longitude"]) if street_addr: kwargs["street_addr"] = street_addr # Create a Geolocation object. geoip = Geolocation(**kwargs) geoip.add_resource(info) results.append(geoip) # Log the location. try: Logger.log_verbose("%s %s is located in %s" % (info.display_name, info.address, geoip)) except Exception, e: fmt = traceback.format_exc() Logger.log_error("Error: %s" % str(e)) Logger.log_error_more_verbose(fmt)
def run(self, info): # This is where we'll collect the data we'll return. results = [] # Augment geolocation data obtained through other means. # (For example: image metadata) if info.is_instance(Geolocation): if not info.street_addr: street_addr = self.query_google(info.latitude, info.longitude) if street_addr: info.street_addr = street_addr # # TODO: parse the street address # Logger.log("Location (%s, %s) is in %s" % \ (info.latitude, info.longitude, street_addr)) return # Extract IPs from traceroute results and geolocate them. if info.is_instance(Traceroute): addr_to_ip = {} for hop in info.hops: if hop is not None: if hop.address and hop.address not in addr_to_ip: addr_to_ip[hop.address] = IP(hop.address) results.extend( addr_to_ip.itervalues() ) coords_to_geoip = {} for res in addr_to_ip.itervalues(): r = self.run(res) if r: for x in r: if not x.is_instance(Geolocation): results.append(x) else: key = (x.latitude, x.longitude) if key not in coords_to_geoip: coords_to_geoip[key] = x results.append(x) else: coords_to_geoip[key].merge(x) return results # Geolocate IP addresses using Freegeoip. if info.is_instance(IP): # Skip unsupported targets. if info.version != 4: return ip = info.address parsed = netaddr.IPAddress(ip) if parsed.is_loopback() or \ parsed.is_private() or \ parsed.is_link_local(): return # Query the freegeoip.net service. kwargs = self.query_freegeoip(ip) if not kwargs: return # Translate the arguments for Geolocation(). kwargs.pop("ip") # Geolocate BSSIDs using Skyhook. elif info.is_instance(BSSID) or info.is_instance(MAC): skyhook = self.query_skyhook(info.address) if not skyhook: return # Translate the arguments for Geolocation(). kwargs = { "latitude": skyhook["latitude"], "longitude": skyhook["longitude"], "accuracy": skyhook["hpe"], "country_name": skyhook["country"], "country_code": skyhook["country_code"], "region_code": skyhook["state_code"], "region_name": skyhook["state"], } # Fail for other data types. else: assert False, "Internal error! Unexpected type: %r" % type(info) # Query the Google Geocoder to get the street address. street_addr = self.query_google( kwargs["latitude"], kwargs["longitude"]) if street_addr: kwargs["street_addr"] = street_addr # Create a Geolocation object. geoip = Geolocation(**kwargs) geoip.add_resource(info) results.append(geoip) # Log the location. try: Logger.log_verbose( "%s %s is located in %s" % (info.display_name, info.address, geoip)) except Exception, e: fmt = traceback.format_exc() Logger.log_error("Error: %s" % str(e)) Logger.log_error_more_verbose(fmt)