def set_mirror_country( mirror_info: Dict[AnyStr, Union[Dict, AnyStr]], ) -> None: """ Set country by IP of a mirror :param mirror_info: Dict with info about a mirror """ mirror_name = mirror_info['name'] try: ip = socket.gethostbyname(mirror_name) except socket.gaierror: logger.error('Can\'t get IP of mirror %s', mirror_name) mirror_info['country'] = 'Unknown' return db = Reader(GEOPIP_DB) logger.info('Set country for mirror "%s"', mirror_name) try: match = db.city(ip) # type: City mirror_info['country'] = match.country.name except AddressNotFoundError: logger.warning( 'GeoIP db does not have information about IP "%s"', ip, ) mirror_info['country'] = 'Unknown'
def geoip_location(ip=None, timeout=DEFAULT_TIMEOUT): geoip_file = os.environ.get('GEOIP2_FILE') if not geoip_file or not os.path.exists(geoip_file): raise LocationServiceException( 'no valid GEOIP2_FILE set -- please download and decompress: ' 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz' # noqa ) if not ip: try: xff = requests.get('http://jsonip.com/').json()['ip'] ip = xff.split(', ')[-1] except Exception: raise LocationServiceException( 'error getting an ip -- are you online?' ) reader = Reader(geoip_file) rec = reader.omni(ip) if not rec.location: raise LocationServiceException( 'unable to geocode ip address ({0})'.format(ip) ) loc = rec.location return Location(loc.latitude, loc.longitude, 'geoip')
def get_location(cls, ip=None): if not ip: ip = cls.get_ip() try: city_client = Reader("./utils/geoip2/GeoLite2-City.mmdb") response = city_client.city(ip) if response.city.name: location = ",".join( [response.country.iso_code, response.city.name]) else: location = ",".join([response.country.iso_code, ""]) except Exception as e: print("fail to get location with geoip2: %s" % str(e)) try: api_key = 'at_IW99hSbVb4uxQq1SbaoIanDbulTbU' api_url = 'https://geo.ipify.org/api/v1?' url = api_url + 'apiKey=' + api_key + '&ipAddress=' + ip temp_region = loads( urlopen(url).read().decode('utf8'))["location"] try: location = ",".join( [temp_region["country"], temp_region["city"]]) except [KeyError, ValueError]: location = temp_region except URLError: location = "network error" return location
def get_geo_info(self): geo_asn = Reader(join(self.path, self.geo[0])).asn(str(self.addr)) geo_city = Reader(join(self.path, self.geo[1])).city(str(self.addr)) return (geo_asn.autonomous_system_number, geo_asn.autonomous_system_organization, geo_city.country.name, geo_city.city.name, geo_city.location.latitude, geo_city.location.longitude)
def main(): args = docopt(__doc__) out = csv.writer(sys.stdout) reader = Reader(args['-f']) try: ips = [x.strip() for x in open(args['IPFILE'], 'r')] except FileNotFoundError: if not args['-q']: print("IP file {} count not be opened, interpreting as single IP!".format(args['IPFILE'])) ips = [args['IPFILE'].strip()] for ip in ips: try: resp = reader.city(ip) except ValueError: if not args['-q']: print("{} is not an IP address, skipping!".format(ip), file=sys.stderr) continue row = [] if args['-i']: row.append(ip) if args['-c']: row.append(resp.country.names.get('en', '-')) row.append(resp.country.iso_code) try: row.append(resp.city.names['en']) except KeyError: row.append('-') out.writerow(row)
def __init__(self, database): """ Constructor :param database: Location of the enterprise-database """ self._reader = Reader(database) # Close reader when app closes down atexit.register(lambda: self._reader.close())
def __init__(self, path_to_db): from geoip2.database import Reader try: self.reader = Reader(path_to_db) except: pass self.cache = {}
def get_location_for_ip(ip_addr): try: reader = Reader( os.path.join(GEOLOCATION_DATABASE_PATH, GEOLOCATION_DATABASE_CITY)) return reader.city(ip_addr) except Exception as e: logger.warn("Failed retrieving geoip info for %s: %s" % (ip_addr, str(e))) return None
def get_asn_for_ip(ip_addr): try: reader = Reader( os.path.join(GEOLOCATION_DATABASE_PATH, GEOLOCATION_DATABASE_ASN)) return reader.asn(ip_addr) except Exception as e: logger.warn("Failed retrieving asn info for %s: %s" % (ip_addr, str(e))) return None
def __open(self): """Get GeoIP Reader, if configured, otherwise none""" path = CONFIG.y("geoip") if path == "" or not path: return try: self.__reader = Reader(path) self.__last_mtime = stat(path).st_mtime LOGGER.info("Loaded GeoIP database", last_write=self.__last_mtime) except OSError as exc: LOGGER.warning("Failed to load GeoIP database", exc=exc)
def getisp(ip): loc_info = {} r = Reader(join(abspath(dirname(__file__)), 'GeoIP2-ISP.mmdb')) obj = r.isp(ip) print obj.raw loc_info['isp'] = { 'organization': obj.raw.get('autonomous_system_organization', ''), 'isp': obj.raw.get('isp', ''), } print loc_info r.close()
def fetch_geoip(ip_address, language=None): # Prepare response object response = {} # Get hostname from IP address, pass if fail try: response['ip_address'] = ip_address response['hostname'] = socket.gethostbyaddr(ip_address)[0] except Exception as ex: pass # Load GeoLite2 City database geoip = Reader(path.join(config.MMDB_PATH, "GeoLite2-City.mmdb")) # Try to fetch data and build response, otherwise raise exception try: data = geoip.city(ip_address) # geoip.city response['city'] = { "name": data.city.name, "id": data.city.geoname_id, "region": data.subdivisions.most_specific.name, "region_code": data.subdivisions.most_specific.iso_code } # geoip.country response['country'] = { "name": data.country.name, "iso_code": data.country.iso_code, "continent": data.continent.name, "continent_code": data.continent.code, "is_eu": data.country.is_in_european_union } # geoip.location response['location'] = { "accuracy_radius": data.location.accuracy_radius, "zip": data.postal.code, "latitude": data.location.latitude, "longitude": data.location.longitude, "timezone": data.location.time_zone } except AddressNotFoundError as ex: raise ex # Close database instances geoip.close() # Load GeoLite2 ASN database (optional) response['asn'] = fetch_asn(ip_address) # Return built response object return response
def find_location(ip): reader = Reader(TannerConfig.get('DATA', 'geo_db')) try: location = reader.city(ip) info = dict( country=location.country.name, country_code=location.country.iso_code, city=location.city.name, zip_code=location.postal.code, ) except geoip2.errors.AddressNotFoundError: info = "NA" # When IP doesn't exist in the db, set info as "NA - Not Available" return info
def get_country(ip): from geoip2.database import Reader from geoip2.errors import AddressNotFoundError try: # Насколько я понимаю spark неспособен создать broadcast верси database.Reader # объект, поэтому приходится его создавать при каждом вызове функции # хотя возможно, производится какая-то оптимизация внутри udf() reader = Reader(giLite2db_path) response = reader.country(ip) country = response.country.name return country if country else "NOT_FOUND" except AddressNotFoundError: return "NOT_FOUND"
class GeoIP: def __init__(self, db='/home/insane/PycharmProjects/octopus/GeoLite2-City.mmdb'): self.r = Reader(db) def get_continent_code(self, ip): return self.r.city(ip).continent def get_longitude(self, ip): return self.r.city(ip).location.longitude def get_latitude(self, ip): return self.r.city(ip).location.latitude def get_iso_code(self, ip): return self.r.city(ip).country.iso_code
def ip2city_py(ip): global reader if reader is None: # assume all work nodes have mmdb installed in the following path reader = Reader( "/home/spark/spark-2.4.5-bin-hadoop2.7/maxmind/GeoLite2-City.mmdb" ) try: response = reader.city(ip) city = response.city.name if city is None: return None return city except: return None
def reader_manager(self, action=None): if action == 'open': try: self.reader = Reader(self.dbfile) except FileNotFoundError: self.logger.error("Could not find MaxMind DB! Downloading!") result_status = self.download() if result_status: self.logger.error( "Could not download MaxMind DB! You may need to manually install it." ) exit(1) else: self.reader = Reader(self.dbfile) else: self.reader.close()
def ip2city_py(ip): global reader if reader is None: # assume all work nodes have mmdb installed in the following path reader = Reader( "/home/cloudera/Desktop/spark_sql_101/maxmind/GeoLite2-City.mmdb" ) try: response = reader.city(ip) city = response.city.name if city is None: return None return city except: return None
def __spawn_maxmind(cls, configuration=None, **ignored): if cls.MAXMIND is None: if configuration is None: from ta_common.taste_config_helper import TasteConf configuration = TasteConf() cls.MAXMIND = Reader(configuration.getMaxMindSource()) return cls.MAXMIND
def process_test_data( raw_data: list, previous_results: dict, country_database_path: str) -> Dict[str, Dict[str, object]]: """Process the raw data of the test.""" result = json.loads(raw_data['general']['data'].decode()) # geoip reader = Reader(country_database_path) result['a_locations'] = _get_countries(result['a_records'], reader) result['mx_locations'] = _get_countries( (ip for mx_a_records in result['mx_a_records'] for ip in mx_a_records[1]), reader) # TODO: reverse mx-a matches mx result['final_url_is_https'] = ('final_url' in result and result['final_url'].startswith('https')) # handle non-https final url if (not result['final_url_is_https'] and 'final_url_content' in raw_data and 'final_https_url_content' in raw_data): similarity = _jaccard_index( raw_data['final_url_content']['data'], raw_data['final_https_url_content']['data']) result['same_content_via_https'] = similarity > MINIMUM_SIMILARITY return result
def load_geoip_database(): global geoip_reader try: geoip_reader = Reader(settings.GEOIP2_DATABASE) except IOError: log.warning('Geolocation is disabled: Cannot load database.')
def load_geoip_database(): global geoip_reader try: geoip_reader = Reader(settings.GEOIP2_DATABASE) except IOError: logger.warning('Geolocation is disabled: Cannot load database.', extra={'code': WARNING_CANNOT_LOAD_DATABASE})
def statics_city(): """统计区域数据""" city = {} geoip2_reader = Reader("GeoLite2-City.mmdb") for _ip, _cnt in statics_all()["vistors"].items(): _city_name = "Unknow" try: _city = geoip2_reader.city(_ip) # _city_name = "{}/{}".format(_city.country.names.get("en","").encode("utf8"),_city.city.names.get("en","").encode("utf8")) _city_name = "{}/{}".format(_city.country.names.get("zh-CN", ""), _city.city.names.get("zh-CN", "")) except BaseException as e: print(e) pass city.setdefault(_city_name, 0) city[_city_name] += _cnt return city
def threaded_localize(self, many_ip, geolite_file): """ Triggers call to localize in a multithreaded fashion """ geo = Reader(geolite_file) with ThreadPool(processes=self.threads) as p: res = [ p.apply_async(self.localize, (ip, geo)) for ip in many_ip ] return [ r.get() for r in res ]
def _insert_geoip(result, logger, options): # GeoIP reader = Reader(_get_geoip_database(options)) result['a_locations'], result['a_locations_unknown'] = _get_countries( result['a_records'], reader) result['mx_locations'], result['mx_locations_unknown'] = _get_countries( (ip for mx_a_records in result['mx_a_records'] for ip in mx_a_records[1]), reader)
def geolocate(url): """ Finds data about url in geolocation database and transfers it mongodb :param url: url to lacate :return: geolocation_id """ from geoip2.database import Reader from geoip2.webservice import Client from geoip2.errors import GeoIP2Error, HTTPError geolocation_data = dict() try: ip = url_to_ip(url) response = None if config.GEOIP2_WEB_SERVICE_USER_ID and config.GEOIP2_WEB_SERVICE_LICENSE_KEY \ and config.GEOIP2_WEB_SERVICE_TYPE: client = Client(config.GEOIP2_WEB_SERVICE_USER_ID, config.GEOIP2_WEB_SERVICE_LICENSE_KEY) if config.GEOIP2_WEB_SERVICE_TYPE == 'country': response = client.country(ip).country elif config.GEOIP2_WEB_SERVICE_TYPE == 'city': response = client.city(ip).city elif config.GEOIP2_WEB_SERVICE_TYPE == 'insights': response = client.insights(ip).insights elif config.GEOIP2_DATABASE_PATH and config.GEOIP2_DATABASE_TYPE: reader = Reader(config.GEOIP2_DATABASE_PATH) if config.GEOIP2_DATABASE_TYPE == 'country': response = reader.country(ip).country if config.GEOIP2_DATABASE_TYPE == 'city': response = reader.city(ip).city else: reader = Reader( '/opt/project/worker/utils/geoloc_databases/GeoLite2-City.mmdb' ) response = reader.city(ip) for name in dir(response): value = getattr(response, name) if not name.startswith('_') and not type(value) == dict: geolocation_data[name] = value.__dict__ except (GeoIP2Error, HTTPError) as error: geolocation_data = {'_error': str(error)} finally: duplicate = db.geolocation.find_one(geolocation_data) if duplicate: return duplicate['_id'] geolocation_id = db.geolocation.insert_one(geolocation_data) return str(geolocation_id.inserted_id)
def __init__( self, geoip_db_path='/config/geoip2db/GeoLite2-City.mmdb', log_path=DEFAULT_LOG_PATH, influxdb_host=DEFAULT_INFLUX_HOST, influxdb_port=DEFAULT_INFLUX_HOST_PORT, influxdb_database=DEFAULT_INFLUX_DATABASE, influxdb_user=DEFAULT_INFLUX_USER, influxdb_user_pass=DEFAULT_INFLUX_PASS, influxdb_retention=DEFAULT_INFLUX_RETENTION, influxdb_shard=DEFAULT_INFLUX_SHARD, geo_measurement=DEFAULT_GEO_MEASUREMENT, log_measurement=DEFAULT_LOG_MEASUREMENT, send_nginx_logs=DEFAULT_SEND_NGINX_LOGS, ): self.geoip_db_path = geoip_db_path self.log_path = log_path self.influxdb_host = influxdb_host self.influxdb_port = influxdb_port self.influxdb_database = influxdb_database self.influxdb_user = influxdb_user self.influxdb_user_pass = influxdb_user_pass self.influxdb_retention = influxdb_retention self.influxdb_shard = influxdb_shard self.geo_measurement = geo_measurement self.log_measurement = log_measurement self.send_nginx_logs = send_nginx_logs self.geoip_db_path = geoip_db_path logging.debug('Log parser config:' + f'\n geoip_db_path :: {self.geoip_db_path}' + f'\n log_path :: {self.log_path}' + f'\n influxdb_host :: {self.influxdb_host}' + f'\n influxdb_port :: {self.influxdb_port}' + f'\n influxdb_database :: {self.influxdb_database}' + f'\n influxdb_retention :: {self.influxdb_retention}' + f'\n influxdb_shard :: {self.influxdb_shard}' + f'\n influxdb_user :: {self.influxdb_user}' + f'\n influxdb_user_pass :: {self.influxdb_user_pass}' + f'\n geo_measurement :: {self.geo_measurement}' + f'\n log_measurement :: {self.log_measurement}' + f'\n send_nginx_logs :: {self.send_nginx_logs}') self.influxdb = self.init_influxdb() self.geoip = Reader(self.geoip_db_path) self.hostname = uname()[1]
def obtener_pais(ip): try: country_database = Reader("GeoLite2-Country.mmdb") except: country_database = None if ip is None or country_database is None: return "Unknown" if ip.startswith("10.") or ip.startswith("192.168."): return "Local" try: response = country_database.country(ip.strip()) if response.country.name: return response.country.name else: return response.continent.name except Exception, exc: print exc return "Unknown"
def city(geoip): with Reader('./Databases/GeoLite2-City.mmdb') as reader: response = reader.city(geoip) country = response.country.name state = response.subdivisions.most_specific.name city = response.city.name postal_code = response.postal.code latitude = response.location.latitude longitude = response.location.longitude return (country, state, city, postal_code, latitude, longitude)
def fetch_asn(ip_address): # Check if INCLUDE_ASN is True before proceeding if not config.INCLUDE_ASN: return None # Load GeoLite2 ASN database geoasn = Reader(path.join(config.MMDB_PATH, "GeoLite2-ASN.mmdb")) # Try to fetch data and build response, otherwise return empty try: data = geoasn.asn(ip_address) return { "name": data.autonomous_system_organization, "id": data.autonomous_system_number } except AddressNotFoundError: return {} finally: geoasn.close()
def get_geoip_reader() -> Optional[Reader]: """Get GeoIP Reader, if configured, otherwise none""" path = CONFIG.y("authentik.geoip") if path == "" or not path: return None try: reader = Reader(path) LOGGER.info("Enabled GeoIP support") return reader except OSError: return None
class MaxMindEnterpriseDatabase(BaseDriver): """ MaxMind Enterprise Database """ def __init__(self, database): """ Constructor :param database: Location of the enterprise-database """ self._reader = Reader(database) # Close reader when app closes down atexit.register(lambda: self._reader.close()) def insights(self, ip): """ Get insights in ip :param ip: The ip :return: Insights :rtype: geoip2.models.City """ return self._reader.enterprise(ip)
def extract_location(line): global reader if reader is None: reader = Reader( "/home/spark/spark-2.4.5-bin-hadoop2.7/maxmind/GeoLite2-City.mmdb") match = pattern.match(line) if match: ip = match.group(1) response = reader.city(ip) country = response.country.name city = response.city.name if city is None: return country else: return "{},{}".format(country, city) else: return "InvalidLogFound"
def get_lng_lat(ip_address): """ Takes an IPv4 or IPv6 address and returns a 2-tuple of (longitude, latitude). """ coord = None if ip_address and is_public_ip_addr(ip_address): city_db_path = get_city_db_path() db_reader = Reader(city_db_path) try: result = db_reader.city(ip_address) coord = (result.location.longitude, result.location.latitude) except AddressNotFoundError: _LOGGER.warning('The address %s is not in the GeoLite2 database.', ip_address) finally: db_reader.close() return coord
class Handler(GeoipService.ContextIface): def __init__(self, city_db_path): # maxmind docs recommend reusing a Reader across requests self.reader = Reader(city_db_path) def is_healthy(self, context): return True def get_country(self, context, ip_address): # TODO: add anonymous info (tor/ec2/rackspace/etc) try: response = self.reader.city(ip_address) except AddressNotFoundError: country_code = "" country_name = "" else: country_code = response.country.iso_code country_name = response.country.name return GeoIpRecord( country_code=country_code, country_name=country_name, )
def __init__(self, city_db_path): # maxmind docs recommend reusing a Reader across requests self.reader = Reader(city_db_path)
def __init__(self, db='/home/insane/PycharmProjects/octopus/GeoLite2-City.mmdb'): self.r = Reader(db)