def set_mirror_country(
mirror_info: Dict[AnyStr, Union[Dict, AnyStr]], ) -> None:
"""
Set country by IP of a mirror
:param mirror_info: Dict with info about a mirror
"""
mirror_name = mirror_info['name']
try:
ip = socket.gethostbyname(mirror_name)
except socket.gaierror:
logger.error('Can\'t get IP of mirror %s', mirror_name)
mirror_info['country'] = 'Unknown'
return
db = Reader(GEOPIP_DB)
logger.info('Set country for mirror "%s"', mirror_name)
try:
match = db.city(ip) # type: City
mirror_info['country'] = match.country.name
except AddressNotFoundError:
logger.warning(
'GeoIP db does not have information about IP "%s"',
ip,
)
mirror_info['country'] = 'Unknown'
def geoip_location(ip=None, timeout=DEFAULT_TIMEOUT):
geoip_file = os.environ.get('GEOIP2_FILE')
if not geoip_file or not os.path.exists(geoip_file):
raise LocationServiceException(
'no valid GEOIP2_FILE set -- please download and decompress: '
'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz' # noqa
)
if not ip:
try:
xff = requests.get('http://jsonip.com/').json()['ip']
ip = xff.split(', ')[-1]
except Exception:
raise LocationServiceException(
'error getting an ip -- are you online?'
)
reader = Reader(geoip_file)
rec = reader.omni(ip)
if not rec.location:
raise LocationServiceException(
'unable to geocode ip address ({0})'.format(ip)
)
loc = rec.location
return Location(loc.latitude, loc.longitude, 'geoip')
def get_location(cls, ip=None):
if not ip:
ip = cls.get_ip()
try:
city_client = Reader("./utils/geoip2/GeoLite2-City.mmdb")
response = city_client.city(ip)
if response.city.name:
location = ",".join(
[response.country.iso_code, response.city.name])
else:
location = ",".join([response.country.iso_code, ""])
except Exception as e:
print("fail to get location with geoip2: %s" % str(e))
try:
api_key = 'at_IW99hSbVb4uxQq1SbaoIanDbulTbU'
api_url = 'https://geo.ipify.org/api/v1?'
url = api_url + 'apiKey=' + api_key + '&ipAddress=' + ip
temp_region = loads(
urlopen(url).read().decode('utf8'))["location"]
try:
location = ",".join(
[temp_region["country"], temp_region["city"]])
except [KeyError, ValueError]:
location = temp_region
except URLError:
location = "network error"
return location
def get_geo_info(self):
geo_asn = Reader(join(self.path, self.geo[0])).asn(str(self.addr))
geo_city = Reader(join(self.path, self.geo[1])).city(str(self.addr))
return (geo_asn.autonomous_system_number,
geo_asn.autonomous_system_organization, geo_city.country.name,
geo_city.city.name, geo_city.location.latitude,
geo_city.location.longitude)
def main():
args = docopt(__doc__)
out = csv.writer(sys.stdout)
reader = Reader(args['-f'])
try:
ips = [x.strip() for x in open(args['IPFILE'], 'r')]
except FileNotFoundError:
if not args['-q']:
print("IP file {} count not be opened, interpreting as single IP!".format(args['IPFILE']))
ips = [args['IPFILE'].strip()]
for ip in ips:
try:
resp = reader.city(ip)
except ValueError:
if not args['-q']:
print("{} is not an IP address, skipping!".format(ip), file=sys.stderr)
continue
row = []
if args['-i']:
row.append(ip)
if args['-c']:
row.append(resp.country.names.get('en', '-'))
row.append(resp.country.iso_code)
try:
row.append(resp.city.names['en'])
except KeyError:
row.append('-')
out.writerow(row)
def __init__(self, database):
"""
Constructor
:param database: Location of the enterprise-database
"""
self._reader = Reader(database)
# Close reader when app closes down
atexit.register(lambda: self._reader.close())
def __init__(self, path_to_db):
from geoip2.database import Reader
try:
self.reader = Reader(path_to_db)
except:
pass
self.cache = {}
def get_location_for_ip(ip_addr):
try:
reader = Reader(
os.path.join(GEOLOCATION_DATABASE_PATH, GEOLOCATION_DATABASE_CITY))
return reader.city(ip_addr)
except Exception as e:
logger.warn("Failed retrieving geoip info for %s: %s" %
(ip_addr, str(e)))
return None
def get_asn_for_ip(ip_addr):
try:
reader = Reader(
os.path.join(GEOLOCATION_DATABASE_PATH, GEOLOCATION_DATABASE_ASN))
return reader.asn(ip_addr)
except Exception as e:
logger.warn("Failed retrieving asn info for %s: %s" %
(ip_addr, str(e)))
return None
def __open(self):
"""Get GeoIP Reader, if configured, otherwise none"""
path = CONFIG.y("geoip")
if path == "" or not path:
return
try:
self.__reader = Reader(path)
self.__last_mtime = stat(path).st_mtime
LOGGER.info("Loaded GeoIP database", last_write=self.__last_mtime)
except OSError as exc:
LOGGER.warning("Failed to load GeoIP database", exc=exc)
def getisp(ip):
loc_info = {}
r = Reader(join(abspath(dirname(__file__)), 'GeoIP2-ISP.mmdb'))
obj = r.isp(ip)
print obj.raw
loc_info['isp'] = {
'organization': obj.raw.get('autonomous_system_organization', ''),
'isp': obj.raw.get('isp', ''),
}
print loc_info
r.close()
def fetch_geoip(ip_address, language=None):
# Prepare response object
response = {}
# Get hostname from IP address, pass if fail
try:
response['ip_address'] = ip_address
response['hostname'] = socket.gethostbyaddr(ip_address)[0]
except Exception as ex:
pass
# Load GeoLite2 City database
geoip = Reader(path.join(config.MMDB_PATH, "GeoLite2-City.mmdb"))
# Try to fetch data and build response, otherwise raise exception
try:
data = geoip.city(ip_address)
# geoip.city
response['city'] = {
"name": data.city.name,
"id": data.city.geoname_id,
"region": data.subdivisions.most_specific.name,
"region_code": data.subdivisions.most_specific.iso_code
}
# geoip.country
response['country'] = {
"name": data.country.name,
"iso_code": data.country.iso_code,
"continent": data.continent.name,
"continent_code": data.continent.code,
"is_eu": data.country.is_in_european_union
}
# geoip.location
response['location'] = {
"accuracy_radius": data.location.accuracy_radius,
"zip": data.postal.code,
"latitude": data.location.latitude,
"longitude": data.location.longitude,
"timezone": data.location.time_zone
}
except AddressNotFoundError as ex:
raise ex
# Close database instances
geoip.close()
# Load GeoLite2 ASN database (optional)
response['asn'] = fetch_asn(ip_address)
# Return built response object
return response
def find_location(ip):
reader = Reader(TannerConfig.get('DATA', 'geo_db'))
try:
location = reader.city(ip)
info = dict(
country=location.country.name,
country_code=location.country.iso_code,
city=location.city.name,
zip_code=location.postal.code,
)
except geoip2.errors.AddressNotFoundError:
info = "NA" # When IP doesn't exist in the db, set info as "NA - Not Available"
return info
def get_country(ip):
from geoip2.database import Reader
from geoip2.errors import AddressNotFoundError
try:
# Насколько я понимаю spark неспособен создать broadcast верси database.Reader
# объект, поэтому приходится его создавать при каждом вызове функции
# хотя возможно, производится какая-то оптимизация внутри udf()
reader = Reader(giLite2db_path)
response = reader.country(ip)
country = response.country.name
return country if country else "NOT_FOUND"
except AddressNotFoundError:
return "NOT_FOUND"
def find_location(ip):
reader = Reader(TannerConfig.get('DATA', 'geo_db'))
try:
location = reader.city(ip)
info = dict(
country=location.country.name,
country_code=location.country.iso_code,
city=location.city.name,
zip_code=location.postal.code,
)
except geoip2.errors.AddressNotFoundError:
info = "NA" # When IP doesn't exist in the db, set info as "NA - Not Available"
return info
class GeoIP:
def __init__(self, db='/home/insane/PycharmProjects/octopus/GeoLite2-City.mmdb'):
self.r = Reader(db)
def get_continent_code(self, ip):
return self.r.city(ip).continent
def get_longitude(self, ip):
return self.r.city(ip).location.longitude
def get_latitude(self, ip):
return self.r.city(ip).location.latitude
def get_iso_code(self, ip):
return self.r.city(ip).country.iso_code
def ip2city_py(ip):
global reader
if reader is None:
# assume all work nodes have mmdb installed in the following path
reader = Reader(
"/home/spark/spark-2.4.5-bin-hadoop2.7/maxmind/GeoLite2-City.mmdb"
)
try:
response = reader.city(ip)
city = response.city.name
if city is None:
return None
return city
except:
return None
def reader_manager(self, action=None):
if action == 'open':
try:
self.reader = Reader(self.dbfile)
except FileNotFoundError:
self.logger.error("Could not find MaxMind DB! Downloading!")
result_status = self.download()
if result_status:
self.logger.error(
"Could not download MaxMind DB! You may need to manually install it."
)
exit(1)
else:
self.reader = Reader(self.dbfile)
else:
self.reader.close()
def ip2city_py(ip):
global reader
if reader is None:
# assume all work nodes have mmdb installed in the following path
reader = Reader(
"/home/cloudera/Desktop/spark_sql_101/maxmind/GeoLite2-City.mmdb"
)
try:
response = reader.city(ip)
city = response.city.name
if city is None:
return None
return city
except:
return None
def __spawn_maxmind(cls, configuration=None, **ignored):
if cls.MAXMIND is None:
if configuration is None:
from ta_common.taste_config_helper import TasteConf
configuration = TasteConf()
cls.MAXMIND = Reader(configuration.getMaxMindSource())
return cls.MAXMIND
def process_test_data(
raw_data: list, previous_results: dict,
country_database_path: str) -> Dict[str, Dict[str, object]]:
"""Process the raw data of the test."""
result = json.loads(raw_data['general']['data'].decode())
# geoip
reader = Reader(country_database_path)
result['a_locations'] = _get_countries(result['a_records'], reader)
result['mx_locations'] = _get_countries(
(ip for mx_a_records in result['mx_a_records']
for ip in mx_a_records[1]), reader)
# TODO: reverse mx-a matches mx
result['final_url_is_https'] = ('final_url' in result and
result['final_url'].startswith('https'))
# handle non-https final url
if (not result['final_url_is_https'] and 'final_url_content' in raw_data
and 'final_https_url_content' in raw_data):
similarity = _jaccard_index(
raw_data['final_url_content']['data'],
raw_data['final_https_url_content']['data'])
result['same_content_via_https'] = similarity > MINIMUM_SIMILARITY
return result
def load_geoip_database():
global geoip_reader
try:
geoip_reader = Reader(settings.GEOIP2_DATABASE)
except IOError:
log.warning('Geolocation is disabled: Cannot load database.')
def load_geoip_database():
global geoip_reader
try:
geoip_reader = Reader(settings.GEOIP2_DATABASE)
except IOError:
logger.warning('Geolocation is disabled: Cannot load database.',
extra={'code': WARNING_CANNOT_LOAD_DATABASE})
def statics_city():
"""统计区域数据"""
city = {}
geoip2_reader = Reader("GeoLite2-City.mmdb")
for _ip, _cnt in statics_all()["vistors"].items():
_city_name = "Unknow"
try:
_city = geoip2_reader.city(_ip)
# _city_name = "{}/{}".format(_city.country.names.get("en","").encode("utf8"),_city.city.names.get("en","").encode("utf8"))
_city_name = "{}/{}".format(_city.country.names.get("zh-CN", ""),
_city.city.names.get("zh-CN", ""))
except BaseException as e:
print(e)
pass
city.setdefault(_city_name, 0)
city[_city_name] += _cnt
return city
def threaded_localize(self, many_ip, geolite_file):
"""
Triggers call to localize in a multithreaded fashion
"""
geo = Reader(geolite_file)
with ThreadPool(processes=self.threads) as p:
res = [ p.apply_async(self.localize, (ip, geo)) for ip in many_ip ]
return [ r.get() for r in res ]
def _insert_geoip(result, logger, options):
# GeoIP
reader = Reader(_get_geoip_database(options))
result['a_locations'], result['a_locations_unknown'] = _get_countries(
result['a_records'], reader)
result['mx_locations'], result['mx_locations_unknown'] = _get_countries(
(ip for mx_a_records in result['mx_a_records']
for ip in mx_a_records[1]), reader)
def geolocate(url):
"""
Finds data about url in geolocation database and transfers it mongodb
:param url: url to lacate
:return: geolocation_id
"""
from geoip2.database import Reader
from geoip2.webservice import Client
from geoip2.errors import GeoIP2Error, HTTPError
geolocation_data = dict()
try:
ip = url_to_ip(url)
response = None
if config.GEOIP2_WEB_SERVICE_USER_ID and config.GEOIP2_WEB_SERVICE_LICENSE_KEY \
and config.GEOIP2_WEB_SERVICE_TYPE:
client = Client(config.GEOIP2_WEB_SERVICE_USER_ID,
config.GEOIP2_WEB_SERVICE_LICENSE_KEY)
if config.GEOIP2_WEB_SERVICE_TYPE == 'country':
response = client.country(ip).country
elif config.GEOIP2_WEB_SERVICE_TYPE == 'city':
response = client.city(ip).city
elif config.GEOIP2_WEB_SERVICE_TYPE == 'insights':
response = client.insights(ip).insights
elif config.GEOIP2_DATABASE_PATH and config.GEOIP2_DATABASE_TYPE:
reader = Reader(config.GEOIP2_DATABASE_PATH)
if config.GEOIP2_DATABASE_TYPE == 'country':
response = reader.country(ip).country
if config.GEOIP2_DATABASE_TYPE == 'city':
response = reader.city(ip).city
else:
reader = Reader(
'/opt/project/worker/utils/geoloc_databases/GeoLite2-City.mmdb'
)
response = reader.city(ip)
for name in dir(response):
value = getattr(response, name)
if not name.startswith('_') and not type(value) == dict:
geolocation_data[name] = value.__dict__
except (GeoIP2Error, HTTPError) as error:
geolocation_data = {'_error': str(error)}
finally:
duplicate = db.geolocation.find_one(geolocation_data)
if duplicate:
return duplicate['_id']
geolocation_id = db.geolocation.insert_one(geolocation_data)
return str(geolocation_id.inserted_id)
def __init__(
self,
geoip_db_path='/config/geoip2db/GeoLite2-City.mmdb',
log_path=DEFAULT_LOG_PATH,
influxdb_host=DEFAULT_INFLUX_HOST,
influxdb_port=DEFAULT_INFLUX_HOST_PORT,
influxdb_database=DEFAULT_INFLUX_DATABASE,
influxdb_user=DEFAULT_INFLUX_USER,
influxdb_user_pass=DEFAULT_INFLUX_PASS,
influxdb_retention=DEFAULT_INFLUX_RETENTION,
influxdb_shard=DEFAULT_INFLUX_SHARD,
geo_measurement=DEFAULT_GEO_MEASUREMENT,
log_measurement=DEFAULT_LOG_MEASUREMENT,
send_nginx_logs=DEFAULT_SEND_NGINX_LOGS,
):
self.geoip_db_path = geoip_db_path
self.log_path = log_path
self.influxdb_host = influxdb_host
self.influxdb_port = influxdb_port
self.influxdb_database = influxdb_database
self.influxdb_user = influxdb_user
self.influxdb_user_pass = influxdb_user_pass
self.influxdb_retention = influxdb_retention
self.influxdb_shard = influxdb_shard
self.geo_measurement = geo_measurement
self.log_measurement = log_measurement
self.send_nginx_logs = send_nginx_logs
self.geoip_db_path = geoip_db_path
logging.debug('Log parser config:' +
f'\n geoip_db_path :: {self.geoip_db_path}' +
f'\n log_path :: {self.log_path}' +
f'\n influxdb_host :: {self.influxdb_host}' +
f'\n influxdb_port :: {self.influxdb_port}' +
f'\n influxdb_database :: {self.influxdb_database}' +
f'\n influxdb_retention :: {self.influxdb_retention}' +
f'\n influxdb_shard :: {self.influxdb_shard}' +
f'\n influxdb_user :: {self.influxdb_user}' +
f'\n influxdb_user_pass :: {self.influxdb_user_pass}' +
f'\n geo_measurement :: {self.geo_measurement}' +
f'\n log_measurement :: {self.log_measurement}' +
f'\n send_nginx_logs :: {self.send_nginx_logs}')
self.influxdb = self.init_influxdb()
self.geoip = Reader(self.geoip_db_path)
self.hostname = uname()[1]
def obtener_pais(ip):
try:
country_database = Reader("GeoLite2-Country.mmdb")
except:
country_database = None
if ip is None or country_database is None:
return "Unknown"
if ip.startswith("10.") or ip.startswith("192.168."):
return "Local"
try:
response = country_database.country(ip.strip())
if response.country.name:
return response.country.name
else:
return response.continent.name
except Exception, exc:
print exc
return "Unknown"
def city(geoip):
with Reader('./Databases/GeoLite2-City.mmdb') as reader:
response = reader.city(geoip)
country = response.country.name
state = response.subdivisions.most_specific.name
city = response.city.name
postal_code = response.postal.code
latitude = response.location.latitude
longitude = response.location.longitude
return (country, state, city, postal_code, latitude, longitude)
def fetch_asn(ip_address):
# Check if INCLUDE_ASN is True before proceeding
if not config.INCLUDE_ASN:
return None
# Load GeoLite2 ASN database
geoasn = Reader(path.join(config.MMDB_PATH, "GeoLite2-ASN.mmdb"))
# Try to fetch data and build response, otherwise return empty
try:
data = geoasn.asn(ip_address)
return {
"name": data.autonomous_system_organization,
"id": data.autonomous_system_number
}
except AddressNotFoundError:
return {}
finally:
geoasn.close()
def get_geoip_reader() -> Optional[Reader]:
"""Get GeoIP Reader, if configured, otherwise none"""
path = CONFIG.y("authentik.geoip")
if path == "" or not path:
return None
try:
reader = Reader(path)
LOGGER.info("Enabled GeoIP support")
return reader
except OSError:
return None
class MaxMindEnterpriseDatabase(BaseDriver):
"""
MaxMind Enterprise Database
"""
def __init__(self, database):
"""
Constructor
:param database: Location of the enterprise-database
"""
self._reader = Reader(database)
# Close reader when app closes down
atexit.register(lambda: self._reader.close())
def insights(self, ip):
"""
Get insights in ip
:param ip: The ip
:return: Insights
:rtype: geoip2.models.City
"""
return self._reader.enterprise(ip)
def extract_location(line):
global reader
if reader is None:
reader = Reader(
"/home/spark/spark-2.4.5-bin-hadoop2.7/maxmind/GeoLite2-City.mmdb")
match = pattern.match(line)
if match:
ip = match.group(1)
response = reader.city(ip)
country = response.country.name
city = response.city.name
if city is None:
return country
else:
return "{},{}".format(country, city)
else:
return "InvalidLogFound"
def get_lng_lat(ip_address):
"""
Takes an IPv4 or IPv6 address and returns a 2-tuple of (longitude, latitude).
"""
coord = None
if ip_address and is_public_ip_addr(ip_address):
city_db_path = get_city_db_path()
db_reader = Reader(city_db_path)
try:
result = db_reader.city(ip_address)
coord = (result.location.longitude, result.location.latitude)
except AddressNotFoundError:
_LOGGER.warning('The address %s is not in the GeoLite2 database.',
ip_address)
finally:
db_reader.close()
return coord
class Handler(GeoipService.ContextIface):
def __init__(self, city_db_path):
# maxmind docs recommend reusing a Reader across requests
self.reader = Reader(city_db_path)
def is_healthy(self, context):
return True
def get_country(self, context, ip_address):
# TODO: add anonymous info (tor/ec2/rackspace/etc)
try:
response = self.reader.city(ip_address)
except AddressNotFoundError:
country_code = ""
country_name = ""
else:
country_code = response.country.iso_code
country_name = response.country.name
return GeoIpRecord(
country_code=country_code,
country_name=country_name,
)
def __init__(self, city_db_path):
# maxmind docs recommend reusing a Reader across requests
self.reader = Reader(city_db_path)
def __init__(self, db='/home/insane/PycharmProjects/octopus/GeoLite2-City.mmdb'):
self.r = Reader(db)
