def _setup_geodb(self):
geoipdb_path = self._enrich_config["geoippath"]
geoipdb_asn_file = self._enrich_config["asndatabasefilename"]
geoipdb_city_file = self._enrich_config["citydatabasefilename"]
self.gi_asn = database.Reader(fileish=os.path.join(
geoipdb_path, geoipdb_asn_file),
mode=maxminddb.const.MODE_MEMORY)
self.gi_cc = database.Reader(fileish=os.path.join(
geoipdb_path, geoipdb_city_file),
mode=maxminddb.const.MODE_MEMORY)
def init_app(self, app):
"""Initialize the GeoIP instance from an application.
:param app: application
:type app: sanic.Sanic
"""
self._geo_asn = database.Reader(app.config.get('MAXMIND_ASN_PATH'))
self._geo_city = database.Reader(app.config.get('MAXMIND_CITY_PATH'))
self._geo_country = database.Reader(
app.config.get('MAXMIND_COUNTRY_PATH'))
self._initialized = False
def index():
temp = request.remote_addr
jj = request.headers
# man = gi.country_name_by_addr('106.210.129.209')
response = requests.get(
"http://ip-api.com/json/{}".format('106.210.129.209'))
js = response.json()
reader = geoip.Reader(path)
try:
country = reader.country(temp)
except Exception as e:
country = " the error constyr"
return (country)
print("indiad")
# print(man)
name = country.country.names['en']
value = f'ma strigngng {temp}'
l = ['india', 'us']
random.shuffle(l)
print(l[-1])
print(type(country))
# print(jsonify(country))
val = country.country
# print(jsonify(val))
val = country.country.names['en']
# value = GenerateTemplate(l[-1])
# print(value)
return val + " the location whixh user come"
def __init__(self, **kwargs):
self.geoipDB = kwargs.get('geoipDB')
self.map_key = kwargs.get('map_key')
self.ip = kwargs.get('ip')
self.private = ipaddress.ip_address(self.ip).is_private
if self.private:
HAS_GEO = False
logger.warning('%s is a local ip, continuing without geodata',
self.ip)
try:
self.reader = database.Reader(self.geoipDB)
except FileNotFoundError:
HAS_GEO = False
logger.warning(
'GeoIP database not found in %s, continuing without geodata',
self.geoipDB)
except errors.AddressNotFoundError:
HAS_GEO = False
logger.warning(
'GeoIP did not find a location for %s, continuing without geodata',
self.ip)
except Exception as e:
logger.error('FATAL ERROR: %s', e)
def partition_handle(mons):
from geoip2 import database
etcd_key = get_etcd_key(cfg)
def ip2subdivision(monitor):
try:
province_name = reader.city(
monitor['pubip']).subdivisions.most_specific.name
monitor['province'] = province_name or 'None'
identity_code = IDENTITY_CODE.get(province_name)
# 获取地区分组类别
monitor['group'] = etcd_key.get(identity_code, 'default')
except:
monitor['province'] = 'None'
monitor['group'] = 'default'
return monitor
def rtt2level(monitor):
for info in monitor.get('req_rtt', []):
info['rtt_level'] = rtt_level_map.get(
bisect.bisect_right(rtt_level_list, info['rtt']),
rtt_level_map[1])
return monitor
reader = database.Reader(SparkFiles.get(geo_db_path))
ip_res = [ip2subdivision(mon) for mon in mons]
rtt_level_res = [rtt2level(mon) for mon in ip_res]
return rtt_level_res
def country_by_ip(request):
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0].strip()
else:
ip = request.META.get('REMOTE_ADDR')
# ip = '80.254.2.190'
path = '%s/%s' % (settings.BASE_DIR, 'GeoLite2-Country.mmdb')
reader = database.Reader(path)
try:
country = reader.country(ip).country.name
headers = {
k: v
for k, v in request.META.iteritems() if k.startswith('HTTP')
}
user_agent = headers.get('HTTP_USER_AGENT')
date_request = timezone.now()
GeoInfo.objects.create(ip=ip,
country=country,
user_agent=user_agent,
headers=headers,
date=date_request)
except:
pass
return HttpResponse()
def ipv4info(ip):
"""Returns a dict containing the country, country_code, and city for
a given IPv4 address"""
result = {
"country": "unknown",
"country_code": "unknown",
"city": "unknown"
}
if not GeoInfo.georeader:
georeader = geodatabase.Reader(
cwd("geodb", "extracted", "geodblite.mmdb"))
if is_reserved_ipv4(ip):
return result
try:
geodata = georeader.city(ip)
if geodata.country.name:
result["country"] = geodata.country.name
if geodata.country.iso_code:
result["country_code"] = geodata.country.iso_code
if geodata.city.name:
result["city"] = geodata.city.name
except (GeoIP2Error, ValueError):
return result
return result
def ip2geo(ip):
if not os.path.exists(os.path.expanduser('~/geo/GeoLite2-City.mmdb')):
os.system(
'mkdir -p $HOME/geo && wget -c -t 10 "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz" -O- |tar -xz -C $HOME && mv $HOME/GeoLite2-City_2019*/GeoLite2-City.mmdb $HOME/geo'
)
ipdb = gd.Reader(os.path.expanduser('~/geo/GeoLite2-City.mmdb'))
return ipdb.city(ip).country.name
def __init__(self, cfg, **kwargs):
self.vpns = cfg.vpns
if 'vpn_id' in kwargs:
vpn = self.vpns[kwargs['vpn_id']]
self._socket_connect(vpn)
if vpn['socket_connected']:
release = self.send_command('version\n')
version = semver(self.parse_version(release).split(' ')[1])
if version.major == 2 and \
version.minor >= 4 and \
'port' not in kwargs:
command = 'client-kill {0!s}\n'.format(kwargs['client_id'])
else:
command = 'kill {0!s}:{1!s}\n'.format(kwargs['ip'], kwargs['port'])
self.send_command(command)
self._socket_disconnect()
geoip_data = cfg.settings['geoip_data']
if geoip_data.endswith('.mmdb') and geoip2_available:
self.gi = database.Reader(geoip_data)
self.geoip_version = 2
elif geoip_data.endswith('.dat') and geoip1_available:
self.gi = geoip1.open(geoip_data, geoip1.GEOIP_STANDARD)
self.geoip_version = 1
else:
warning('No compatible geoip1 or geoip2 data/libraries found.')
self.geoip_version = None
self.gi = None
for key, vpn in list(self.vpns.items()):
self._socket_connect(vpn)
if vpn['socket_connected']:
self.collect_data(vpn)
self._socket_disconnect()
def _main():
import sys
reload(sys)
sys.setdefaultencoding('utf8')
_api_path = os.path.dirname(
os.path.abspath(__file__)) + os.path.sep + ".." + os.path.sep + "api"
_mmdb = "GeoLite2-City.mmdb"
_mmdb = _api_path + os.path.sep + "geolite2" + os.path.sep + _mmdb
if not os.path.exists(_mmdb):
logging.error(
"no geolite2 mmdb, run scripts/download_geolite2.sh to download.")
sys.exit()
tornado.options.parse_command_line()
_app = ApiApp()
_app.geoip_reader = database.Reader(_mmdb)
_http_server = tornado.httpserver.HTTPServer(_app)
_http_server.listen(tornado.options.options.port)
logging.info("Starting API servcie.")
tornado.ioloop.IOLoop.instance().start()
return
def transform(raw_data):
"""
Takes the raw data returned from the journalctl command and filters / parses it
down into a database-ready collection of rows.
"""
data = [json.loads(line) for line in raw_data]
rows = []
user_patt = re.compile("user (.*?) from")
network_patt = re.compile("from (.*?) port (.*?)$")
db_path = os.path.expanduser("~/GeoLite/GeoLite2-City.mmdb")
db_reader = geo_db.Reader(db_path)
for d in data:
if user_patt.findall(d["MESSAGE"]) and "Invalid" in d["MESSAGE"]:
row = {}
row["timestamp"] = datetime.fromtimestamp(
int(d["__REALTIME_TIMESTAMP"]) /
1e6).strftime("%Y-%m-%d %H:%M:%S")
row["username"] = user_patt.findall(d["MESSAGE"])[0]
ip, port = network_patt.findall(d["MESSAGE"])[0]
location = db_reader.city(ip)
row["port"] = int(port)
row["city"] = location.city.name
row["country"] = location.country.name
row["latitude"] = location.location.latitude
row["longitude"] = location.location.longitude
rows.append(row)
return rows
def __init__(self, *args, db_path=DB_FILE, **kwargs):
# https://dev.maxmind.com/geoip/geoip2/geolite2/
super().__init__(*args, **kwargs)
try:
self.db_reader = geo_db.Reader(db_path)
self.log.info('Reading GeoDB {}'.format(db_path))
except:
self.log.exception('Could not read from {}!'.format(db_path))
def f(iter):
from geoip2 import database
reader = database.Reader("GeoLite2-Country.mmdb")
try:
country = reader.country(iter[0]).country.name
except:
country = 'NOT_FOUND'
return country, iter[1]
def init():
"""
Initialiation routine
"""
gi = ipdb.Reader("/usr/share/GeoIP/GeoLite2-City.mmdb")
return gi
def localizar(ip):
try:
localizador = geo.Reader('./GeoLite2-City.mmdb')
posicao = localizador.city(ip)
return posicao
except Exception as e:
print(e)
print("\narquivo não encontrado")
sys.exit()
def print_record2(tgt):
db = database.Reader("GeoLite2/GeoLite2-City.mmdb")
rec = db.city(tgt)
print(rec)
print(rec.country)
print(rec.city)
print(rec.subdivisions)
print(rec.subdivisions[0].names["zh-CN"])
def get_city(ip):
if ip is None:
return None
geocity = database.Reader(SparkFiles.get(city_db))
try:
record = geocity.city(ip)
return record.city.name
except geoip2.errors.AddressNotFoundError:
return None
def _reader(self, ipaddr):
reader_city = db.Reader('./db/GeoLite2-City.mmdb')
reader_asn = db.Reader('./db/GeoLite2-ASN.mmdb')
result_city = None
result_asn = None
try:
result_city = reader_city.city(ipaddr)
except err.AddressNotFoundError:
self.addr_notfound = True
try:
result_asn = reader_asn.asn(ipaddr)
except err.AddressNotFoundError:
self.asn_notfound = True
reader_city.close()
reader_asn.close()
return result_city, result_asn
def load_ip2geo():
_api_path = os.path.dirname(os.path.abspath(__file__))
_mmdb = "GeoLite2-City.mmdb"
_mmdb = _api_path + os.path.sep + "geolite2" + os.path.sep + _mmdb
if not os.path.exists(_mmdb):
logging.error("no geolite2 mmdb, run scripts/download_geolite2.sh to download and restart api.")
return None
_reader = database.Reader(_mmdb)
return _reader
def includeme(config): # pragma: no cache
geoip_path = config.registry.settings["geoip.path"]
if not geoip_path:
return
geoip = geoip_db.Reader(geoip_path)
def geoip_factory(context, request):
return geoip
config.register_service_factory(geoip, name="geoip")
def ip_to_country_iso(ip: str):
reader = database.Reader('GeoLite2_DB/GeoLite2-City.mmdb')
geoip_models_city = reader.city(ip)
try:
iso = geoip_models_city.country.iso_code
except:
iso = geoip_models_city.registered_country.iso_code
return iso
def ip_to_country(ip: str):
reader = database.Reader('GeoLite2_DB/GeoLite2-City.mmdb')
geoip_models_city = reader.city(ip)
try:
country = geoip_models_city.country.names["ja"]
except:
country = geoip_models_city.registered_country.names["ja"]
return country
def get_city():
URL = "https://ip8.com/ip"
page = requests.get(URL)
data = BeautifulSoup(page.text, 'html.parser')
ip = str(data)
reader = geo.Reader("GeoLite2-City.mmdb")
response = reader.city(ip)
city = response.city.name
reader.close()
return city
def extract_geodata(db, ip, input):
reader = database.Reader(input)
try:
data = reader.country(ip)
except AddressNotFoundError:
return
country_code = data.registered_country.iso_code
if country_code:
update_data(db, ip, {'country_code': country_code})
def _get_geoip_lib():
global GEO_IP
if not GEOIP_DATA_LOCATION:
return None
try:
GEO_IP = geoip2_db.Reader(GEOIP_DATA_LOCATION)
except:
return None
return GEO_IP
def __init__(self):
Logging.info("Loading geoip.")
self.__reader = database.Reader('files/GeoLite2-Country.mmdb')
self.__languages = {
Language.PT: {"PT"},
Language.BR: {"BR"},
Language.EN: {"US", "AG", "CA", "AU", "BS", "BB", "BZ"},
Language.ES: {
"AR", "PY", "UY", "CO", "PE", "CL", "EC", "BO", "VE", "SV",
"NI", "GT", "CR", "CU", "GQ", "HN", "PA", "DO", "MX", "ES"
}
}
def __init__(self, ptid=None, begin=None, end=None):
#读取配置文件
self.reader = gd.Reader('GeoIP2-City.mmdb')
self.cf = ConfigParser.ConfigParser()
self.cf.read('imsiConfig2.conf')
self.redshift = RedShift()
self.mysql = Mysql()
#初始化开始结束日期，初始化产品id
self.today = end if end else datetime.date.today().strftime("%Y-%m-%d")
self.yesterday = begin if begin else (
datetime.date.today() -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
self.before_yesterday = (
datetime.datetime.strptime(str(self.yesterday), "%Y-%m-%d") +
datetime.timedelta(days=-1)).strftime("%Y-%m-%d")
self.date_7day_ago = (
datetime.datetime.strptime(str(self.yesterday), "%Y-%m-%d") +
datetime.timedelta(days=-7)).strftime("%Y-%m-%d")
self.ptid = ptid if ptid else '600025'
print self.today
print self.yesterday
print self.ptid
self.day_yesterday = self.yesterday.replace('-', '') #20100101
self.day_before_yesterday = self.getPlusDay(self.yesterday,
-1) #20100101
self.day_7day_ago = self.getPlusDay(self.yesterday, -7) #20100101
#构建数据查询sql
self.sql_channel_cnt = "select media_source,count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source;"
self.sql_channel_ip_cnt = "select media_source,sum(cnt) from (select media_source,ip,count(distinct token) as cnt from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,ip having count(distinct token)>=5) group by media_source;"
self.sql_channel_ip_imsi = "select media_source,country_iso,city,carrier,ip,split_part(imsi,'_',2) as imsi from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ;"
self.sql_channel_imsi_cnt = "select media_source,sum(case when lenimsi=15 then 1 else 0 end) as cnt from (select distinct media_source,token,len(split_part(imsi,'_',2)) as lenimsi from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ) t group by media_source;"
self.sql_channel_imei_cnt = "select media_source,sum(case when lenimei=15 then 1 else 0 end) as cnt from (select distinct media_source,token,len(imei) as lenimei from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ) t group by media_source;"
self.sql_channel_hour_cnt = "select media_source,date_part(hour, install_time),count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,date_part(hour, install_time);"
self.sql_channel_wifi_cnt = "select media_source,sum(case when wifi='t' then cnt else 0 end) as cnt from (select media_source,wifi,count(distinct token) as cnt from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,wifi) t group by media_source;"
self.sql_channel_ver_cnt = "select media_source,ver,count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,ver;"
self.sql_channel_device_cnt = "select media_source,device_brand,count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,device_brand;"
self.sql_channel_IPseg_cnt = "select media_source,sum(cnt) from (select media_source,ip,count(token) as cnt from (select distinct media_source,token,split_part(ip,'.',1)||'.'||split_part(ip,'.',2)||'.'||split_part(ip,'.',3) as ip from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ) t group by media_source,ip having count(token)>=5) t2 group by media_source;"
self.sql_channel_date = "select distinct media_source,'" + self.yesterday + "','" + self.ptid + "' from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null;"
#构建t分布预测模型
y = np.array([0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.98])
x = np.array([0.674, 0.842, 1.036, 1.282, 1.645, 1.960, 2.326])
x = x.reshape(x.shape[0], 1)
X = self.buildComplexDataset(x)
alpha = 0.001
l1_ratio = 0.001
enet = ElasticNet(alpha=alpha, l1_ratio=l1_ratio)
self.model = enet.fit(X, y)
def ip_location(address):
split_list = address.split('/')
ip_address = split_list[2]
location_dict = {}
if ip_address:
with database.Reader(os.getcwd() \
+ '/geoip/GeoLite2-City.mmdb') as reader:
response = reader.city(ip_address)
location_dict['country_name'] = response.country.name
location_dict['city_name'] = response.city.name
location_dict['location_latitude'] = response.location.latitude
location_dict['location_longitude'] = response.location.longitude
return location_dict
def iplocation(self):
try:
if self.isIp():
IP = self.extractIp()
else:
IP = self.domain2ip()
dbpath = os.path.join(GlobalConf().progpath['location'],
'Heaven_Hell/GeoLite2-City.mmdb')
reader = ipdb.Reader(dbpath)
locate = reader.city(IP)
return locate
except:
return dict()
def ip2gps_partition(ips, db_path):
from geoip2 import database
def ip2gps(ip):
try:
response = reader.city(ip['ip'])
city = Row(ip=ip['ip'],
latitude=response.location.latitude,
longitude=response.location.longitude)
except Exception:
city = Row(ip="-1", latitude=-1.0, longitude=-1.0)
return city
reader = database.Reader(SparkFiles.get(db_path))
return [ip2gps(ip) for ip in ips]
