def _setup_geodb(self): geoipdb_path = self._enrich_config["geoippath"] geoipdb_asn_file = self._enrich_config["asndatabasefilename"] geoipdb_city_file = self._enrich_config["citydatabasefilename"] self.gi_asn = database.Reader(fileish=os.path.join( geoipdb_path, geoipdb_asn_file), mode=maxminddb.const.MODE_MEMORY) self.gi_cc = database.Reader(fileish=os.path.join( geoipdb_path, geoipdb_city_file), mode=maxminddb.const.MODE_MEMORY)
def init_app(self, app): """Initialize the GeoIP instance from an application. :param app: application :type app: sanic.Sanic """ self._geo_asn = database.Reader(app.config.get('MAXMIND_ASN_PATH')) self._geo_city = database.Reader(app.config.get('MAXMIND_CITY_PATH')) self._geo_country = database.Reader( app.config.get('MAXMIND_COUNTRY_PATH')) self._initialized = False
def index(): temp = request.remote_addr jj = request.headers # man = gi.country_name_by_addr('106.210.129.209') response = requests.get( "http://ip-api.com/json/{}".format('106.210.129.209')) js = response.json() reader = geoip.Reader(path) try: country = reader.country(temp) except Exception as e: country = " the error constyr" return (country) print("indiad") # print(man) name = country.country.names['en'] value = f'ma strigngng {temp}' l = ['india', 'us'] random.shuffle(l) print(l[-1]) print(type(country)) # print(jsonify(country)) val = country.country # print(jsonify(val)) val = country.country.names['en'] # value = GenerateTemplate(l[-1]) # print(value) return val + " the location whixh user come"
def __init__(self, **kwargs): self.geoipDB = kwargs.get('geoipDB') self.map_key = kwargs.get('map_key') self.ip = kwargs.get('ip') self.private = ipaddress.ip_address(self.ip).is_private if self.private: HAS_GEO = False logger.warning('%s is a local ip, continuing without geodata', self.ip) try: self.reader = database.Reader(self.geoipDB) except FileNotFoundError: HAS_GEO = False logger.warning( 'GeoIP database not found in %s, continuing without geodata', self.geoipDB) except errors.AddressNotFoundError: HAS_GEO = False logger.warning( 'GeoIP did not find a location for %s, continuing without geodata', self.ip) except Exception as e: logger.error('FATAL ERROR: %s', e)
def partition_handle(mons): from geoip2 import database etcd_key = get_etcd_key(cfg) def ip2subdivision(monitor): try: province_name = reader.city( monitor['pubip']).subdivisions.most_specific.name monitor['province'] = province_name or 'None' identity_code = IDENTITY_CODE.get(province_name) # 获取地区分组类别 monitor['group'] = etcd_key.get(identity_code, 'default') except: monitor['province'] = 'None' monitor['group'] = 'default' return monitor def rtt2level(monitor): for info in monitor.get('req_rtt', []): info['rtt_level'] = rtt_level_map.get( bisect.bisect_right(rtt_level_list, info['rtt']), rtt_level_map[1]) return monitor reader = database.Reader(SparkFiles.get(geo_db_path)) ip_res = [ip2subdivision(mon) for mon in mons] rtt_level_res = [rtt2level(mon) for mon in ip_res] return rtt_level_res
def country_by_ip(request): x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') if x_forwarded_for: ip = x_forwarded_for.split(',')[0].strip() else: ip = request.META.get('REMOTE_ADDR') # ip = '80.254.2.190' path = '%s/%s' % (settings.BASE_DIR, 'GeoLite2-Country.mmdb') reader = database.Reader(path) try: country = reader.country(ip).country.name headers = { k: v for k, v in request.META.iteritems() if k.startswith('HTTP') } user_agent = headers.get('HTTP_USER_AGENT') date_request = timezone.now() GeoInfo.objects.create(ip=ip, country=country, user_agent=user_agent, headers=headers, date=date_request) except: pass return HttpResponse()
def ipv4info(ip): """Returns a dict containing the country, country_code, and city for a given IPv4 address""" result = { "country": "unknown", "country_code": "unknown", "city": "unknown" } if not GeoInfo.georeader: georeader = geodatabase.Reader( cwd("geodb", "extracted", "geodblite.mmdb")) if is_reserved_ipv4(ip): return result try: geodata = georeader.city(ip) if geodata.country.name: result["country"] = geodata.country.name if geodata.country.iso_code: result["country_code"] = geodata.country.iso_code if geodata.city.name: result["city"] = geodata.city.name except (GeoIP2Error, ValueError): return result return result
def ip2geo(ip): if not os.path.exists(os.path.expanduser('~/geo/GeoLite2-City.mmdb')): os.system( 'mkdir -p $HOME/geo && wget -c -t 10 "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz" -O- |tar -xz -C $HOME && mv $HOME/GeoLite2-City_2019*/GeoLite2-City.mmdb $HOME/geo' ) ipdb = gd.Reader(os.path.expanduser('~/geo/GeoLite2-City.mmdb')) return ipdb.city(ip).country.name
def __init__(self, cfg, **kwargs): self.vpns = cfg.vpns if 'vpn_id' in kwargs: vpn = self.vpns[kwargs['vpn_id']] self._socket_connect(vpn) if vpn['socket_connected']: release = self.send_command('version\n') version = semver(self.parse_version(release).split(' ')[1]) if version.major == 2 and \ version.minor >= 4 and \ 'port' not in kwargs: command = 'client-kill {0!s}\n'.format(kwargs['client_id']) else: command = 'kill {0!s}:{1!s}\n'.format(kwargs['ip'], kwargs['port']) self.send_command(command) self._socket_disconnect() geoip_data = cfg.settings['geoip_data'] if geoip_data.endswith('.mmdb') and geoip2_available: self.gi = database.Reader(geoip_data) self.geoip_version = 2 elif geoip_data.endswith('.dat') and geoip1_available: self.gi = geoip1.open(geoip_data, geoip1.GEOIP_STANDARD) self.geoip_version = 1 else: warning('No compatible geoip1 or geoip2 data/libraries found.') self.geoip_version = None self.gi = None for key, vpn in list(self.vpns.items()): self._socket_connect(vpn) if vpn['socket_connected']: self.collect_data(vpn) self._socket_disconnect()
def _main(): import sys reload(sys) sys.setdefaultencoding('utf8') _api_path = os.path.dirname( os.path.abspath(__file__)) + os.path.sep + ".." + os.path.sep + "api" _mmdb = "GeoLite2-City.mmdb" _mmdb = _api_path + os.path.sep + "geolite2" + os.path.sep + _mmdb if not os.path.exists(_mmdb): logging.error( "no geolite2 mmdb, run scripts/download_geolite2.sh to download.") sys.exit() tornado.options.parse_command_line() _app = ApiApp() _app.geoip_reader = database.Reader(_mmdb) _http_server = tornado.httpserver.HTTPServer(_app) _http_server.listen(tornado.options.options.port) logging.info("Starting API servcie.") tornado.ioloop.IOLoop.instance().start() return
def transform(raw_data): """ Takes the raw data returned from the journalctl command and filters / parses it down into a database-ready collection of rows. """ data = [json.loads(line) for line in raw_data] rows = [] user_patt = re.compile("user (.*?) from") network_patt = re.compile("from (.*?) port (.*?)$") db_path = os.path.expanduser("~/GeoLite/GeoLite2-City.mmdb") db_reader = geo_db.Reader(db_path) for d in data: if user_patt.findall(d["MESSAGE"]) and "Invalid" in d["MESSAGE"]: row = {} row["timestamp"] = datetime.fromtimestamp( int(d["__REALTIME_TIMESTAMP"]) / 1e6).strftime("%Y-%m-%d %H:%M:%S") row["username"] = user_patt.findall(d["MESSAGE"])[0] ip, port = network_patt.findall(d["MESSAGE"])[0] location = db_reader.city(ip) row["port"] = int(port) row["city"] = location.city.name row["country"] = location.country.name row["latitude"] = location.location.latitude row["longitude"] = location.location.longitude rows.append(row) return rows
def __init__(self, *args, db_path=DB_FILE, **kwargs): # https://dev.maxmind.com/geoip/geoip2/geolite2/ super().__init__(*args, **kwargs) try: self.db_reader = geo_db.Reader(db_path) self.log.info('Reading GeoDB {}'.format(db_path)) except: self.log.exception('Could not read from {}!'.format(db_path))
def f(iter): from geoip2 import database reader = database.Reader("GeoLite2-Country.mmdb") try: country = reader.country(iter[0]).country.name except: country = 'NOT_FOUND' return country, iter[1]
def init(): """ Initialiation routine """ gi = ipdb.Reader("/usr/share/GeoIP/GeoLite2-City.mmdb") return gi
def localizar(ip): try: localizador = geo.Reader('./GeoLite2-City.mmdb') posicao = localizador.city(ip) return posicao except Exception as e: print(e) print("\narquivo não encontrado") sys.exit()
def print_record2(tgt): db = database.Reader("GeoLite2/GeoLite2-City.mmdb") rec = db.city(tgt) print(rec) print(rec.country) print(rec.city) print(rec.subdivisions) print(rec.subdivisions[0].names["zh-CN"])
def get_city(ip): if ip is None: return None geocity = database.Reader(SparkFiles.get(city_db)) try: record = geocity.city(ip) return record.city.name except geoip2.errors.AddressNotFoundError: return None
def _reader(self, ipaddr): reader_city = db.Reader('./db/GeoLite2-City.mmdb') reader_asn = db.Reader('./db/GeoLite2-ASN.mmdb') result_city = None result_asn = None try: result_city = reader_city.city(ipaddr) except err.AddressNotFoundError: self.addr_notfound = True try: result_asn = reader_asn.asn(ipaddr) except err.AddressNotFoundError: self.asn_notfound = True reader_city.close() reader_asn.close() return result_city, result_asn
def load_ip2geo(): _api_path = os.path.dirname(os.path.abspath(__file__)) _mmdb = "GeoLite2-City.mmdb" _mmdb = _api_path + os.path.sep + "geolite2" + os.path.sep + _mmdb if not os.path.exists(_mmdb): logging.error("no geolite2 mmdb, run scripts/download_geolite2.sh to download and restart api.") return None _reader = database.Reader(_mmdb) return _reader
def includeme(config): # pragma: no cache geoip_path = config.registry.settings["geoip.path"] if not geoip_path: return geoip = geoip_db.Reader(geoip_path) def geoip_factory(context, request): return geoip config.register_service_factory(geoip, name="geoip")
def ip_to_country_iso(ip: str): reader = database.Reader('GeoLite2_DB/GeoLite2-City.mmdb') geoip_models_city = reader.city(ip) try: iso = geoip_models_city.country.iso_code except: iso = geoip_models_city.registered_country.iso_code return iso
def ip_to_country(ip: str): reader = database.Reader('GeoLite2_DB/GeoLite2-City.mmdb') geoip_models_city = reader.city(ip) try: country = geoip_models_city.country.names["ja"] except: country = geoip_models_city.registered_country.names["ja"] return country
def get_city(): URL = "https://ip8.com/ip" page = requests.get(URL) data = BeautifulSoup(page.text, 'html.parser') ip = str(data) reader = geo.Reader("GeoLite2-City.mmdb") response = reader.city(ip) city = response.city.name reader.close() return city
def extract_geodata(db, ip, input): reader = database.Reader(input) try: data = reader.country(ip) except AddressNotFoundError: return country_code = data.registered_country.iso_code if country_code: update_data(db, ip, {'country_code': country_code})
def _get_geoip_lib(): global GEO_IP if not GEOIP_DATA_LOCATION: return None try: GEO_IP = geoip2_db.Reader(GEOIP_DATA_LOCATION) except: return None return GEO_IP
def __init__(self): Logging.info("Loading geoip.") self.__reader = database.Reader('files/GeoLite2-Country.mmdb') self.__languages = { Language.PT: {"PT"}, Language.BR: {"BR"}, Language.EN: {"US", "AG", "CA", "AU", "BS", "BB", "BZ"}, Language.ES: { "AR", "PY", "UY", "CO", "PE", "CL", "EC", "BO", "VE", "SV", "NI", "GT", "CR", "CU", "GQ", "HN", "PA", "DO", "MX", "ES" } }
def __init__(self, ptid=None, begin=None, end=None): #读取配置文件 self.reader = gd.Reader('GeoIP2-City.mmdb') self.cf = ConfigParser.ConfigParser() self.cf.read('imsiConfig2.conf') self.redshift = RedShift() self.mysql = Mysql() #初始化开始结束日期,初始化产品id self.today = end if end else datetime.date.today().strftime("%Y-%m-%d") self.yesterday = begin if begin else ( datetime.date.today() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") self.before_yesterday = ( datetime.datetime.strptime(str(self.yesterday), "%Y-%m-%d") + datetime.timedelta(days=-1)).strftime("%Y-%m-%d") self.date_7day_ago = ( datetime.datetime.strptime(str(self.yesterday), "%Y-%m-%d") + datetime.timedelta(days=-7)).strftime("%Y-%m-%d") self.ptid = ptid if ptid else '600025' print self.today print self.yesterday print self.ptid self.day_yesterday = self.yesterday.replace('-', '') #20100101 self.day_before_yesterday = self.getPlusDay(self.yesterday, -1) #20100101 self.day_7day_ago = self.getPlusDay(self.yesterday, -7) #20100101 #构建数据查询sql self.sql_channel_cnt = "select media_source,count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source;" self.sql_channel_ip_cnt = "select media_source,sum(cnt) from (select media_source,ip,count(distinct token) as cnt from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,ip having count(distinct token)>=5) group by media_source;" self.sql_channel_ip_imsi = "select media_source,country_iso,city,carrier,ip,split_part(imsi,'_',2) as imsi from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ;" self.sql_channel_imsi_cnt = "select media_source,sum(case when lenimsi=15 then 1 else 0 end) as cnt from (select distinct media_source,token,len(split_part(imsi,'_',2)) as lenimsi from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ) t group by media_source;" self.sql_channel_imei_cnt = "select media_source,sum(case when lenimei=15 then 1 else 0 end) as cnt from (select distinct media_source,token,len(imei) as lenimei from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ) t group by media_source;" self.sql_channel_hour_cnt = "select media_source,date_part(hour, install_time),count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,date_part(hour, install_time);" self.sql_channel_wifi_cnt = "select media_source,sum(case when wifi='t' then cnt else 0 end) as cnt from (select media_source,wifi,count(distinct token) as cnt from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,wifi) t group by media_source;" self.sql_channel_ver_cnt = "select media_source,ver,count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,ver;" self.sql_channel_device_cnt = "select media_source,device_brand,count(distinct token) from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null group by media_source,device_brand;" self.sql_channel_IPseg_cnt = "select media_source,sum(cnt) from (select media_source,ip,count(token) as cnt from (select distinct media_source,token,split_part(ip,'.',1)||'.'||split_part(ip,'.',2)||'.'||split_part(ip,'.',3) as ip from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null ) t group by media_source,ip having count(token)>=5) t2 group by media_source;" self.sql_channel_date = "select distinct media_source,'" + self.yesterday + "','" + self.ptid + "' from view_appsflyer_postback_ios_install_token where install_time>='" + self.yesterday + "' and install_time<'" + self.today + "' and product_id=" + self.ptid + " and token is not null;" #构建t分布预测模型 y = np.array([0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.98]) x = np.array([0.674, 0.842, 1.036, 1.282, 1.645, 1.960, 2.326]) x = x.reshape(x.shape[0], 1) X = self.buildComplexDataset(x) alpha = 0.001 l1_ratio = 0.001 enet = ElasticNet(alpha=alpha, l1_ratio=l1_ratio) self.model = enet.fit(X, y)
def ip_location(address): split_list = address.split('/') ip_address = split_list[2] location_dict = {} if ip_address: with database.Reader(os.getcwd() \ + '/geoip/GeoLite2-City.mmdb') as reader: response = reader.city(ip_address) location_dict['country_name'] = response.country.name location_dict['city_name'] = response.city.name location_dict['location_latitude'] = response.location.latitude location_dict['location_longitude'] = response.location.longitude return location_dict
def iplocation(self): try: if self.isIp(): IP = self.extractIp() else: IP = self.domain2ip() dbpath = os.path.join(GlobalConf().progpath['location'], 'Heaven_Hell/GeoLite2-City.mmdb') reader = ipdb.Reader(dbpath) locate = reader.city(IP) return locate except: return dict()
def ip2gps_partition(ips, db_path): from geoip2 import database def ip2gps(ip): try: response = reader.city(ip['ip']) city = Row(ip=ip['ip'], latitude=response.location.latitude, longitude=response.location.longitude) except Exception: city = Row(ip="-1", latitude=-1.0, longitude=-1.0) return city reader = database.Reader(SparkFiles.get(db_path)) return [ip2gps(ip) for ip in ips]