def test_old_reader(self): reader = maxminddb.Reader( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb') record = reader.get('::1.1.1.0') self.assertEqual(record['array'], [1, 2, 3]) reader.close()
def __init__(self, patharg, ip=[], ua=[]): dbpath = "%s" % os.path.realpath(patharg) prepare = not os.path.exists(dbpath) self.db = dbapi.connect(dbpath) self.ip = set(ip) self.ua = set(ua) self.geo = maxminddb.Reader(sys.path[0] + '/GeoLite2-City.mmdb') if prepare: for sql in DBSCHEMA: self.db.execute(sql) cur = self.db.execute("pragma user_version") # migration if cur.fetchone()[0] < 1: for sql in MIGRATION_1: self.db.execute(sql) cur = self.db.execute("select distinct ip from listener") while True: res = cur.fetchmany(1000) if not res: break for ip in res: iso, country = self.geoip(ip[0]) self.db.execute( "update listener set iso_code = ? , country = ?" "where ip = ?", (iso, country, ip[0])) self.db.commit()
def test_old_reader(self): reader = maxminddb.Reader( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb") record = reader.get("::1.1.1.0") self.assertEqual(record["array"], [1, 2, 3]) reader.close()
def get_ip_location(remote_addr): """Gets the location of the specified remote_addr :param remote_addr: the ip address to get the location of :returns: the location as a dictionary """ # this code may need to be rethought for scalability, but for now, # we're going with just opening a reader each time we get a location location = { 'country': None, 'state': None, 'city': None, 'zip': None, 'lat': None, 'lon': None } reader = maxminddb.Reader(app.config['MMDB_PATH']) mmloc = reader.get(remote_addr) if (mmloc is not None): if ('country' in mmloc): location['country'] = mmloc['country']['names']['en'] if ('subdivisions' in mmloc): location['state'] = mmloc['subdivisions'][0]['names']['en'] if ('city' in mmloc): location['city'] = mmloc['city']['names']['en'] if ('postal' in mmloc): location['zip'] = mmloc['postal']['code'] if ('location' in mmloc): location['lat'] = mmloc['location']['latitude'] location['lon'] = mmloc['location']['longitude'] reader.close() return location
def ipInfo(ip): readerCity = maxminddb.Reader(GeoLit2_city) cityInfo = readerCity.get(ip) # print("\033[1;33;40m") print("% s 's Geo information:" % ip) print('中国'.encode('gbk').decode('gb2312')) # print(cityInfo.encode('utf-8').decode('gb2312')) # print("\033[0m") print(json.dumps(cityInfo, sort_keys=True, indent=2, ensure_ascii=False)) readerCity.close()
def load_mmdb(db_path): reader = maxminddb.Reader(db_path) def fetch_geo_data(ip): data = reader.get(ip) return { 'country_code': data['country']['iso_code'].lower(), 'country_name': data['country']['names']['en'], } return fetch_geo_data
def __init__(self, filename, locales=None): if locales is None: locales = ['en'] self.db_reader = maxminddb.Reader(filename) self.locales = locales
'The comma separated list of schemas to run the query on, defaults to values in configuration file or coursera_index table' ) parser.add_argument( '--sql', action='store', required="True", help= 'The query to run, must return list of IP addresses. E.g. SELECT DISTINCT last_access_ip FROM users WHERE last_access_ip IS NOT NULL AND last_access_ip NOT LIKE ""' ) args = parser.parse_args() logger = get_logger("geolocate.py", args.verbose) geolitedb = "GeoLite2-City.mmdb" try: reader = maxminddb.Reader(geolitedb) except: logger.error("File {} not found".format(geolitedb)) sys.exit() conn = get_connection() if (args.schemas != None): schemas = args.schemas.split(",") else: schemas = get_coursera_schema_list() if (args.clean): query = "DROP TABLE IF EXISTS coursera_geolocate" try: conn.execute(query)
#! /bin/env python # This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com # Date: 2014-01-30 import os, sys, string, struct, socket, threading, time, re, maxminddb, MySQLdb sys.path.append("/home/kun/mnt") from webcrawl import * ISOTIMEFORMAT = '%Y-%m-%d %X' #geo = geoip2.database.Reader(os.path.abspath('.')+os.path.sep+'GeoLite2-City.mmdb') reader = maxminddb.Reader('/home/kun/topology/GeoLite2-City.mmdb') pm1 = MySQLdb.connect(host='127.0.0.1', user='******', db='webserver') cur1 = pm1.cursor() cnt = cur1.execute("select distinct ip from mnt.proximity_rtt") result = cur1.fetchall() for ip, in result: asn = ip2asn(ip) start = time.time() # ip -> prefix -> web server cmd = "select webdomain from ipv4prefix2vantage where start<={0} and end>={0} and asn='{1}'".format( struct.unpack("!I", socket.inet_aton(ip))[0], asn[2:]) cur1.execute(cmd) row = cur1.fetchone() end = time.time() - start print end continue ''' start = time.time() record = reader.get(ip) if(record is None):
# of the Redis list every time. counter += 1 if args.benchmark: if not counter % 1000: pipe.execute() else: if counter >= 1000: pipe.execute() counter = 0 statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS)) if __name__ == '__main__': register_signals(handle_signals) try: geo = maxminddb.Reader(args.file) except IOError: log.error('ERROR: Can\'t find MaxMind Database file (%s). ' 'Try setting the --file flag.' % args.file) sys.exit(1) if args.benchmark: bench_count = redis.llen(rkeys.IPLOGS) if not bench_count: print 'No IPs to process' sys.exit(1) print 'Starting benchmark of {} records'.format(bench_count) bench_start = time.time() return_code = main()