def test_old_reader(self):
reader = maxminddb.Reader(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb')
record = reader.get('::1.1.1.0')
self.assertEqual(record['array'], [1, 2, 3])
reader.close()
def __init__(self, patharg, ip=[], ua=[]):
dbpath = "%s" % os.path.realpath(patharg)
prepare = not os.path.exists(dbpath)
self.db = dbapi.connect(dbpath)
self.ip = set(ip)
self.ua = set(ua)
self.geo = maxminddb.Reader(sys.path[0] + '/GeoLite2-City.mmdb')
if prepare:
for sql in DBSCHEMA:
self.db.execute(sql)
cur = self.db.execute("pragma user_version")
# migration
if cur.fetchone()[0] < 1:
for sql in MIGRATION_1:
self.db.execute(sql)
cur = self.db.execute("select distinct ip from listener")
while True:
res = cur.fetchmany(1000)
if not res:
break
for ip in res:
iso, country = self.geoip(ip[0])
self.db.execute(
"update listener set iso_code = ? , country = ?"
"where ip = ?", (iso, country, ip[0]))
self.db.commit()
def test_old_reader(self):
reader = maxminddb.Reader(
"tests/data/test-data/MaxMind-DB-test-decoder.mmdb")
record = reader.get("::1.1.1.0")
self.assertEqual(record["array"], [1, 2, 3])
reader.close()
def get_ip_location(remote_addr):
"""Gets the location of the specified remote_addr
:param remote_addr: the ip address to get the location of
:returns: the location as a dictionary
"""
# this code may need to be rethought for scalability, but for now,
# we're going with just opening a reader each time we get a location
location = {
'country': None,
'state': None,
'city': None,
'zip': None,
'lat': None,
'lon': None
}
reader = maxminddb.Reader(app.config['MMDB_PATH'])
mmloc = reader.get(remote_addr)
if (mmloc is not None):
if ('country' in mmloc):
location['country'] = mmloc['country']['names']['en']
if ('subdivisions' in mmloc):
location['state'] = mmloc['subdivisions'][0]['names']['en']
if ('city' in mmloc):
location['city'] = mmloc['city']['names']['en']
if ('postal' in mmloc):
location['zip'] = mmloc['postal']['code']
if ('location' in mmloc):
location['lat'] = mmloc['location']['latitude']
location['lon'] = mmloc['location']['longitude']
reader.close()
return location
def ipInfo(ip):
readerCity = maxminddb.Reader(GeoLit2_city)
cityInfo = readerCity.get(ip)
# print("\033[1;33;40m")
print("% s 's Geo information:" % ip)
print('中国'.encode('gbk').decode('gb2312'))
# print(cityInfo.encode('utf-8').decode('gb2312'))
# print("\033[0m")
print(json.dumps(cityInfo, sort_keys=True, indent=2, ensure_ascii=False))
readerCity.close()
def load_mmdb(db_path):
reader = maxminddb.Reader(db_path)
def fetch_geo_data(ip):
data = reader.get(ip)
return {
'country_code': data['country']['iso_code'].lower(),
'country_name': data['country']['names']['en'],
}
return fetch_geo_data
def __init__(self, filename, locales=None):
if locales is None:
locales = ['en']
self.db_reader = maxminddb.Reader(filename)
self.locales = locales
'The comma separated list of schemas to run the query on, defaults to values in configuration file or coursera_index table'
)
parser.add_argument(
'--sql',
action='store',
required="True",
help=
'The query to run, must return list of IP addresses. E.g. SELECT DISTINCT last_access_ip FROM users WHERE last_access_ip IS NOT NULL AND last_access_ip NOT LIKE ""'
)
args = parser.parse_args()
logger = get_logger("geolocate.py", args.verbose)
geolitedb = "GeoLite2-City.mmdb"
try:
reader = maxminddb.Reader(geolitedb)
except:
logger.error("File {} not found".format(geolitedb))
sys.exit()
conn = get_connection()
if (args.schemas != None):
schemas = args.schemas.split(",")
else:
schemas = get_coursera_schema_list()
if (args.clean):
query = "DROP TABLE IF EXISTS coursera_geolocate"
try:
conn.execute(query)
#! /bin/env python
# This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com
# Date: 2014-01-30
import os, sys, string, struct, socket, threading, time, re, maxminddb, MySQLdb
sys.path.append("/home/kun/mnt")
from webcrawl import *
ISOTIMEFORMAT = '%Y-%m-%d %X'
#geo = geoip2.database.Reader(os.path.abspath('.')+os.path.sep+'GeoLite2-City.mmdb')
reader = maxminddb.Reader('/home/kun/topology/GeoLite2-City.mmdb')
pm1 = MySQLdb.connect(host='127.0.0.1', user='******', db='webserver')
cur1 = pm1.cursor()
cnt = cur1.execute("select distinct ip from mnt.proximity_rtt")
result = cur1.fetchall()
for ip, in result:
asn = ip2asn(ip)
start = time.time()
# ip -> prefix -> web server
cmd = "select webdomain from ipv4prefix2vantage where start<={0} and end>={0} and asn='{1}'".format(
struct.unpack("!I", socket.inet_aton(ip))[0], asn[2:])
cur1.execute(cmd)
row = cur1.fetchone()
end = time.time() - start
print end
continue
'''
start = time.time()
record = reader.get(ip)
if(record is None):
# of the Redis list every time.
counter += 1
if args.benchmark:
if not counter % 1000:
pipe.execute()
else:
if counter >= 1000:
pipe.execute()
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
if __name__ == '__main__':
register_signals(handle_signals)
try:
geo = maxminddb.Reader(args.file)
except IOError:
log.error('ERROR: Can\'t find MaxMind Database file (%s). '
'Try setting the --file flag.' % args.file)
sys.exit(1)
if args.benchmark:
bench_count = redis.llen(rkeys.IPLOGS)
if not bench_count:
print 'No IPs to process'
sys.exit(1)
print 'Starting benchmark of {} records'.format(bench_count)
bench_start = time.time()
return_code = main()
