def test4_resolve(self):
r = redis.Redis(self.redisHost, self.redisPort, self.redisDB)
#resolve by coords
loc = City.getByLatLon(34.05223, -118.24368, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
#resolve by ip
ip = '4.3.68.1'
loc = IPRange.getCity(ip, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
#resolve zip by lat,lon
loc = ZIPCode.getByLatLon(34.0452, -118.284, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.name == '90006')
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
#resolve zip bu ip
loc = IPRange.getZIP(ip, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.name == '90001')
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
def test4_resolve(self):
r = redis.Redis(self.redisHost, self.redisPort, self.redisDB)
#resolve by coords
loc = City.getByLatLon(34.05223, -118.24368, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
#resolve by ip
ip = '4.3.68.1'
loc = IPRange.getCity(ip, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
#resolve zip by lat,lon
loc = ZIPCode.getByLatLon(34.0452, -118.284, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.name == '90006')
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
#resolve zip bu ip
loc = IPRange.getZIP(ip, r)
self.assertTrue(loc is not None)
self.assertTrue(loc.name == '90001')
self.assertTrue(loc.country == 'United States')
self.assertTrue(loc.state == 'CA' or loc.state == 'California')
def get(ip, redisConn):
"""
Get a range and all its data by ip
"""
ipnum = IPRange.ip2long(ip)
#get the location record from redis
record = redisConn.zrangebyscore(IPRange._indexKey, ipnum ,'+inf', 0, 1, True)
if not record:
#not found? k!
return None
#extract location id
try:
geoKey,rng = record[0][0].split('@')
lat,lon = hasher.decode(long(geoKey))
rngMin, rngMax, zipcode = rng.split(':')
rngMin = int(rngMin)
rngMax = int(rngMax)
except IndexError:
return None
#address not in any range
if not rngMin <= ipnum <= rngMax:
return None
return IPRange(rngMin, rngMax, lat, lon, zipcode)
class IP2LocationImporter(Importer):
def runImport(self, reset=False):
"""
File Format:
"67134976","67135231","US","UNITED STATES","CALIFORNIA","LOS ANGELES","34.045200","-118.284000","90001"
"""
if reset:
print "Deleting old ip data..."
self.redis.delete(IPRange._indexKey)
print "Starting import..."
try:
fp = open(self.fileName)
except Exception, e:
logging.error("could not open file %s for reading: %s" %
(self.fileName, e))
return False
reader = csv.reader(fp, delimiter=',', quotechar='"')
pipe = self.redis.pipeline()
i = 0
for row in reader:
try:
#parse the row
countryCode = row[3]
rangeMin = int(row[0])
rangeMax = int(row[1])
lat = float(row[6])
lon = float(row[7])
#take the zipcode if possible
try:
zipcode = row[8]
except:
zipcode = ''
#junk record
if countryCode == '-' and (not lat and not lon):
continue
range = IPRange(rangeMin, rangeMax, lat, lon, zipcode)
range.save(pipe)
except Exception, e:
logging.error("Could not save record: %s" % e)
i += 1
if i % 10000 == 0:
logging.info("Dumping pipe. did %d ranges" % i)
pipe.execute()
def getZIP(ip, redisConn):
"""
Get a zipcode location object based on an IP
will return None if you are outside the US
"""
range = IPRange.get(ip, redisConn)
if not range or not re.match('^[0-9]{5}$', range.zipcode):
return None
return ZIPCode.load('ZIPCode:%s' % range.zipcode, redisConn)
def getCity(ip, redisConn):
"""
Get location object by resolving an IP address
@param ip IPv4 address string (e.g. 127.0.0.1)
@oaram redisConn redis connection to the database
@return a Location object if we can resolve this ip, else None
"""
range = IPRange.get(ip, redisConn)
if not range:
return None
#load a location by the
return City.getByGeohash(hasher.encode(range.lat, range.lon), redisConn)
def get(ip, redisConn):
"""
Get a range and all its data by ip
"""
ipnum = IPRange.ip2long(ip)
#get the location record from redis
record = redisConn.zrangebyscore(IPLocation._indexKey, ipnum, '+inf',
0, 1, True)
if not record:
#not found? k!
return None
#extract location id
try:
#logging.info(record[0][0])
#logging.info(type (record[0][0]))
data = json.loads(record[0][0])
rngMin = int(data["rangeMin"])
rngMax = int(data["rangeMax"])
#geoKey,rng = record[0][0].split('@')
#lat,lon = hasher.decode(long(geoKey))
#rngMin, rngMax, zipcode = rng.split(':')
#rngMin = int(rngMin)
#rngMax = int(rngMax)
except IndexError:
return None
#address not in any range
if not rngMin <= ipnum <= rngMax:
return None
return "found: %s" % record
def resolveIP(ip):
global redis_host, redis_port, redis_db
r = redis.Redis(host = redis_host, port = redis_port, db = redis_db)
loc = IPRange.getZIP(ip, r)
print loc
def benchResolveIPs(num):
ips = ['209.85.238.11',
'209.85.238.4',
'216.239.33.96',
'216.239.33.97',
'216.239.33.98',
'216.239.33.99',
'216.239.37.98',
'216.239.37.99',
'216.239.39.98',
'216.239.39.99',
'216.239.41.96',
'216.239.41.97',
'216.239.41.98',
'216.239.41.99',
'216.239.45.4',
'216.239.51.96',
'216.239.51.97',
'216.239.51.98',
'216.239.51.99',
'216.239.53.98',
'216.239.53.99',
'216.239.57.96',
'216.239.57.97',
'216.239.57.98',
'216.239.57.99',
'216.239.59.98',
'216.239.59.99',
'216.33.229.163',
'64.233.173.193',
'64.233.173.194',
'64.233.173.195',
'64.233.173.196',
'64.233.173.197',
'64.233.173.198',
'64.233.173.199',
'64.233.173.200',
'64.233.173.201',
'64.233.173.202',
'64.233.173.203',
'64.233.173.204',
'64.233.173.205',
'64.233.173.206',
'64.233.173.207',
'64.233.173.208',
'64.233.173.209',
'64.233.173.210',
'64.233.173.211',
'64.233.173.212',
'64.233.173.213',
'64.233.173.214',
'64.233.173.215',
'64.233.173.216',
'64.233.173.217',
'64.233.173.218',
'64.233.173.219',
'64.233.173.220',
'64.233.173.221',
'64.233.173.222',
'64.233.173.223',
'64.233.173.224',
'64.233.173.225',
'64.233.173.226',
'64.233.173.227',
'64.233.173.228',
'64.233.173.229',
'64.233.173.230',
'64.233.173.231',
'64.233.173.232',
'64.233.173.233',
'64.233.173.234',
'64.233.173.235',
'64.233.173.236',
'64.233.173.237',
'64.233.173.238',
'64.233.173.239',
'64.233.173.240',
'64.233.173.241',
'64.233.173.242',
'64.233.173.243',
'64.233.173.244',
'64.233.173.245',
'64.233.173.246',
'64.233.173.247',
'64.233.173.248',
'64.233.173.249',
'64.233.173.250',
'64.233.173.251',
'64.233.173.252',
'64.233.173.253',
'64.233.173.254',
'64.233.173.255',
'64.68.90.1',
'64.68.90.10',
'64.68.90.11',
'64.68.90.12',
'64.68.90.129',
'64.68.90.13',
'64.68.90.130',
'64.68.90.131',
'64.68.90.132',
'64.68.90.133',
'64.68.90.134',
'64.68.90.135',
'64.68.90.136',
'64.68.90.137',
'64.68.90.138',
'64.68.90.139',
'64.68.90.14',
'64.68.90.140',
'64.68.90.141',
'64.68.90.142',
'64.68.90.143',
'64.68.90.144',
'64.68.90.145',
'64.68.90.146',
'64.68.90.147',
'64.68.90.148',
'64.68.90.149',
'64.68.90.15',
'64.68.90.150',
'64.68.90.151',
'64.68.90.152',
'64.68.90.153',
'64.68.90.154',
'64.68.90.155',
'64.68.90.156',
'64.68.90.157',
'64.68.90.158',
'64.68.90.159',
'64.68.90.16',
'64.68.90.160',
'64.68.90.161',
'64.68.90.162',
'64.68.90.163',
'64.68.90.164',
'64.68.90.165',
'64.68.90.166',
'64.68.90.167',
'64.68.90.168',
'64.68.90.169',
'64.68.90.17',
'64.68.90.170',
'64.68.90.171',
'64.68.90.172',
'64.68.90.173',
'64.68.90.174',
'64.68.90.175',
'64.68.90.176',
'64.68.90.177',
'64.68.90.178',
'64.68.90.179',
'64.68.90.18',
'64.68.90.180',
'64.68.90.181',
'64.68.90.182',
'64.68.90.183',
'64.68.90.184',
'64.68.90.185',
'64.68.90.186',
'64.68.90.187',
'64.68.90.188',
'64.68.90.189',
'64.68.90.19',
'64.68.90.190',
'64.68.90.191',
'64.68.90.192',
'64.68.90.193',
'64.68.90.194',
'64.68.90.195',
'64.68.90.196',
'64.68.90.197',
'64.68.90.198',
'64.68.90.199',
'64.68.90.2',
'64.68.90.20',
'64.68.90.200',
'64.68.90.201',
'64.68.90.202',
'64.68.90.203',
'64.68.90.204',
'64.68.90.205',
'64.68.90.206',
'64.68.90.207',
'64.68.90.208',
'64.68.90.21',
'64.68.90.22',
'64.68.90.23',
'64.68.90.24',
'64.68.90.25',
'64.68.90.26',
'64.68.90.27',
'64.68.90.28',
'64.68.90.29',
'64.68.90.3',
'64.68.90.30',
'64.68.90.31',
'64.68.90.32',
'64.68.90.33',
'64.68.90.34',
'64.68.90.35',
'64.68.90.36',
'64.68.90.37',
'64.68.90.38',
'64.68.90.39',
'64.68.90.4',
'64.68.90.40',
'64.68.90.41',
'64.68.90.42',
'64.68.90.43',
'64.68.90.44',
'64.68.90.45',
'64.68.90.46',
'64.68.90.47',
'64.68.90.48',
'64.68.90.49',
'64.68.90.5',
'64.68.90.50',
'64.68.90.51',
'64.68.90.52',
'64.68.90.53',
'64.68.90.54',
'64.68.90.55',
'64.68.90.56',
'64.68.90.57',
'64.68.90.58',
'64.68.90.59',
'64.68.90.6',
'64.68.90.60',
'64.68.90.61',
'64.68.90.62',
'64.68.90.63',
'64.68.90.64',
'64.68.90.65',
'64.68.90.66',
'64.68.90.67',
'64.68.90.68',
'64.68.90.69',
'64.68.90.7',
'64.68.90.70',
'64.68.90.71',
'64.68.90.72',
'64.68.90.73',
'64.68.90.74',
'64.68.90.75',
'64.68.90.76',
'64.68.90.77',
'64.68.90.78',
'64.68.90.79',
'64.68.90.8',
'64.68.90.80',
'64.68.90.9']
#ips = ['166.205.138.92', '62.0.18.221', '69.147.125.65', '188.127.241.156', '79.178.26.33']
r = redis.Redis()
nips = len(ips)
for i in xrange(num):
ip = ips[i % nips]
loc = IPRange.getCity(ip, r)
return num
def benchResolveIPs(num):
ips = [
'209.85.238.11', '209.85.238.4', '216.239.33.96', '216.239.33.97',
'216.239.33.98', '216.239.33.99', '216.239.37.98', '216.239.37.99',
'216.239.39.98', '216.239.39.99', '216.239.41.96', '216.239.41.97',
'216.239.41.98', '216.239.41.99', '216.239.45.4', '216.239.51.96',
'216.239.51.97', '216.239.51.98', '216.239.51.99', '216.239.53.98',
'216.239.53.99', '216.239.57.96', '216.239.57.97', '216.239.57.98',
'216.239.57.99', '216.239.59.98', '216.239.59.99', '216.33.229.163',
'64.233.173.193', '64.233.173.194', '64.233.173.195', '64.233.173.196',
'64.233.173.197', '64.233.173.198', '64.233.173.199', '64.233.173.200',
'64.233.173.201', '64.233.173.202', '64.233.173.203', '64.233.173.204',
'64.233.173.205', '64.233.173.206', '64.233.173.207', '64.233.173.208',
'64.233.173.209', '64.233.173.210', '64.233.173.211', '64.233.173.212',
'64.233.173.213', '64.233.173.214', '64.233.173.215', '64.233.173.216',
'64.233.173.217', '64.233.173.218', '64.233.173.219', '64.233.173.220',
'64.233.173.221', '64.233.173.222', '64.233.173.223', '64.233.173.224',
'64.233.173.225', '64.233.173.226', '64.233.173.227', '64.233.173.228',
'64.233.173.229', '64.233.173.230', '64.233.173.231', '64.233.173.232',
'64.233.173.233', '64.233.173.234', '64.233.173.235', '64.233.173.236',
'64.233.173.237', '64.233.173.238', '64.233.173.239', '64.233.173.240',
'64.233.173.241', '64.233.173.242', '64.233.173.243', '64.233.173.244',
'64.233.173.245', '64.233.173.246', '64.233.173.247', '64.233.173.248',
'64.233.173.249', '64.233.173.250', '64.233.173.251', '64.233.173.252',
'64.233.173.253', '64.233.173.254', '64.233.173.255', '64.68.90.1',
'64.68.90.10', '64.68.90.11', '64.68.90.12', '64.68.90.129',
'64.68.90.13', '64.68.90.130', '64.68.90.131', '64.68.90.132',
'64.68.90.133', '64.68.90.134', '64.68.90.135', '64.68.90.136',
'64.68.90.137', '64.68.90.138', '64.68.90.139', '64.68.90.14',
'64.68.90.140', '64.68.90.141', '64.68.90.142', '64.68.90.143',
'64.68.90.144', '64.68.90.145', '64.68.90.146', '64.68.90.147',
'64.68.90.148', '64.68.90.149', '64.68.90.15', '64.68.90.150',
'64.68.90.151', '64.68.90.152', '64.68.90.153', '64.68.90.154',
'64.68.90.155', '64.68.90.156', '64.68.90.157', '64.68.90.158',
'64.68.90.159', '64.68.90.16', '64.68.90.160', '64.68.90.161',
'64.68.90.162', '64.68.90.163', '64.68.90.164', '64.68.90.165',
'64.68.90.166', '64.68.90.167', '64.68.90.168', '64.68.90.169',
'64.68.90.17', '64.68.90.170', '64.68.90.171', '64.68.90.172',
'64.68.90.173', '64.68.90.174', '64.68.90.175', '64.68.90.176',
'64.68.90.177', '64.68.90.178', '64.68.90.179', '64.68.90.18',
'64.68.90.180', '64.68.90.181', '64.68.90.182', '64.68.90.183',
'64.68.90.184', '64.68.90.185', '64.68.90.186', '64.68.90.187',
'64.68.90.188', '64.68.90.189', '64.68.90.19', '64.68.90.190',
'64.68.90.191', '64.68.90.192', '64.68.90.193', '64.68.90.194',
'64.68.90.195', '64.68.90.196', '64.68.90.197', '64.68.90.198',
'64.68.90.199', '64.68.90.2', '64.68.90.20', '64.68.90.200',
'64.68.90.201', '64.68.90.202', '64.68.90.203', '64.68.90.204',
'64.68.90.205', '64.68.90.206', '64.68.90.207', '64.68.90.208',
'64.68.90.21', '64.68.90.22', '64.68.90.23', '64.68.90.24',
'64.68.90.25', '64.68.90.26', '64.68.90.27', '64.68.90.28',
'64.68.90.29', '64.68.90.3', '64.68.90.30', '64.68.90.31',
'64.68.90.32', '64.68.90.33', '64.68.90.34', '64.68.90.35',
'64.68.90.36', '64.68.90.37', '64.68.90.38', '64.68.90.39',
'64.68.90.4', '64.68.90.40', '64.68.90.41', '64.68.90.42',
'64.68.90.43', '64.68.90.44', '64.68.90.45', '64.68.90.46',
'64.68.90.47', '64.68.90.48', '64.68.90.49', '64.68.90.5',
'64.68.90.50', '64.68.90.51', '64.68.90.52', '64.68.90.53',
'64.68.90.54', '64.68.90.55', '64.68.90.56', '64.68.90.57',
'64.68.90.58', '64.68.90.59', '64.68.90.6', '64.68.90.60',
'64.68.90.61', '64.68.90.62', '64.68.90.63', '64.68.90.64',
'64.68.90.65', '64.68.90.66', '64.68.90.67', '64.68.90.68',
'64.68.90.69', '64.68.90.7', '64.68.90.70', '64.68.90.71',
'64.68.90.72', '64.68.90.73', '64.68.90.74', '64.68.90.75',
'64.68.90.76', '64.68.90.77', '64.68.90.78', '64.68.90.79',
'64.68.90.8', '64.68.90.80', '64.68.90.9'
]
#ips = ['166.205.138.92', '62.0.18.221', '69.147.125.65', '188.127.241.156', '79.178.26.33']
r = redis.Redis()
nips = len(ips)
for i in xrange(num):
ip = ips[i % nips]
loc = IPRange.getCity(ip, r)
return num
def resolveIP(ip):
global redis_host, redis_port, redis_db
r = redis.Redis(host=redis_host, port=redis_port, db=redis_db)
loc = IPRange.getZIP(ip, r)
print loc
def resolveIP(ip):
global redis_host, redis_port, redis_db
loc = IPRange.get(ip, r)
print loc
rangeMin = int(row[0])
rangeMax = int(row[1])
lat = float(location['latitude'])
lon = float(location['longitude'])
#take the zipcode if possible
try:
zipcode = location['postalCode']
except:
zipcode = ''
#junk record
if countryCode == '-' and (not lat and not lon):
continue
rng = IPRange(rangeMin, rangeMax, lat, lon, zipcode)
rng.save(pipe)
except Exception, e:
logging.error("Could not save record: %s" % e)
i += 1
if i % 10000 == 0:
logging.info("Dumping pipe. did %d ranges" % i)
pipe.execute()
pipe.execute()
logging.info("Imported %d locations" % i)
return i