Beispiel #1
0
    def test4_resolve(self):
        r = redis.Redis(self.redisHost, self.redisPort, self.redisDB)
        #resolve by coords

        loc = City.getByLatLon(34.05223, -118.24368, r)

        self.assertTrue(loc is not None)
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')

        #resolve by ip
        ip = '4.3.68.1'

        loc = IPRange.getCity(ip, r)

        self.assertTrue(loc is not None)
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')

        #resolve zip by lat,lon
        loc = ZIPCode.getByLatLon(34.0452, -118.284, r)

        self.assertTrue(loc is not None)
        self.assertTrue(loc.name == '90006')
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')

        #resolve zip bu ip
        loc = IPRange.getZIP(ip, r)
        self.assertTrue(loc is not None)
        self.assertTrue(loc.name == '90001')
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')
Beispiel #2
0
    def test4_resolve(self):
        r = redis.Redis(self.redisHost, self.redisPort, self.redisDB)
        #resolve by coords
        
        loc = City.getByLatLon(34.05223, -118.24368, r)
        
        self.assertTrue(loc is not None)
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')
        
        #resolve by ip
        ip = '4.3.68.1'

        loc = IPRange.getCity(ip, r)
        
        self.assertTrue(loc is not None)
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')

        #resolve zip by lat,lon
        loc = ZIPCode.getByLatLon(34.0452, -118.284, r)
        
        self.assertTrue(loc is not None)
        self.assertTrue(loc.name == '90006')
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')

        #resolve zip bu ip
        loc = IPRange.getZIP(ip, r)
        self.assertTrue(loc is not None)
        self.assertTrue(loc.name == '90001')
        self.assertTrue(loc.country == 'United States')
        self.assertTrue(loc.state == 'CA' or loc.state == 'California')
Beispiel #3
0
    def get(ip, redisConn):
        """
        Get a range and all its data by ip
        """
        
        ipnum = IPRange.ip2long(ip)

        #get the location record from redis
        record = redisConn.zrangebyscore(IPRange._indexKey, ipnum ,'+inf', 0, 1, True)
        if not record:
            #not found? k!
            return None

        #extract location id
        try:
            geoKey,rng = record[0][0].split('@')
            
            lat,lon = hasher.decode(long(geoKey))
            
            rngMin, rngMax, zipcode =  rng.split(':')
            rngMin = int(rngMin)
            rngMax = int(rngMax)
        except IndexError:
            return None

        #address not in any range
        if not rngMin <= ipnum <= rngMax:
            return None
        
        return IPRange(rngMin, rngMax, lat, lon, zipcode)
Beispiel #4
0
class IP2LocationImporter(Importer):
    def runImport(self, reset=False):
        """
        File Format:
        "67134976","67135231","US","UNITED STATES","CALIFORNIA","LOS ANGELES","34.045200","-118.284000","90001"

        """
        if reset:
            print "Deleting old ip data..."
            self.redis.delete(IPRange._indexKey)

        print "Starting import..."

        try:
            fp = open(self.fileName)
        except Exception, e:
            logging.error("could not open file %s for reading: %s" %
                          (self.fileName, e))
            return False

        reader = csv.reader(fp, delimiter=',', quotechar='"')
        pipe = self.redis.pipeline()

        i = 0
        for row in reader:

            try:
                #parse the row
                countryCode = row[3]
                rangeMin = int(row[0])
                rangeMax = int(row[1])
                lat = float(row[6])
                lon = float(row[7])

                #take the zipcode if possible
                try:
                    zipcode = row[8]
                except:
                    zipcode = ''

                #junk record
                if countryCode == '-' and (not lat and not lon):
                    continue

                range = IPRange(rangeMin, rangeMax, lat, lon, zipcode)
                range.save(pipe)

            except Exception, e:
                logging.error("Could not save record: %s" % e)

            i += 1
            if i % 10000 == 0:
                logging.info("Dumping pipe. did %d ranges" % i)
                pipe.execute()
Beispiel #5
0
    def getZIP(ip, redisConn):
        """
        Get a zipcode location object based on an IP
        will return None if you are outside the US
        """

        range = IPRange.get(ip, redisConn)
        
        if not range or not re.match('^[0-9]{5}$', range.zipcode):
            return None

        return ZIPCode.load('ZIPCode:%s' % range.zipcode, redisConn)
Beispiel #6
0
    def getCity(ip, redisConn):
        """
        Get location object by resolving an IP address
        @param ip IPv4 address string (e.g. 127.0.0.1)
        @oaram redisConn redis connection to the database
        @return a Location object if we can resolve this ip, else None
        """

        range = IPRange.get(ip, redisConn)
        if not range:
            return None

        

        #load a location by the
        return City.getByGeohash(hasher.encode(range.lat, range.lon), redisConn)
Beispiel #7
0
    def get(ip, redisConn):
        """
        Get a range and all its data by ip
        """

        ipnum = IPRange.ip2long(ip)

        #get the location record from redis
        record = redisConn.zrangebyscore(IPLocation._indexKey, ipnum, '+inf',
                                         0, 1, True)
        if not record:
            #not found? k!
            return None

        #extract location id
        try:
            #logging.info(record[0][0])

            #logging.info(type (record[0][0]))
            data = json.loads(record[0][0])

            rngMin = int(data["rangeMin"])
            rngMax = int(data["rangeMax"])
            #geoKey,rng = record[0][0].split('@')

            #lat,lon = hasher.decode(long(geoKey))

            #rngMin, rngMax, zipcode =  rng.split(':')
            #rngMin = int(rngMin)
            #rngMax = int(rngMax)
        except IndexError:
            return None

        #address not in any range
        if not rngMin <= ipnum <= rngMax:
            return None

        return "found: %s" % record
Beispiel #8
0
def resolveIP(ip):
    global redis_host, redis_port, redis_db
    r = redis.Redis(host = redis_host, port = redis_port, db = redis_db)

    loc = IPRange.getZIP(ip, r)
    print loc
Beispiel #9
0
def benchResolveIPs(num):
    ips = ['209.85.238.11',
    '209.85.238.4',
    '216.239.33.96',
    '216.239.33.97',
    '216.239.33.98',
    '216.239.33.99',
    '216.239.37.98',
    '216.239.37.99',
    '216.239.39.98',
    '216.239.39.99',
    '216.239.41.96',
    '216.239.41.97',
    '216.239.41.98',
    '216.239.41.99',
    '216.239.45.4',
    '216.239.51.96',
    '216.239.51.97',
    '216.239.51.98',
    '216.239.51.99',
    '216.239.53.98',
    '216.239.53.99',
    '216.239.57.96',
    '216.239.57.97',
    '216.239.57.98',
    '216.239.57.99',
    '216.239.59.98',
    '216.239.59.99',
    '216.33.229.163',
    '64.233.173.193',
    '64.233.173.194',
    '64.233.173.195',
    '64.233.173.196',
    '64.233.173.197',
    '64.233.173.198',
    '64.233.173.199',
    '64.233.173.200',
    '64.233.173.201',
    '64.233.173.202',
    '64.233.173.203',
    '64.233.173.204',
    '64.233.173.205',
    '64.233.173.206',
    '64.233.173.207',
    '64.233.173.208',
    '64.233.173.209',
    '64.233.173.210',
    '64.233.173.211',
    '64.233.173.212',
    '64.233.173.213',
    '64.233.173.214',
    '64.233.173.215',
    '64.233.173.216',
    '64.233.173.217',
    '64.233.173.218',
    '64.233.173.219',
    '64.233.173.220',
    '64.233.173.221',
    '64.233.173.222',
    '64.233.173.223',
    '64.233.173.224',
    '64.233.173.225',
    '64.233.173.226',
    '64.233.173.227',
    '64.233.173.228',
    '64.233.173.229',
    '64.233.173.230',
    '64.233.173.231',
    '64.233.173.232',
    '64.233.173.233',
    '64.233.173.234',
    '64.233.173.235',
    '64.233.173.236',
    '64.233.173.237',
    '64.233.173.238',
    '64.233.173.239',
    '64.233.173.240',
    '64.233.173.241',
    '64.233.173.242',
    '64.233.173.243',
    '64.233.173.244',
    '64.233.173.245',
    '64.233.173.246',
    '64.233.173.247',
    '64.233.173.248',
    '64.233.173.249',
    '64.233.173.250',
    '64.233.173.251',
    '64.233.173.252',
    '64.233.173.253',
    '64.233.173.254',
    '64.233.173.255',
    '64.68.90.1',
    '64.68.90.10',
    '64.68.90.11',
    '64.68.90.12',
    '64.68.90.129',
    '64.68.90.13',
    '64.68.90.130',
    '64.68.90.131',
    '64.68.90.132',
    '64.68.90.133',
    '64.68.90.134',
    '64.68.90.135',
    '64.68.90.136',
    '64.68.90.137',
    '64.68.90.138',
    '64.68.90.139',
    '64.68.90.14',
    '64.68.90.140',
    '64.68.90.141',
    '64.68.90.142',
    '64.68.90.143',
    '64.68.90.144',
    '64.68.90.145',
    '64.68.90.146',
    '64.68.90.147',
    '64.68.90.148',
    '64.68.90.149',
    '64.68.90.15',
    '64.68.90.150',
    '64.68.90.151',
    '64.68.90.152',
    '64.68.90.153',
    '64.68.90.154',
    '64.68.90.155',
    '64.68.90.156',
    '64.68.90.157',
    '64.68.90.158',
    '64.68.90.159',
    '64.68.90.16',
    '64.68.90.160',
    '64.68.90.161',
    '64.68.90.162',
    '64.68.90.163',
    '64.68.90.164',
    '64.68.90.165',
    '64.68.90.166',
    '64.68.90.167',
    '64.68.90.168',
    '64.68.90.169',
    '64.68.90.17',
    '64.68.90.170',
    '64.68.90.171',
    '64.68.90.172',
    '64.68.90.173',
    '64.68.90.174',
    '64.68.90.175',
    '64.68.90.176',
    '64.68.90.177',
    '64.68.90.178',
    '64.68.90.179',
    '64.68.90.18',
    '64.68.90.180',
    '64.68.90.181',
    '64.68.90.182',
    '64.68.90.183',
    '64.68.90.184',
    '64.68.90.185',
    '64.68.90.186',
    '64.68.90.187',
    '64.68.90.188',
    '64.68.90.189',
    '64.68.90.19',
    '64.68.90.190',
    '64.68.90.191',
    '64.68.90.192',
    '64.68.90.193',
    '64.68.90.194',
    '64.68.90.195',
    '64.68.90.196',
    '64.68.90.197',
    '64.68.90.198',
    '64.68.90.199',
    '64.68.90.2',
    '64.68.90.20',
    '64.68.90.200',
    '64.68.90.201',
    '64.68.90.202',
    '64.68.90.203',
    '64.68.90.204',
    '64.68.90.205',
    '64.68.90.206',
    '64.68.90.207',
    '64.68.90.208',
    '64.68.90.21',
    '64.68.90.22',
    '64.68.90.23',
    '64.68.90.24',
    '64.68.90.25',
    '64.68.90.26',
    '64.68.90.27',
    '64.68.90.28',
    '64.68.90.29',
    '64.68.90.3',
    '64.68.90.30',
    '64.68.90.31',
    '64.68.90.32',
    '64.68.90.33',
    '64.68.90.34',
    '64.68.90.35',
    '64.68.90.36',
    '64.68.90.37',
    '64.68.90.38',
    '64.68.90.39',
    '64.68.90.4',
    '64.68.90.40',
    '64.68.90.41',
    '64.68.90.42',
    '64.68.90.43',
    '64.68.90.44',
    '64.68.90.45',
    '64.68.90.46',
    '64.68.90.47',
    '64.68.90.48',
    '64.68.90.49',
    '64.68.90.5',
    '64.68.90.50',
    '64.68.90.51',
    '64.68.90.52',
    '64.68.90.53',
    '64.68.90.54',
    '64.68.90.55',
    '64.68.90.56',
    '64.68.90.57',
    '64.68.90.58',
    '64.68.90.59',
    '64.68.90.6',
    '64.68.90.60',
    '64.68.90.61',
    '64.68.90.62',
    '64.68.90.63',
    '64.68.90.64',
    '64.68.90.65',
    '64.68.90.66',
    '64.68.90.67',
    '64.68.90.68',
    '64.68.90.69',
    '64.68.90.7',
    '64.68.90.70',
    '64.68.90.71',
    '64.68.90.72',
    '64.68.90.73',
    '64.68.90.74',
    '64.68.90.75',
    '64.68.90.76',
    '64.68.90.77',
    '64.68.90.78',
    '64.68.90.79',
    '64.68.90.8',
    '64.68.90.80',
    '64.68.90.9']

    #ips = ['166.205.138.92', '62.0.18.221',  '69.147.125.65', '188.127.241.156', '79.178.26.33']
    r = redis.Redis()
    nips = len(ips)
    for i in xrange(num):
        ip = ips[i % nips]
        loc = IPRange.getCity(ip, r)
        
    return num
Beispiel #10
0
def benchResolveIPs(num):
    ips = [
        '209.85.238.11', '209.85.238.4', '216.239.33.96', '216.239.33.97',
        '216.239.33.98', '216.239.33.99', '216.239.37.98', '216.239.37.99',
        '216.239.39.98', '216.239.39.99', '216.239.41.96', '216.239.41.97',
        '216.239.41.98', '216.239.41.99', '216.239.45.4', '216.239.51.96',
        '216.239.51.97', '216.239.51.98', '216.239.51.99', '216.239.53.98',
        '216.239.53.99', '216.239.57.96', '216.239.57.97', '216.239.57.98',
        '216.239.57.99', '216.239.59.98', '216.239.59.99', '216.33.229.163',
        '64.233.173.193', '64.233.173.194', '64.233.173.195', '64.233.173.196',
        '64.233.173.197', '64.233.173.198', '64.233.173.199', '64.233.173.200',
        '64.233.173.201', '64.233.173.202', '64.233.173.203', '64.233.173.204',
        '64.233.173.205', '64.233.173.206', '64.233.173.207', '64.233.173.208',
        '64.233.173.209', '64.233.173.210', '64.233.173.211', '64.233.173.212',
        '64.233.173.213', '64.233.173.214', '64.233.173.215', '64.233.173.216',
        '64.233.173.217', '64.233.173.218', '64.233.173.219', '64.233.173.220',
        '64.233.173.221', '64.233.173.222', '64.233.173.223', '64.233.173.224',
        '64.233.173.225', '64.233.173.226', '64.233.173.227', '64.233.173.228',
        '64.233.173.229', '64.233.173.230', '64.233.173.231', '64.233.173.232',
        '64.233.173.233', '64.233.173.234', '64.233.173.235', '64.233.173.236',
        '64.233.173.237', '64.233.173.238', '64.233.173.239', '64.233.173.240',
        '64.233.173.241', '64.233.173.242', '64.233.173.243', '64.233.173.244',
        '64.233.173.245', '64.233.173.246', '64.233.173.247', '64.233.173.248',
        '64.233.173.249', '64.233.173.250', '64.233.173.251', '64.233.173.252',
        '64.233.173.253', '64.233.173.254', '64.233.173.255', '64.68.90.1',
        '64.68.90.10', '64.68.90.11', '64.68.90.12', '64.68.90.129',
        '64.68.90.13', '64.68.90.130', '64.68.90.131', '64.68.90.132',
        '64.68.90.133', '64.68.90.134', '64.68.90.135', '64.68.90.136',
        '64.68.90.137', '64.68.90.138', '64.68.90.139', '64.68.90.14',
        '64.68.90.140', '64.68.90.141', '64.68.90.142', '64.68.90.143',
        '64.68.90.144', '64.68.90.145', '64.68.90.146', '64.68.90.147',
        '64.68.90.148', '64.68.90.149', '64.68.90.15', '64.68.90.150',
        '64.68.90.151', '64.68.90.152', '64.68.90.153', '64.68.90.154',
        '64.68.90.155', '64.68.90.156', '64.68.90.157', '64.68.90.158',
        '64.68.90.159', '64.68.90.16', '64.68.90.160', '64.68.90.161',
        '64.68.90.162', '64.68.90.163', '64.68.90.164', '64.68.90.165',
        '64.68.90.166', '64.68.90.167', '64.68.90.168', '64.68.90.169',
        '64.68.90.17', '64.68.90.170', '64.68.90.171', '64.68.90.172',
        '64.68.90.173', '64.68.90.174', '64.68.90.175', '64.68.90.176',
        '64.68.90.177', '64.68.90.178', '64.68.90.179', '64.68.90.18',
        '64.68.90.180', '64.68.90.181', '64.68.90.182', '64.68.90.183',
        '64.68.90.184', '64.68.90.185', '64.68.90.186', '64.68.90.187',
        '64.68.90.188', '64.68.90.189', '64.68.90.19', '64.68.90.190',
        '64.68.90.191', '64.68.90.192', '64.68.90.193', '64.68.90.194',
        '64.68.90.195', '64.68.90.196', '64.68.90.197', '64.68.90.198',
        '64.68.90.199', '64.68.90.2', '64.68.90.20', '64.68.90.200',
        '64.68.90.201', '64.68.90.202', '64.68.90.203', '64.68.90.204',
        '64.68.90.205', '64.68.90.206', '64.68.90.207', '64.68.90.208',
        '64.68.90.21', '64.68.90.22', '64.68.90.23', '64.68.90.24',
        '64.68.90.25', '64.68.90.26', '64.68.90.27', '64.68.90.28',
        '64.68.90.29', '64.68.90.3', '64.68.90.30', '64.68.90.31',
        '64.68.90.32', '64.68.90.33', '64.68.90.34', '64.68.90.35',
        '64.68.90.36', '64.68.90.37', '64.68.90.38', '64.68.90.39',
        '64.68.90.4', '64.68.90.40', '64.68.90.41', '64.68.90.42',
        '64.68.90.43', '64.68.90.44', '64.68.90.45', '64.68.90.46',
        '64.68.90.47', '64.68.90.48', '64.68.90.49', '64.68.90.5',
        '64.68.90.50', '64.68.90.51', '64.68.90.52', '64.68.90.53',
        '64.68.90.54', '64.68.90.55', '64.68.90.56', '64.68.90.57',
        '64.68.90.58', '64.68.90.59', '64.68.90.6', '64.68.90.60',
        '64.68.90.61', '64.68.90.62', '64.68.90.63', '64.68.90.64',
        '64.68.90.65', '64.68.90.66', '64.68.90.67', '64.68.90.68',
        '64.68.90.69', '64.68.90.7', '64.68.90.70', '64.68.90.71',
        '64.68.90.72', '64.68.90.73', '64.68.90.74', '64.68.90.75',
        '64.68.90.76', '64.68.90.77', '64.68.90.78', '64.68.90.79',
        '64.68.90.8', '64.68.90.80', '64.68.90.9'
    ]

    #ips = ['166.205.138.92', '62.0.18.221',  '69.147.125.65', '188.127.241.156', '79.178.26.33']
    r = redis.Redis()
    nips = len(ips)
    for i in xrange(num):
        ip = ips[i % nips]
        loc = IPRange.getCity(ip, r)

    return num
Beispiel #11
0
def resolveIP(ip):
    global redis_host, redis_port, redis_db
    r = redis.Redis(host=redis_host, port=redis_port, db=redis_db)

    loc = IPRange.getZIP(ip, r)
    print loc
Beispiel #12
0
def resolveIP(ip):
    global redis_host, redis_port, redis_db
    loc = IPRange.get(ip, r)
    print loc
Beispiel #13
0
                rangeMin = int(row[0])
                rangeMax = int(row[1])
                lat = float(location['latitude'])
                lon = float(location['longitude'])

                #take the zipcode if possible
                try:
                    zipcode = location['postalCode']
                except:
                    zipcode = ''

                #junk record
                if countryCode == '-' and (not lat and not lon):
                    continue

                rng = IPRange(rangeMin, rangeMax, lat, lon, zipcode)
                rng.save(pipe)

            except Exception, e:
                logging.error("Could not save record: %s" % e)

            i += 1
            if i % 10000 == 0:
                logging.info("Dumping pipe. did %d ranges" % i)
                pipe.execute()

        pipe.execute()
        logging.info("Imported %d locations" % i)

        return i