class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111 @classmethod def setUpClass(cls): cls.delta = 0.04 cls.geocoder = GeocodeFarm(api_key=env["GEOCODEFARM_KEY"], format_string="%s US") def test_geocode(self): """ OpenCage.geocode """ self.geocode_run( {"query": "435 north michigan ave, chicago il 60611 usa"}, {"latitude": 41.890, "longitude": -87.624} ) def test_reverse_string(self): """ GeocodeFarm.reverse string """ self.reverse_run( {"query": "40.75376406311989,-73.98489005863667"}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) def test_reverse_point(self): """ GeocodeFarm.reverse Point """ self.reverse_run( {"query": Point(40.75376406311989, -73.98489005863667)}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) def test_authentication_failure(self): """ GeocodeFarm authentication failure """ self.geocoder = GeocodeFarm(api_key="invalid") with self.assertRaises(exc.GeocoderAuthenticationFailure): address = "435 north michigan ave, chicago il 60611" self.geocoder.geocode(address) def test_quota_exceeded(self): """ GeocodeFarm quota exceeded """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return {"geocoding_results": {"STATUS": {"access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED"}}} self.geocoder._call_geocoder = types.MethodType(mock_call_geocoder, self.geocoder) with self.assertRaises(exc.GeocoderQuotaExceeded): self.geocoder.geocode("435 north michigan ave, chicago il 60611") def test_unhandled_api_error(self): """ GeocodeFarm unhandled error """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return {"geocoding_results": {"STATUS": {"access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED"}}} self.geocoder._call_geocoder = types.MethodType(mock_call_geocoder, self.geocoder) with self.assertRaises(exc.GeocoderServiceError): self.geocoder.geocode("435 north michigan ave, chicago il 60611")
class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111 @classmethod def setUpClass(cls): cls.delta = 0.04 cls.geocoder = GeocodeFarm( api_key=env['GEOCODEFARM_KEY'], format_string="%s US" ) def test_geocode(self): """ OpenCage.geocode """ self.geocode_run( {"query": u"435 north michigan ave, chicago il 60611 usa"}, {"latitude": 41.890, "longitude": -87.624}, ) def test_reverse_string(self): """ GeocodeFarm.reverse string """ self.reverse_run( {"query": u"40.75376406311989,-73.98489005863667"}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) def test_reverse_point(self): """ GeocodeFarm.reverse Point """ self.reverse_run( {"query": Point(40.75376406311989, -73.98489005863667)}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) def test_authentication_failure(self): """ GeocodeFarm authentication failure """ self.geocoder = GeocodeFarm(api_key="invalid") with self.assertRaises(exc.GeocoderAuthenticationFailure): address = '435 north michigan ave, chicago il 60611' self.geocoder.geocode(address) def test_quota_exceeded(self): """ GeocodeFarm quota exceeded """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return { "geocoding_results": { "STATUS": { "access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED" } } } self.geocoder._call_geocoder = types.MethodType( mock_call_geocoder, self.geocoder ) with self.assertRaises(exc.GeocoderQuotaExceeded): self.geocoder.geocode(u'435 north michigan ave, chicago il 60611') def test_unhandled_api_error(self): """ GeocodeFarm unhandled error """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return { "geocoding_results": { "STATUS": { "access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED" } } } self.geocoder._call_geocoder = types.MethodType( mock_call_geocoder, self.geocoder ) with self.assertRaises(exc.GeocoderServiceError): self.geocoder.geocode(u'435 north michigan ave, chicago il 60611')
class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111 @classmethod def setUpClass(cls): cls.delta = 0.04 cls.geocoder = GeocodeFarm( api_key=env.get('GEOCODEFARM_KEY'), # None api_key will use free tier on GeocodeFarm timeout=60, ) def setUp(self): # Store the original _call_geocoder in case we replace it with a mock self._original_call_geocoder = self.geocoder._call_geocoder def tearDown(self): # Restore the original _call_geocoder in case we replaced it with a mock self.geocoder._call_geocoder = self._original_call_geocoder def test_geocode(self): """ GeocodeFarm.geocode """ self.geocode_run( {"query": "435 north michigan ave, chicago il 60611 usa"}, {"latitude": 41.890, "longitude": -87.624}, ) def test_reverse_string(self): """ GeocodeFarm.reverse string """ self.reverse_run( {"query": "40.75376406311989,-73.98489005863667"}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) def test_reverse_point(self): """ GeocodeFarm.reverse Point """ self.reverse_run( {"query": Point(40.75376406311989, -73.98489005863667)}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) def test_authentication_failure(self): """ GeocodeFarm authentication failure """ self.geocoder = GeocodeFarm(api_key="invalid") try: with self.assertRaises(exc.GeocoderAuthenticationFailure): address = '435 north michigan ave, chicago il 60611' self.geocoder.geocode(address) except exc.GeocoderTimedOut: raise unittest.SkipTest("GeocodeFarm timed out") def test_quota_exceeded(self): """ GeocodeFarm quota exceeded """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return { "geocoding_results": { "STATUS": { "access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED" } } } self.geocoder._call_geocoder = types.MethodType( mock_call_geocoder, self.geocoder ) with self.assertRaises(exc.GeocoderQuotaExceeded): self.geocoder.geocode('435 north michigan ave, chicago il 60611') def test_unhandled_api_error(self): """ GeocodeFarm unhandled error """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return { "geocoding_results": { "STATUS": { "access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED" } } } self.geocoder._call_geocoder = types.MethodType( mock_call_geocoder, self.geocoder ) with self.assertRaises(exc.GeocoderServiceError): self.geocoder.geocode('435 north michigan ave, chicago il 60611')
class GeocodeFarmTestCase(GeocoderTestBase): @classmethod def setUpClass(cls): cls.delta = 0.04 cls.geocoder = GeocodeFarm( # None api_key will use free tier on GeocodeFarm. api_key=env.get('GEOCODEFARM_KEY'), timeout=10, ) def test_user_agent_custom(self): geocoder = GeocodeFarm(user_agent='my_user_agent/1.0') self.assertEqual(geocoder.headers['User-Agent'], 'my_user_agent/1.0') def test_geocode(self): """ GeocodeFarm.geocode """ location = self.geocode_run( {"query": "435 north michigan ave, chicago il 60611 usa"}, { "latitude": 41.890, "longitude": -87.624 }, ) self.assertIn("chicago", location.address.lower()) def test_location_address(self): self.geocode_run({"query": "moscow"}, { "address": "Moscow, Russia", "latitude": 55.7558913503453, "longitude": 37.6172961632184 }) def test_reverse(self): location = self.reverse_run( {"query": Point(40.75376406311989, -73.98489005863667)}, { "latitude": 40.75376406311989, "longitude": -73.98489005863667 }, ) self.assertIn("new york", location.address.lower()) def test_authentication_failure(self): """ GeocodeFarm authentication failure """ self.geocoder = GeocodeFarm(api_key="invalid") with self.assertRaises(exc.GeocoderAuthenticationFailure): self.geocode_run( {"query": '435 north michigan ave, chicago il 60611'}, {}, expect_failure=True, ) def test_quota_exceeded(self): """ GeocodeFarm quota exceeded """ def mock_call_geocoder(*args, **kwargs): return { "geocoding_results": { "STATUS": { "access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED" } } } with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \ self.assertRaises(exc.GeocoderQuotaExceeded): self.geocoder.geocode('435 north michigan ave, chicago il 60611') def test_no_results(self): self.geocode_run({"query": "gibberish kdjhsakdjh skjdhsakjdh"}, {}, expect_failure=True) def test_unhandled_api_error(self): """ GeocodeFarm unhandled error """ def mock_call_geocoder(*args, **kwargs): return { "geocoding_results": { "STATUS": { "access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED" } } } with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \ self.assertRaises(exc.GeocoderServiceError): self.geocoder.geocode('435 north michigan ave, chicago il 60611')
except (KeyboardInterrupt, EOFError): exit(0) if __name__ == '__main__': entry = {} entry['name'] = input('Name> ') entry['type'] = choose( ['school', 'company', 'research institute', 'other'], 'Type> ') address = input('Address> ') geolocator = GeocodeFarm() location = geolocator.geocode(address, timeout=15) if location is None: print('Geocoder failed') exit() entry['lat'] = location.latitude entry['long'] = location.longitude if location.longitude < -28: # Recife, Brazil @ -34, Reykjavik, Iceland @ -22 - still not correct for eastern Russia! datafile = 'america' else: datafile = choose(datafiles, 'Chose a datafile> ') include_address = choose(['yes', 'no'], 'Include address in entry?> ') if include_address == 'yes': entry['address'] = location.raw['formatted_address']
class GeocodeFarmTestCase(GeocoderTestBase): @classmethod def setUpClass(cls): cls.delta = 0.04 cls.geocoder = GeocodeFarm( # None api_key will use free tier on GeocodeFarm. api_key=env.get('GEOCODEFARM_KEY'), timeout=10, ) def test_user_agent_custom(self): geocoder = GeocodeFarm( user_agent='my_user_agent/1.0' ) self.assertEqual(geocoder.headers['User-Agent'], 'my_user_agent/1.0') def test_geocode(self): """ GeocodeFarm.geocode """ location = self.geocode_run( {"query": "435 north michigan ave, chicago il 60611 usa"}, {"latitude": 41.890, "longitude": -87.624}, ) self.assertIn("chicago", location.address.lower()) def test_location_address(self): self.geocode_run( {"query": "moscow"}, {"address": "Moscow, Russia", "latitude": 55.7558913503453, "longitude": 37.6172961632184} ) def test_reverse(self): location = self.reverse_run( {"query": Point(40.75376406311989, -73.98489005863667)}, {"latitude": 40.75376406311989, "longitude": -73.98489005863667}, ) self.assertIn("new york", location.address.lower()) def test_authentication_failure(self): """ GeocodeFarm authentication failure """ self.geocoder = GeocodeFarm(api_key="invalid") with self.assertRaises(exc.GeocoderAuthenticationFailure): self.geocode_run( {"query": '435 north michigan ave, chicago il 60611'}, {}, expect_failure=True, ) def test_quota_exceeded(self): """ GeocodeFarm quota exceeded """ def mock_call_geocoder(*args, **kwargs): return { "geocoding_results": { "STATUS": { "access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED" } } } with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \ self.assertRaises(exc.GeocoderQuotaExceeded): self.geocoder.geocode('435 north michigan ave, chicago il 60611') def test_no_results(self): self.geocode_run( {"query": "gibberish kdjhsakdjh skjdhsakjdh"}, {}, expect_failure=True ) def test_unhandled_api_error(self): """ GeocodeFarm unhandled error """ def mock_call_geocoder(*args, **kwargs): return { "geocoding_results": { "STATUS": { "access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED" } } } with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \ self.assertRaises(exc.GeocoderServiceError): self.geocoder.geocode('435 north michigan ave, chicago il 60611')
class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111 @classmethod def setUpClass(cls): cls.delta = 0.04 cls.geocoder = GeocodeFarm( api_key=env.get( 'GEOCODEFARM_KEY' ), # None api_key will use free tier on GeocodeFarm timeout=60, ) def setUp(self): # Store the original _call_geocoder in case we replace it with a mock self._original_call_geocoder = self.geocoder._call_geocoder def tearDown(self): # Restore the original _call_geocoder in case we replaced it with a mock self.geocoder._call_geocoder = self._original_call_geocoder def test_user_agent_custom(self): geocoder = GeocodeFarm(user_agent='my_user_agent/1.0') self.assertEqual(geocoder.headers['User-Agent'], 'my_user_agent/1.0') def test_geocode(self): """ GeocodeFarm.geocode """ self.geocode_run( {"query": "435 north michigan ave, chicago il 60611 usa"}, { "latitude": 41.890, "longitude": -87.624 }, ) def test_reverse_string(self): """ GeocodeFarm.reverse string """ self.reverse_run( {"query": "40.75376406311989,-73.98489005863667"}, { "latitude": 40.75376406311989, "longitude": -73.98489005863667 }, ) def test_reverse_point(self): """ GeocodeFarm.reverse Point """ self.reverse_run( {"query": Point(40.75376406311989, -73.98489005863667)}, { "latitude": 40.75376406311989, "longitude": -73.98489005863667 }, ) def test_authentication_failure(self): """ GeocodeFarm authentication failure """ self.geocoder = GeocodeFarm(api_key="invalid") try: with self.assertRaises(exc.GeocoderAuthenticationFailure): address = '435 north michigan ave, chicago il 60611' self.geocoder.geocode(address) except exc.GeocoderTimedOut: raise unittest.SkipTest("GeocodeFarm timed out") def test_quota_exceeded(self): """ GeocodeFarm quota exceeded """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return { "geocoding_results": { "STATUS": { "access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED" } } } self.geocoder._call_geocoder = types.MethodType( mock_call_geocoder, self.geocoder) with self.assertRaises(exc.GeocoderQuotaExceeded): self.geocoder.geocode('435 north michigan ave, chicago il 60611') def test_unhandled_api_error(self): """ GeocodeFarm unhandled error """ def mock_call_geocoder(*args, **kwargs): """ Mock API call to return bad response. """ return { "geocoding_results": { "STATUS": { "access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED" } } } self.geocoder._call_geocoder = types.MethodType( mock_call_geocoder, self.geocoder) with self.assertRaises(exc.GeocoderServiceError): self.geocoder.geocode('435 north michigan ave, chicago il 60611')
class NamedEntityTagger(object): def __init__(self): self.db = NamedEntityMap() self.tagger = ner.SocketNER(host='localhost', port=2020, output_format='slashTags') #self.geolocator = GeoNames(username='******') #self.geolocator = GoogleV3() self.geolocator = GeocodeFarm(api_key='3ae1656ecc66581156dc23e6bf7a182d1fccdf49') self.dstk = dstk.DSTK({'apiBase':'http://localhost:8085'}) def lookup(self, entity): location = self.db.lookup(entity) if location: return location location = self.geolocate(entity) if location == None: return None address = location.raw['ADDRESS'] print '%s -> %s' % (entity, address['address_returned']) coordinates = location.raw['COORDINATES'] point = Point(coordinates['latitude'], coordinates['longitude']) politics = self.dstk.coordinates2politics((point.latitude, point.longitude))[0]['politics'] country = None if politics: for c in politics: if c['type'] == 'admin2': country = c['code'] if country == None: return None return self.db.add(entity, point, country) def geolocate(self, entity): timeout = 1 for i in range(30): try: location = self.geolocator.geocode(entity) return location except (GeocoderServiceError, GeocoderAuthenticationFailure, GeocoderQueryError, GeocoderInsufficientPrivileges): return None except (socket.timeout, GeocoderTimedOut): print 'received timeout, retrying in %d seconds', timeout time.sleep(timeout) timeout = timeout * 2 def tag_file(self, in_filename, out_filename): if os.path.isfile(out_filename): print 'skipping', in_filename return print 'creating features file:', in_filename, '->', out_filename outfile_data = [] with open(in_filename, 'r') as infile: for line in infile: entities = self.tagger.get_entities(line) countries = [] if 'LOCATION' in entities: entities = set([s.lower() for s in entities['LOCATION']]) for e in entities: location = self.lookup(e) if location == None: continue countries.append(location.country) clist = ['%s=1' % c for c in sorted(set(countries))] outfile_data.append(' '.join(clist)) with open(out_filename, 'w') as outfile: for line in outfile_data: outfile.write(line + '\n') # save the db self.db.save()
class GeocodeFarmTestCases(unittest.TestCase): def setUp(self): self.address = "Sunnersta" #static address to be found self.address2 = "Mackenzie" #static address for DataBC only self.userlocation = (59.8585107, 17.6368508) self.addrNone = "abcdefghijklmnopqrstuvwxyz zyxwvutsrqponmlkjihgfedcba" #non-existing address self.scheme = "https" self.plainscheme = "http" #set up for geocodeFarm self.geolocator5 = GeocodeFarm() self.geourlapi = "https://www.geocode.farm/v3/json/forward/" def testGeocodeFarm(self): #assert if the object's structure is the same as class Geocode Farm self.assertIsInstance( self.geolocator5, GeocodeFarm, "The object is not an instance of class Geocode Farm") #assert the url built self.assertEqual(self.geolocator5.scheme, self.scheme) self.assertEqual(self.geolocator5.api, self.geourlapi) def testDataTypeNone(self): #time.sleep(2) #without userlocation location = self.geolocator5.geocode(self.addrNone) self.assertIsNone(location, "location found! " + str(self.geolocator5)) #with userlocation location = self.geolocator5.geocode(self.addrNone, self.userlocation) #print(location) #print(location.latitude) self.assertIsNone(location, "location found! " + str(self.geolocator5)) def testDataTypeSingle(self): #time.sleep(2) location = self.geolocator5.geocode(self.address) self.assertIsNotNone(location, "location not found! ") self.assertIsInstance( location, Location, "location is not an instance of class Location! ") self.assertIsNot(type(location), list, "location is multiple! ") self.assertIs(type(location), Location, "location is not of type Location! ") self.assertIs(type(location.address), unicode, "address is not of type unicode! ") self.assertIs(type(location.latitude), float, "latitude is not of type float! ") self.assertIs(type(location.longitude), float, "longitude is not of type float! ") self.assertIs(type(location.altitude), float, "altitude is not of type float! ") self.assertIs(type(location.raw), dict, "raw is not of type dict! ") #print(location) def testDataTypeMultiple(self): #time.sleep(2) location = self.geolocator5.geocode(self.address, False) self.assertIsNotNone(location, "location not found! ") self.assertIs(type(location), list, "location is single! ") for l in range(len(location)): self.assertIsInstance( location[l], Location, "location is not an instance of class Location! ") self.assertIs(type(location[l]), Location, "location is not of type Location! ") self.assertIs(type(location[l].address), unicode, "address is not of type unicode! ") self.assertIs(type(location[l].latitude), float, "latitude is not of type float! ") self.assertIs(type(location[l].longitude), float, "longitude is not of type float! ") self.assertIs(type(location[l].altitude), float, "altitude is not of type float! ") self.assertIs(type(location[l].raw), dict, "raw is not of type dict! ") #print(location[l]) def testAddressSingle(self): #time.sleep(2) location = self.geolocator5.geocode(self.address) self.assertIn(self.address, location.raw['formatted_address']) def testAddressMultiple(self): #time.sleep(2) location = self.geolocator5.geocode(self.address, exactly_one=False) for gl1 in range(len(location)): self.assertIn(self.address, location[gl1].raw['formatted_address']) def testAddressSingleChanges(self): #time.sleep(2) location = self.geolocator5.geocode(self.address, userlocation=self.userlocation, exactly_one=True) self.assertIn(self.address, location.raw['formatted_address']) def testAddressMultipleChanges(self): #time.sleep(2) location = self.geolocator5.geocode(self.address, userlocation=self.userlocation, exactly_one=False) for gl1 in range(len(location)): self.assertIn(self.address, location[gl1].raw['formatted_address']) def testOrderedData(self): #time.sleep(2) location = self.geolocator5.geocode(self.address, userlocation=self.userlocation, exactly_one=False) #put all distance in array distance = [] for l in range(len(location)): distance.append( vincenty(self.userlocation, (location[l].latitude, location[l].longitude))) #compare all distance with the first distance, the first one should be the smallest min_distance = distance[0] for l in range(len(distance)): self.assertLessEqual(min_distance, distance[l], "The order of data is wrong")
#pw = getpass('Enter uow proxy password: '******'GEOCODEFARM_API_KEY'] #intialize geocoder to geocode addresses in only NSW, Australia (addresses will be interpolated to add NSW, Australia) gc = GeocodeFarm(api_key,"%s, NSW, Australia",1)#,proxies={"http": "http://*****:*****@proxy.domain:8080"}) outstring = "" for addressfile in sys.argv[1:]: with open(addressfile, "r") as inputfile: with open(addressfile[:-4]+"_output.csv", "w") as outputfile: for address in inputfile: outstring = address.rstrip() print datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] try: location = gc.geocode(address) outstring = outstring + "\t" + str(location.latitude) + "\t" + str(location.longitude) + "\t" + location.raw["accuracy"] + "\n" outputfile.write(outstring) except: outstring = outstring + "\t" + "address couldn't be geolocated" + "\n" outputfile.write(outstring) print ("address couldn't be geolocated") print datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] time.sleep(0.6) print datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] #set outstring to empty outstring = "" outputfile.close() inputfile.close()