class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111
@classmethod
def setUpClass(cls):
cls.delta = 0.04
cls.geocoder = GeocodeFarm(api_key=env["GEOCODEFARM_KEY"], format_string="%s US")
def test_geocode(self):
"""
OpenCage.geocode
"""
self.geocode_run(
{"query": "435 north michigan ave, chicago il 60611 usa"}, {"latitude": 41.890, "longitude": -87.624}
)
def test_reverse_string(self):
"""
GeocodeFarm.reverse string
"""
self.reverse_run(
{"query": "40.75376406311989,-73.98489005863667"},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
def test_reverse_point(self):
"""
GeocodeFarm.reverse Point
"""
self.reverse_run(
{"query": Point(40.75376406311989, -73.98489005863667)},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
def test_authentication_failure(self):
"""
GeocodeFarm authentication failure
"""
self.geocoder = GeocodeFarm(api_key="invalid")
with self.assertRaises(exc.GeocoderAuthenticationFailure):
address = "435 north michigan ave, chicago il 60611"
self.geocoder.geocode(address)
def test_quota_exceeded(self):
"""
GeocodeFarm quota exceeded
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {"geocoding_results": {"STATUS": {"access": "OVER_QUERY_LIMIT", "status": "FAILED, ACCESS_DENIED"}}}
self.geocoder._call_geocoder = types.MethodType(mock_call_geocoder, self.geocoder)
with self.assertRaises(exc.GeocoderQuotaExceeded):
self.geocoder.geocode("435 north michigan ave, chicago il 60611")
def test_unhandled_api_error(self):
"""
GeocodeFarm unhandled error
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {"geocoding_results": {"STATUS": {"access": "BILL_PAST_DUE", "status": "FAILED, ACCESS_DENIED"}}}
self.geocoder._call_geocoder = types.MethodType(mock_call_geocoder, self.geocoder)
with self.assertRaises(exc.GeocoderServiceError):
self.geocoder.geocode("435 north michigan ave, chicago il 60611")
class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111
@classmethod
def setUpClass(cls):
cls.delta = 0.04
cls.geocoder = GeocodeFarm(
api_key=env['GEOCODEFARM_KEY'],
format_string="%s US"
)
def test_geocode(self):
"""
OpenCage.geocode
"""
self.geocode_run(
{"query": u"435 north michigan ave, chicago il 60611 usa"},
{"latitude": 41.890, "longitude": -87.624},
)
def test_reverse_string(self):
"""
GeocodeFarm.reverse string
"""
self.reverse_run(
{"query": u"40.75376406311989,-73.98489005863667"},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
def test_reverse_point(self):
"""
GeocodeFarm.reverse Point
"""
self.reverse_run(
{"query": Point(40.75376406311989, -73.98489005863667)},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
def test_authentication_failure(self):
"""
GeocodeFarm authentication failure
"""
self.geocoder = GeocodeFarm(api_key="invalid")
with self.assertRaises(exc.GeocoderAuthenticationFailure):
address = '435 north michigan ave, chicago il 60611'
self.geocoder.geocode(address)
def test_quota_exceeded(self):
"""
GeocodeFarm quota exceeded
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {
"geocoding_results": {
"STATUS": {
"access": "OVER_QUERY_LIMIT",
"status": "FAILED, ACCESS_DENIED"
}
}
}
self.geocoder._call_geocoder = types.MethodType(
mock_call_geocoder,
self.geocoder
)
with self.assertRaises(exc.GeocoderQuotaExceeded):
self.geocoder.geocode(u'435 north michigan ave, chicago il 60611')
def test_unhandled_api_error(self):
"""
GeocodeFarm unhandled error
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {
"geocoding_results": {
"STATUS": {
"access": "BILL_PAST_DUE",
"status": "FAILED, ACCESS_DENIED"
}
}
}
self.geocoder._call_geocoder = types.MethodType(
mock_call_geocoder,
self.geocoder
)
with self.assertRaises(exc.GeocoderServiceError):
self.geocoder.geocode(u'435 north michigan ave, chicago il 60611')
class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111
@classmethod
def setUpClass(cls):
cls.delta = 0.04
cls.geocoder = GeocodeFarm(
api_key=env.get('GEOCODEFARM_KEY'), # None api_key will use free tier on GeocodeFarm
timeout=60,
)
def setUp(self):
# Store the original _call_geocoder in case we replace it with a mock
self._original_call_geocoder = self.geocoder._call_geocoder
def tearDown(self):
# Restore the original _call_geocoder in case we replaced it with a mock
self.geocoder._call_geocoder = self._original_call_geocoder
def test_geocode(self):
"""
GeocodeFarm.geocode
"""
self.geocode_run(
{"query": "435 north michigan ave, chicago il 60611 usa"},
{"latitude": 41.890, "longitude": -87.624},
)
def test_reverse_string(self):
"""
GeocodeFarm.reverse string
"""
self.reverse_run(
{"query": "40.75376406311989,-73.98489005863667"},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
def test_reverse_point(self):
"""
GeocodeFarm.reverse Point
"""
self.reverse_run(
{"query": Point(40.75376406311989, -73.98489005863667)},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
def test_authentication_failure(self):
"""
GeocodeFarm authentication failure
"""
self.geocoder = GeocodeFarm(api_key="invalid")
try:
with self.assertRaises(exc.GeocoderAuthenticationFailure):
address = '435 north michigan ave, chicago il 60611'
self.geocoder.geocode(address)
except exc.GeocoderTimedOut:
raise unittest.SkipTest("GeocodeFarm timed out")
def test_quota_exceeded(self):
"""
GeocodeFarm quota exceeded
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {
"geocoding_results": {
"STATUS": {
"access": "OVER_QUERY_LIMIT",
"status": "FAILED, ACCESS_DENIED"
}
}
}
self.geocoder._call_geocoder = types.MethodType(
mock_call_geocoder,
self.geocoder
)
with self.assertRaises(exc.GeocoderQuotaExceeded):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
def test_unhandled_api_error(self):
"""
GeocodeFarm unhandled error
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {
"geocoding_results": {
"STATUS": {
"access": "BILL_PAST_DUE",
"status": "FAILED, ACCESS_DENIED"
}
}
}
self.geocoder._call_geocoder = types.MethodType(
mock_call_geocoder,
self.geocoder
)
with self.assertRaises(exc.GeocoderServiceError):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
class GeocodeFarmTestCase(GeocoderTestBase):
@classmethod
def setUpClass(cls):
cls.delta = 0.04
cls.geocoder = GeocodeFarm(
# None api_key will use free tier on GeocodeFarm.
api_key=env.get('GEOCODEFARM_KEY'),
timeout=10,
)
def test_user_agent_custom(self):
geocoder = GeocodeFarm(user_agent='my_user_agent/1.0')
self.assertEqual(geocoder.headers['User-Agent'], 'my_user_agent/1.0')
def test_geocode(self):
"""
GeocodeFarm.geocode
"""
location = self.geocode_run(
{"query": "435 north michigan ave, chicago il 60611 usa"},
{
"latitude": 41.890,
"longitude": -87.624
},
)
self.assertIn("chicago", location.address.lower())
def test_location_address(self):
self.geocode_run({"query": "moscow"}, {
"address": "Moscow, Russia",
"latitude": 55.7558913503453,
"longitude": 37.6172961632184
})
def test_reverse(self):
location = self.reverse_run(
{"query": Point(40.75376406311989, -73.98489005863667)},
{
"latitude": 40.75376406311989,
"longitude": -73.98489005863667
},
)
self.assertIn("new york", location.address.lower())
def test_authentication_failure(self):
"""
GeocodeFarm authentication failure
"""
self.geocoder = GeocodeFarm(api_key="invalid")
with self.assertRaises(exc.GeocoderAuthenticationFailure):
self.geocode_run(
{"query": '435 north michigan ave, chicago il 60611'},
{},
expect_failure=True,
)
def test_quota_exceeded(self):
"""
GeocodeFarm quota exceeded
"""
def mock_call_geocoder(*args, **kwargs):
return {
"geocoding_results": {
"STATUS": {
"access": "OVER_QUERY_LIMIT",
"status": "FAILED, ACCESS_DENIED"
}
}
}
with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \
self.assertRaises(exc.GeocoderQuotaExceeded):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
def test_no_results(self):
self.geocode_run({"query": "gibberish kdjhsakdjh skjdhsakjdh"}, {},
expect_failure=True)
def test_unhandled_api_error(self):
"""
GeocodeFarm unhandled error
"""
def mock_call_geocoder(*args, **kwargs):
return {
"geocoding_results": {
"STATUS": {
"access": "BILL_PAST_DUE",
"status": "FAILED, ACCESS_DENIED"
}
}
}
with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \
self.assertRaises(exc.GeocoderServiceError):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
except (KeyboardInterrupt, EOFError):
exit(0)
if __name__ == '__main__':
entry = {}
entry['name'] = input('Name> ')
entry['type'] = choose(
['school', 'company', 'research institute', 'other'], 'Type> ')
address = input('Address> ')
geolocator = GeocodeFarm()
location = geolocator.geocode(address, timeout=15)
if location is None:
print('Geocoder failed')
exit()
entry['lat'] = location.latitude
entry['long'] = location.longitude
if location.longitude < -28: # Recife, Brazil @ -34, Reykjavik, Iceland @ -22 - still not correct for eastern Russia!
datafile = 'america'
else:
datafile = choose(datafiles, 'Chose a datafile> ')
include_address = choose(['yes', 'no'], 'Include address in entry?> ')
if include_address == 'yes':
entry['address'] = location.raw['formatted_address']
class GeocodeFarmTestCase(GeocoderTestBase):
@classmethod
def setUpClass(cls):
cls.delta = 0.04
cls.geocoder = GeocodeFarm(
# None api_key will use free tier on GeocodeFarm.
api_key=env.get('GEOCODEFARM_KEY'),
timeout=10,
)
def test_user_agent_custom(self):
geocoder = GeocodeFarm(
user_agent='my_user_agent/1.0'
)
self.assertEqual(geocoder.headers['User-Agent'], 'my_user_agent/1.0')
def test_geocode(self):
"""
GeocodeFarm.geocode
"""
location = self.geocode_run(
{"query": "435 north michigan ave, chicago il 60611 usa"},
{"latitude": 41.890, "longitude": -87.624},
)
self.assertIn("chicago", location.address.lower())
def test_location_address(self):
self.geocode_run(
{"query": "moscow"},
{"address": "Moscow, Russia",
"latitude": 55.7558913503453, "longitude": 37.6172961632184}
)
def test_reverse(self):
location = self.reverse_run(
{"query": Point(40.75376406311989, -73.98489005863667)},
{"latitude": 40.75376406311989, "longitude": -73.98489005863667},
)
self.assertIn("new york", location.address.lower())
def test_authentication_failure(self):
"""
GeocodeFarm authentication failure
"""
self.geocoder = GeocodeFarm(api_key="invalid")
with self.assertRaises(exc.GeocoderAuthenticationFailure):
self.geocode_run(
{"query": '435 north michigan ave, chicago il 60611'},
{},
expect_failure=True,
)
def test_quota_exceeded(self):
"""
GeocodeFarm quota exceeded
"""
def mock_call_geocoder(*args, **kwargs):
return {
"geocoding_results": {
"STATUS": {
"access": "OVER_QUERY_LIMIT",
"status": "FAILED, ACCESS_DENIED"
}
}
}
with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \
self.assertRaises(exc.GeocoderQuotaExceeded):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
def test_no_results(self):
self.geocode_run(
{"query": "gibberish kdjhsakdjh skjdhsakjdh"},
{},
expect_failure=True
)
def test_unhandled_api_error(self):
"""
GeocodeFarm unhandled error
"""
def mock_call_geocoder(*args, **kwargs):
return {
"geocoding_results": {
"STATUS": {
"access": "BILL_PAST_DUE",
"status": "FAILED, ACCESS_DENIED"
}
}
}
with patch.object(self.geocoder, '_call_geocoder', mock_call_geocoder), \
self.assertRaises(exc.GeocoderServiceError):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
class GeocodeFarmTestCase(GeocoderTestBase): # pylint: disable=R0904,C0111
@classmethod
def setUpClass(cls):
cls.delta = 0.04
cls.geocoder = GeocodeFarm(
api_key=env.get(
'GEOCODEFARM_KEY'
), # None api_key will use free tier on GeocodeFarm
timeout=60,
)
def setUp(self):
# Store the original _call_geocoder in case we replace it with a mock
self._original_call_geocoder = self.geocoder._call_geocoder
def tearDown(self):
# Restore the original _call_geocoder in case we replaced it with a mock
self.geocoder._call_geocoder = self._original_call_geocoder
def test_user_agent_custom(self):
geocoder = GeocodeFarm(user_agent='my_user_agent/1.0')
self.assertEqual(geocoder.headers['User-Agent'], 'my_user_agent/1.0')
def test_geocode(self):
"""
GeocodeFarm.geocode
"""
self.geocode_run(
{"query": "435 north michigan ave, chicago il 60611 usa"},
{
"latitude": 41.890,
"longitude": -87.624
},
)
def test_reverse_string(self):
"""
GeocodeFarm.reverse string
"""
self.reverse_run(
{"query": "40.75376406311989,-73.98489005863667"},
{
"latitude": 40.75376406311989,
"longitude": -73.98489005863667
},
)
def test_reverse_point(self):
"""
GeocodeFarm.reverse Point
"""
self.reverse_run(
{"query": Point(40.75376406311989, -73.98489005863667)},
{
"latitude": 40.75376406311989,
"longitude": -73.98489005863667
},
)
def test_authentication_failure(self):
"""
GeocodeFarm authentication failure
"""
self.geocoder = GeocodeFarm(api_key="invalid")
try:
with self.assertRaises(exc.GeocoderAuthenticationFailure):
address = '435 north michigan ave, chicago il 60611'
self.geocoder.geocode(address)
except exc.GeocoderTimedOut:
raise unittest.SkipTest("GeocodeFarm timed out")
def test_quota_exceeded(self):
"""
GeocodeFarm quota exceeded
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {
"geocoding_results": {
"STATUS": {
"access": "OVER_QUERY_LIMIT",
"status": "FAILED, ACCESS_DENIED"
}
}
}
self.geocoder._call_geocoder = types.MethodType(
mock_call_geocoder, self.geocoder)
with self.assertRaises(exc.GeocoderQuotaExceeded):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
def test_unhandled_api_error(self):
"""
GeocodeFarm unhandled error
"""
def mock_call_geocoder(*args, **kwargs):
"""
Mock API call to return bad response.
"""
return {
"geocoding_results": {
"STATUS": {
"access": "BILL_PAST_DUE",
"status": "FAILED, ACCESS_DENIED"
}
}
}
self.geocoder._call_geocoder = types.MethodType(
mock_call_geocoder, self.geocoder)
with self.assertRaises(exc.GeocoderServiceError):
self.geocoder.geocode('435 north michigan ave, chicago il 60611')
class NamedEntityTagger(object):
def __init__(self):
self.db = NamedEntityMap()
self.tagger = ner.SocketNER(host='localhost', port=2020, output_format='slashTags')
#self.geolocator = GeoNames(username='******')
#self.geolocator = GoogleV3()
self.geolocator = GeocodeFarm(api_key='3ae1656ecc66581156dc23e6bf7a182d1fccdf49')
self.dstk = dstk.DSTK({'apiBase':'http://localhost:8085'})
def lookup(self, entity):
location = self.db.lookup(entity)
if location:
return location
location = self.geolocate(entity)
if location == None:
return None
address = location.raw['ADDRESS']
print '%s -> %s' % (entity, address['address_returned'])
coordinates = location.raw['COORDINATES']
point = Point(coordinates['latitude'], coordinates['longitude'])
politics = self.dstk.coordinates2politics((point.latitude, point.longitude))[0]['politics']
country = None
if politics:
for c in politics:
if c['type'] == 'admin2':
country = c['code']
if country == None:
return None
return self.db.add(entity, point, country)
def geolocate(self, entity):
timeout = 1
for i in range(30):
try:
location = self.geolocator.geocode(entity)
return location
except (GeocoderServiceError, GeocoderAuthenticationFailure, GeocoderQueryError, GeocoderInsufficientPrivileges):
return None
except (socket.timeout, GeocoderTimedOut):
print 'received timeout, retrying in %d seconds', timeout
time.sleep(timeout)
timeout = timeout * 2
def tag_file(self, in_filename, out_filename):
if os.path.isfile(out_filename):
print 'skipping', in_filename
return
print 'creating features file:', in_filename, '->', out_filename
outfile_data = []
with open(in_filename, 'r') as infile:
for line in infile:
entities = self.tagger.get_entities(line)
countries = []
if 'LOCATION' in entities:
entities = set([s.lower() for s in entities['LOCATION']])
for e in entities:
location = self.lookup(e)
if location == None:
continue
countries.append(location.country)
clist = ['%s=1' % c for c in sorted(set(countries))]
outfile_data.append(' '.join(clist))
with open(out_filename, 'w') as outfile:
for line in outfile_data:
outfile.write(line + '\n')
# save the db
self.db.save()
class GeocodeFarmTestCases(unittest.TestCase):
def setUp(self):
self.address = "Sunnersta" #static address to be found
self.address2 = "Mackenzie" #static address for DataBC only
self.userlocation = (59.8585107, 17.6368508)
self.addrNone = "abcdefghijklmnopqrstuvwxyz zyxwvutsrqponmlkjihgfedcba" #non-existing address
self.scheme = "https"
self.plainscheme = "http"
#set up for geocodeFarm
self.geolocator5 = GeocodeFarm()
self.geourlapi = "https://www.geocode.farm/v3/json/forward/"
def testGeocodeFarm(self):
#assert if the object's structure is the same as class Geocode Farm
self.assertIsInstance(
self.geolocator5, GeocodeFarm,
"The object is not an instance of class Geocode Farm")
#assert the url built
self.assertEqual(self.geolocator5.scheme, self.scheme)
self.assertEqual(self.geolocator5.api, self.geourlapi)
def testDataTypeNone(self):
#time.sleep(2)
#without userlocation
location = self.geolocator5.geocode(self.addrNone)
self.assertIsNone(location, "location found! " + str(self.geolocator5))
#with userlocation
location = self.geolocator5.geocode(self.addrNone, self.userlocation)
#print(location)
#print(location.latitude)
self.assertIsNone(location, "location found! " + str(self.geolocator5))
def testDataTypeSingle(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address)
self.assertIsNotNone(location, "location not found! ")
self.assertIsInstance(
location, Location,
"location is not an instance of class Location! ")
self.assertIsNot(type(location), list, "location is multiple! ")
self.assertIs(type(location), Location,
"location is not of type Location! ")
self.assertIs(type(location.address), unicode,
"address is not of type unicode! ")
self.assertIs(type(location.latitude), float,
"latitude is not of type float! ")
self.assertIs(type(location.longitude), float,
"longitude is not of type float! ")
self.assertIs(type(location.altitude), float,
"altitude is not of type float! ")
self.assertIs(type(location.raw), dict, "raw is not of type dict! ")
#print(location)
def testDataTypeMultiple(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address, False)
self.assertIsNotNone(location, "location not found! ")
self.assertIs(type(location), list, "location is single! ")
for l in range(len(location)):
self.assertIsInstance(
location[l], Location,
"location is not an instance of class Location! ")
self.assertIs(type(location[l]), Location,
"location is not of type Location! ")
self.assertIs(type(location[l].address), unicode,
"address is not of type unicode! ")
self.assertIs(type(location[l].latitude), float,
"latitude is not of type float! ")
self.assertIs(type(location[l].longitude), float,
"longitude is not of type float! ")
self.assertIs(type(location[l].altitude), float,
"altitude is not of type float! ")
self.assertIs(type(location[l].raw), dict,
"raw is not of type dict! ")
#print(location[l])
def testAddressSingle(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address)
self.assertIn(self.address, location.raw['formatted_address'])
def testAddressMultiple(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address, exactly_one=False)
for gl1 in range(len(location)):
self.assertIn(self.address, location[gl1].raw['formatted_address'])
def testAddressSingleChanges(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address,
userlocation=self.userlocation,
exactly_one=True)
self.assertIn(self.address, location.raw['formatted_address'])
def testAddressMultipleChanges(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address,
userlocation=self.userlocation,
exactly_one=False)
for gl1 in range(len(location)):
self.assertIn(self.address, location[gl1].raw['formatted_address'])
def testOrderedData(self):
#time.sleep(2)
location = self.geolocator5.geocode(self.address,
userlocation=self.userlocation,
exactly_one=False)
#put all distance in array
distance = []
for l in range(len(location)):
distance.append(
vincenty(self.userlocation,
(location[l].latitude, location[l].longitude)))
#compare all distance with the first distance, the first one should be the smallest
min_distance = distance[0]
for l in range(len(distance)):
self.assertLessEqual(min_distance, distance[l],
"The order of data is wrong")
#pw = getpass('Enter uow proxy password: '******'GEOCODEFARM_API_KEY']
#intialize geocoder to geocode addresses in only NSW, Australia (addresses will be interpolated to add NSW, Australia)
gc = GeocodeFarm(api_key,"%s, NSW, Australia",1)#,proxies={"http": "http://*****:*****@proxy.domain:8080"})
outstring = ""
for addressfile in sys.argv[1:]:
with open(addressfile, "r") as inputfile:
with open(addressfile[:-4]+"_output.csv", "w") as outputfile:
for address in inputfile:
outstring = address.rstrip()
print datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
try:
location = gc.geocode(address)
outstring = outstring + "\t" + str(location.latitude) + "\t" + str(location.longitude) + "\t" + location.raw["accuracy"] + "\n"
outputfile.write(outstring)
except:
outstring = outstring + "\t" + "address couldn't be geolocated" + "\n"
outputfile.write(outstring)
print ("address couldn't be geolocated")
print datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
time.sleep(0.6)
print datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
#set outstring to empty
outstring = ""
outputfile.close()
inputfile.close()
