def handle(self, *args, **options):
# from omgeo import Geocoder
g = Geocoder([
[
'omgeo.services.Bing', {
'settings': {
'api_key':
'AnoU4IZjQyL3BLIVJNmfpUgqh86Z8jVEk4n2uQNRzSWAu1Suu5I1NvmjGslFS1Dk'
}
}
],
# ['omgeo.services.Nominatim',{}],
# ['omgeo.services.MapQuest',{
# 'settings':{
# 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6'
# }
# }]
])
for s in Location.objects.all():
#get geocoded address for this address
address_string = '%s, %s' % (s.name, 'Champaign, IL')
geocode_results = g.geocode(address_string, True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if c.confidence == 'High':
s.point_location = GEOSGeometry(
'POINT(%s %s)' % (
c.x,
c.y,
), 4326)
s.save()
self.stdout.write("Location found: %s %s\n" % (
c.x,
c.y,
))
if not s.point_location:
address_string = '%s, %s' % (s.name, 'Urbana, IL')
geocode_results = g.geocode(address_string, True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if c.confidence == 'High':
s.point_location = GEOSGeometry(
'POINT(%s %s)' % (
c.x,
c.y,
), 4326)
s.save()
self.stdout.write("Location found: %s %s\n" % (
c.x,
c.y,
))
if not s.point_location:
self.stdout.write("No good geocode found\n")
def geocode(address):
"""Given an address, returns candidates."""
pq = PlaceQuery(address)
global _geocoder
if _geocoder is None:
_geocoder = Geocoder()
return _geocoder.geocode(pq)
def handle(self, *args, **options):
# from omgeo import Geocoder
g = Geocoder([
['omgeo.services.Bing',{
'settings':{
'api_key':'AnoU4IZjQyL3BLIVJNmfpUgqh86Z8jVEk4n2uQNRzSWAu1Suu5I1NvmjGslFS1Dk'
}
}],
# ['omgeo.services.Nominatim',{}],
# ['omgeo.services.MapQuest',{
# 'settings':{
# 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6'
# }
# }]
])
for s in BPStation.objects.filter(address__isnull=False):
#get geocoded address for this address
address_string = '%s, %s, %s, %s' % (s.address, s.city, s.state, s.zip,)
geocode_results = g.geocode(address_string,True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if c.confidence == 'High':
s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326)
s.save()
self.stdout.write("Location found: %s %s\n" % (c.x, c.y,))
if not s.point_location:
self.stdout.write("No good geocode found\n")
#manual override of locations that don't geocode well
try:
if len(args) > 0:
self.stdout.write('Using manual CSV location ...\n')
file_path = args[0]
else:
self.stdout.write('Using default CSV location ...\n')
working_dir = os.path.join(SITE_ROOT, '../data')
file_path = os.path.join(working_dir, 'geocodestragglers20120830.csv')
handle = csv.reader(open(file_path, 'rU'), delimiter=',', quotechar='"')
header = handle.next()
for h in handle:
print h
try:
station = BPStation.objects.get(name=h[1].strip())
station.point_location = GEOSGeometry('POINT(%s %s)' % (h[7].strip(), h[8].strip(),),4326)
station.save()
self.stdout.write('%s: %s %s\n' % (h[0].strip(), h[7].strip(), h[8].strip(),))
except:
raise
self.stdout.write('Successfully loaded manual Border Patrol geocodes.\n')
except AttributeError:
raise
def load_data(inPath, outPath):
outFile = open(outPath, 'w')
counter = 0
with open(inPath,'Ur') as inFile:
for line in inFile:
GarageID,Zip,Street,City,State,cnt = line.split(",")
address = Street+" "+City+" "+State+" "+"USA"
g = Geocoder()
result = g.geocode(address)
outFile.write('%s\n' % (result))
def handle(self, *args, **options):
# from omgeo import Geocoder
g = Geocoder([
['omgeo.services.Bing',{
'settings':{
'api_key':'AnoU4IZjQyL3BLIVJNmfpUgqh86Z8jVEk4n2uQNRzSWAu1Suu5I1NvmjGslFS1Dk'
}
}],
# ['omgeo.services.Nominatim',{}],
# ['omgeo.services.MapQuest',{
# 'settings':{
# 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6'
# }
# }]
])
for s in Location.objects.all():
#get geocoded address for this address
address_string = '%s, %s' % (s.name, 'Champaign, IL')
geocode_results = g.geocode(address_string,True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if c.confidence == 'High':
s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326)
s.save()
self.stdout.write("Location found: %s %s\n" % (c.x, c.y,))
if not s.point_location:
address_string = '%s, %s' % (s.name, 'Urbana, IL')
geocode_results = g.geocode(address_string,True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if c.confidence == 'High':
s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326)
s.save()
self.stdout.write("Location found: %s %s\n" % (c.x, c.y,))
if not s.point_location:
self.stdout.write("No good geocode found\n")
def __init__(self, address):
"""Geocode given address and set self.coords to (x,y) tuple."""
geocoder = Geocoder()
georesult = geocoder.geocode(address)
geocandidates = georesult['candidates']
coord_pairs = [(c.x, c.y) for c in geocandidates]
num_coord_pairs = len(coord_pairs)
if num_coord_pairs == 0:
raise Exception('No geocoding results for %s' % address)
if num_coord_pairs > 1:
choices = [c.match_addr for c in geocandidates]
raise Exception('Ambiguous address. Choices are:\n%s' % choices)
self.coords = coord_pairs[0]
def handle(self, *args, **options):
parser = StreetAddressParser()
#try:
g = Geocoder([
['omgeo.services.Bing',{
'settings':{
'api_key':'AufKQbvq8uGM3qsQWwMwJNtlf6LLxe5bPvOAVcxi-79Qp0tDl0T2qScdOQiBNKkE'
}
}],
])
for location in Location.objects.filter(point_location__isnull=True):
string = location.address.strip()
if location.city is not None:
string = string + ", " + location.city
else:
if location.agency:
string = string + ", " + location.agency.name + " IL"
else:
string = string + ", Champaign, IL"
block_pattern = re.compile('BLOCK OF')
cleaned_address = re.sub(block_pattern,'', string)
print cleaned_address
if location.intersection_indicator:
slash = re.compile('/')
string = re.sub(slash,'at', string)
print string
if len(location.address.strip()) > 0:
geocode_results = g.geocode(cleaned_address,True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if math.floor(c.x) == -89.0 and math.floor(c.y) == 40.0:
print "within acceptable ranges"
if c.confidence == 'High':
location.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326)
location.save()
self.stdout.write("Location found: %s %s\n" % (c.x, c.y,))
if not location.point_location:
self.stdout.write("No good geocode found for %s \n" % (cleaned_address))
#location.attempted = True
location.save()
#except (RuntimeError, TypeError, NameError):
# print "whoopsie"
def geocode_address(request, address):
def result_in_bounding_box(result):
x = float(result.x)
y = float(result.y)
left = float(settings.BOUNDING_BOX['left'])
top = float(settings.BOUNDING_BOX['top'])
right = float(settings.BOUNDING_BOX['right'])
bottom = float(settings.BOUNDING_BOX['bottom'])
return x > left and x < right and y > bottom and y < top
if address is None or len(address) == 0:
raise HttpBadRequestException("No address specfified")
query = PlaceQuery(address, viewbox=Viewbox(
settings.BOUNDING_BOX['left'],
settings.BOUNDING_BOX['top'],
settings.BOUNDING_BOX['right'],
settings.BOUNDING_BOX['bottom'])
)
if (('OMGEO_GEOCODER_SOURCES' in dir(settings)
and settings.OMGEO_GEOCODER_SOURCES is not None)):
geocoder = Geocoder(settings.OMGEO_GEOCODER_SOURCES)
else:
geocoder = Geocoder()
results = geocoder.geocode(query)
if results is not False:
response = []
for result in results:
# some geocoders do not support passing a bounding box filter
if result_in_bounding_box(result):
response.append({
"match_addr": result.match_addr,
"x": result.x,
"y": result.y,
"score": result.score,
"locator": result.locator,
"geoservice": result.geoservice,
"wkid": result.wkid,
})
return response
else:
# This is not a very helpful error message, but omgeo as of
# v1.2 does not report failure details.
return {"error": "The geocoder failed to generate a list of results."}
def handle(self, *args, **options):
try:
g = Geocoder([
['omgeo.services.Bing',{
'settings':{
'api_key':'AufKQbvq8uGM3qsQWwMwJNtlf6LLxe5bPvOAVcxi-79Qp0tDl0T2qScdOQiBNKkE'
}
}],
# ['omgeo.services.Nominatim',{}],
# ['omgeo.services.MapQuest',{
# 'settings':{
# 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6'
# }
# }]
])
for s in Address.objects.all():
#get geocoded address for this address
address_string = s.string
print address_string
homeless = address_string.find('HOMELESS')
print homeless
if homeless == -1:
if len(address_string) > 0:
geocode_results = g.geocode(address_string,True)
geocode_candidates = geocode_results['candidates']
for c in geocode_candidates:
if c.confidence == 'High':
s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326)
s.save()
self.stdout.write(address_string)
self.stdout.write("Location found: %s %s\n" % (c.x, c.y,))
if not s.point_location:
self.stdout.write("No good geocode found for %s \n" % (address_string))
s.attempted = True
s.save()
except:
print "whoopsie for ", address_string
raise
class GeocodingFilter(filters.Filter):
""" Geocode contributions based on address.
"""
def __init__(self, *args, **kwargs):
super(GeocodingFilter, self).__init__(*args, **kwargs)
self.geocoder = Geocoder()
self.cache = {}
with open(os.path.join(PWD, 'data', 'geocache.csv')) as infile:
for record in csv.reader(infile):
self.cache[record[0]] = (record[1], record[2])
def process_record(self, record):
if not record.get('lat') and not record.get('lon'):
vals = (record['address'], record['city'], record['state'], record['zip'])
addr = "%s %s %s %s" % vals
if addr in self.cache:
ll = self.cache[addr]
record['lat'] = ll[0]
record['lon'] = ll[1]
else:
result = self.geocoder.geocode(addr)
candidates = result.get('candidates', None)
if candidates:
c = candidates[0]
record['lat'] = c.y
record['lon'] = c.x
return record
class GeocoderTest(OmgeoTestCase):
g = None # not set until set up
BING_KEY_REQUIRED_MSG = 'Enter a Bing Maps API key to run the bing tests'
def setUp(self):
# Viewbox objects
vb = {}
vb['callowhill'] = Viewbox(-75.162628, 39.962769, -75.150963, 39.956322)
# PlaceQuery objects
self.pq = {}
# North American Addresses:
self.pq['azavea'] = PlaceQuery('340 N 12th St Ste 402 Philadelphia PA')
self.pq['ambiguous_azavea'] = PlaceQuery('340 12th St Ste 402 Philadelphia PA')
self.pq['wolf'] = PlaceQuery('Wolf Building')
self.pq['wolf_philly'] = PlaceQuery('Wolf Building, Philadelphia PA')
self.pq['wolf_bounded'] = PlaceQuery('Wolf Building', viewbox=vb['callowhill'])
self.pq['alpha_774R_W_Central_Ave'] = PlaceQuery('774R W Central Ave Alpha NJ')
self.pq['alpha_774_W_Central_Ave_Rear'] = PlaceQuery('774 W Central Ave Rear Alpha NJ')
self.pq['8_kirkbride'] = PlaceQuery('8 Kirkbride Rd 08822')
self.pq['george_washington'] = PlaceQuery('201 W Montmorency Blvd, George, Washington')
self.pq['pine_needles_dr'] = PlaceQuery('11761 pine needles providence forge')
self.pq['pine_needles_ct'] = PlaceQuery('5328 pine needles providence forge')
self.pq['pine_needles_terr'] = PlaceQuery('5359 pine needles providence forge')
self.pq['moorestown_hyphenated'] = PlaceQuery('111-113 W Main St Moorestown NJ')
self.pq['willow_street'] = PlaceQuery('2819F Willow Street Pike Willow Street PA')
self.pq['quebec'] = PlaceQuery('756 Rue Berri Montreal QC', country='CA')
self.pq['quebec_accent'] = PlaceQuery('527 Ch. Beauséjour, Saint-Elzéar-de-Témiscouata QC')
self.pq['quebec_hyphenated'] = PlaceQuery('227-227A Rue Commerciale, Saint-Louis-du-Ha! Ha! QC')
# European Addresses:
self.pq['london_pieces'] = PlaceQuery(address='31 Maiden Lane', city='London', country='UK')
self.pq['london_one_line'] = PlaceQuery('31 Maiden Lane, London WC2E', country='UK')
self.pq['london_pieces_hyphenated'] = PlaceQuery(address='31-32 Maiden Lane', city='London', country='UK')
self.pq['london_one_line_hyphenated'] = PlaceQuery('31-32 Maiden Lane London WC2E', country='UK')
# Oceanian Addresses:
self.pq['karori'] = PlaceQuery('102 Karori Road Karori Wellington', country='NZ')
# Set up geocoders
if ESRI_MAPS_API_KEY is None:
esri_settings = {}
else:
esri_settings = dict(api_key=ESRI_MAPS_API_KEY)
self.service_esri_na = ['omgeo.services.EsriNA', dict(settings=esri_settings)]
self.service_esri_eu = ['omgeo.services.EsriEU', dict(settings=esri_settings)]
g_sources = [self.service_esri_na, self.service_esri_eu]
self.g_esri_na = Geocoder([self.service_esri_na])
self.g_esri_eu = Geocoder([self.service_esri_eu])
if BING_MAPS_API_KEY is not None:
bing_settings = dict(api_key=BING_MAPS_API_KEY)
self.service_bing = ['omgeo.services.Bing', dict(settings=bing_settings)]
g_sources.append(self.service_bing)
self.g_bing = Geocoder([self.service_bing])
self.service_nom = ['omgeo.services.Nominatim', {}]
g_sources.append(self.service_nom)
self.g_nom = Geocoder([self.service_nom])
self.g = Geocoder(sources=g_sources) # main geocoder used for tests
# other geocoders for services not used in self.g
self.g_dc = Geocoder([['omgeo.services.CitizenAtlas', {}]])
self.g_esri_na_soap = Geocoder([['omgeo.services.EsriNASoap', {}]])
self.g_esri_eu_soap = Geocoder([['omgeo.services.EsriEUSoap', {}]])
def tearDown(self):
pass
def test_geocode_azavea(self):
candidates = self.g.geocode(self.pq['azavea'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.')
def test_geocode_snap_points_1(self):
candidates = self.g.geocode(self.pq['8_kirkbride'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.')
@unittest.skipIf(BING_MAPS_API_KEY is None, BING_KEY_REQUIRED_MSG)
def test_geocode_snap_points_2(self):
candidates = self.g.geocode(self.pq['alpha_774_W_Central_Ave_Rear'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.')
def test_geocode_esri_na_us_soap(self):
candidates = self.g_esri_na_soap.geocode(PlaceQuery('340 N 12th St., Philadelphia, PA, US'))
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_esri_na_us(self):
candidates = self.g_esri_na.geocode(self.pq['alpha_774_W_Central_Ave_Rear'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_esri_eu_soap(self):
candidates = self.g_esri_eu_soap.geocode(PlaceQuery(
address='31 Maiden Lane', city='London', country='UK'))
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_esri_na_nz(self):
candidates = self.g_esri_na.geocode(self.pq['karori'])
self.assertEqual(len(candidates) > 0, False,
'Found New Zealand address when this should only'
'be using the North American ESRI geocoder.')
@unittest.skipIf(BING_MAPS_API_KEY is None, BING_KEY_REQUIRED_MSG)
def test_geocode_bing(self):
candidates = self.g_bing.geocode(self.pq['azavea'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_nom(self):
candidates = self.g_nom.geocode(PlaceQuery('1200 Callowhill St, Philadelphia, PA, 19123'))
x_type = type(candidates[0].x)
y_type = type(candidates[0].y)
self.assertEqual(x_type == float, True, 'x coord is of type %s instead of float' % x_type)
self.assertEqual(y_type == float, True, 'y coord is of type %s instead of float' % y_type)
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_dc_address(self):
candidates = self.g_dc.geocode(PlaceQuery('1600 pennsylvania'))
self.assertTrue(len(candidates) > 0, 'No candidates returned.')
self.assertTrue(candidates[0].locator == 'DC Address',
'Expected 1600 pennsylvania to be an address match')
def test_geocode_dc_intersection(self):
candidates = self.g_dc.geocode(PlaceQuery('h and 15th'))
self.assertTrue(len(candidates) > 0, 'No candidates returned.')
self.assertTrue(candidates[0].locator == 'DC Intersection',
'h and 15th to be an intersection match')
def test_geocode_dupepicker(self):
candidates = self.g.geocode(self.pq['ambiguous_azavea'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
@unittest.skipIf(BING_MAPS_API_KEY is None, BING_KEY_REQUIRED_MSG)
def test_geocode_karori(self):
def bldg_no_and_postal_in_addr(c):
return ('102' in c.match_addr and '6012' in c.match_addr)
candidates = self.g.geocode(self.pq['karori'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
self.assertEqual(any([bldg_no_and_postal_in_addr(c) for c in candidates]), True,
'Could not find bldg. no. "102" and postcode "6012" in any address.')
def _test_geocode_results_all_(self, verbosity=0, geocoder=Geocoder(),
expected_results=16):
"""
Geocode a list of addresses. Some of these only work with Bing so
fewer results are expected when Bing is not used as a geocoder
"""
if verbosity > 1:
logger.setLevel(logging.INFO)
queries_with_results = 0
for place in self.pq:
logger.info(place)
logger.info(len(place) * '-')
candidates = geocoder.geocode(self.pq[place])
if len(candidates) == 0:
logger.info('Input: %s\n(no results)' % self.pq[place].query)
else:
queries_with_results += 1
logger.info('Input: %s' % self.pq[place].query)
logger.info(map(lambda c: 'Output: %r (%s %s)\n' %
(c.match_addr,
c.geoservice,
[c.locator, c.score, c.confidence, c.entity]),
candidates))
self.assertEqual(expected_results, queries_with_results,
'Got results for %d of %d queries.' % (queries_with_results, len(self.pq)))
def _test_geocode_results_all(self):
if BING_MAPS_API_KEY is None:
expected_results=16
else:
self.g.add_source(['omgeo.services.Bing',
{'settings':{'api_key':BING_MAPS_API_KEY}}])
expected_results=len(self.pq)
self._test_geocode_results_all_(geocoder=self.g, expected_results=len(self.pq))
def test_esri_geocoder_na_default_override(self):
# Bug in 3.1 - the EsriNA and EsriEU append processors rather than
# replace
geocoder = Geocoder([['omgeo.services.EsriNA',
{'postprocessors': [AttrFilter([
'rooftop',
'interpolation',
'postal_specific'],
'locator')]}]])
self.assertEqual(1, len(geocoder._sources[0]._postprocessors),
'EsriNA geocoder incorrectly processed defaults')
self.assertEqual('AttrFilter',
geocoder._sources[0]._postprocessors[0].__class__.__name__,
'EsriNA geocoder incorrectly processed defaults')
def test_esri_geocoder_eu_default_override(self):
# Bug in 3.1 - the EsriNA and EsriEU append processors rather than
# replace
geocoder = Geocoder([['omgeo.services.EsriEU',
{'postprocessors': [AttrFilter([
'rooftop',
'interpolation',
'postal_specific'],
'locator')]}]])
self.assertEqual(1, len(geocoder._sources[0]._postprocessors),
'EsriEU geocoder incorrectly processed defaults')
self.assertEqual('AttrFilter',
geocoder._sources[0]._postprocessors[0].__class__.__name__,
'EsriEU geocoder incorrectly processed defaults')
class AddressConverter(object):
def __init__(self, curs):
self.curs = curs
self.geolocator = GoogleV3()
self.nominatim = nominatim.Geocoder()
self.geocoder = Geocoder()
self.re_numbers = re.compile("^\d+$")
def reject_all_numbers(self, address):
if self.re_numbers.search(address):
logger.info("Rejected by reject_all_numbers")
return True
return False
def reject_short_words(self, address, length=4):
if len(address) <= length:
logger.info("Rejected by reject_short_words")
return True
return False
def reject_large_main_places(self, address, threshold=15000):
if address in main_places:
population = main_places[address]["Population"]
if int(population) >= threshold:
return True
return False
def reject_resolution_to_main_place(self, address, threshold=15000):
parts = address.split(",")
first = parts[0].strip().lower()
return self.reject_large_main_places(first, threshold)
def reject_partial_match(self, result):
if "partial_match" in result and result["partial_match"]:
return True
return False
def resolve_address_google(self, address, **kwargs):
try:
encoded_address = encode(address)
address = urllib.quote(encoded_address)
url = "https://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false®ion=za&key=%s" % (address, configuration["environment"]["google_key"])
response = urllib2.urlopen(url)
js = response.read()
try:
js = json.loads(js)
except ValueError:
logger.exception("Error trying to resolve %s" % address)
return None
results = []
if "results" in js and len(js["results"]) > 0:
for result in js["results"]:
res = self.reject_partial_match(result)
if res: continue
if "reject_resolution_to_main_place" in kwargs:
try:
res = self.reject_resolution_to_main_place(result["formatted_address"], int(kwargs["reject_resolution_to_main_place"][0]))
except (ValueError, TypeError):
res = self.resolution_to_main_place(result["formatted_address"])
if res: continue
geom = result["geometry"]["location"]
results.append({
"lat" : geom["lat"],
"lng" : geom["lng"],
"formatted_address" : result["formatted_address"],
"source" : "Google Geocoding API",
})
if len(results) == 0: return None
return results
except Exception:
logger.exception("Error trying to resolve %s" % address)
return None
def resolve_address_esri(self, address):
result = self.geocoder.geocode(address + ", South Africa")
if not result: return None
if not "candidates" in result: return None
if len(result["candidates"]) == 0: return None
candidate = result["candidates"][0]
address = candidate.match_addr
latitude = candidate.y
longitude = candidate.x
return address, latitude, longitude
def resolve_address_nominatim(self, address, **kwargs):
encoded_address = encode(address)
results = self.nominatim.geocode(encoded_address)
return [
{
"lat" : r["lat"],
"lng" : r["lon"],
"formatted_address" : r["display_name"],
"source" : "Nominatim",
}
for r in results
]
def convert_address(self, address, **kwargs):
address = address.strip()
if address == "": return None
if "reject_numbers" in kwargs and self.reject_all_numbers(address): return None
if "reject_short_words" in kwargs:
try:
val = self.reject_short_words(address, int(kwargs["reject_short_words"][0]))
except (TypeError, ValueError):
val = self.reject_short_words(address)
if val: return None
if "reject_large_main_places" in kwargs:
try:
val = self.reject_large_main_places(address, int(kwargs["reject_large_main_places"][0]))
except (TypeError, ValueError):
val = self.reject_large_main_places(address)
if val: return None
if not "south africa" in address.lower():
address = address + ", South Africa"
if "enable_nominatim" in kwargs:
results = self.resolve_address_nominatim(address, **kwargs)
if len(results) > 0:
return results
return self.resolve_address_google(address, **kwargs)
def convert_to_geography(self, sql, latitude, longitude):
poi = (longitude, latitude)
self.curs.execute(sql, poi)
return self.curs.fetchall()
class AddressConverter(object):
def __init__(self, curs):
self.curs = curs
self.geolocator = GoogleV3()
self.nominatim = nominatim.Nominatim()
self.geocoder = Geocoder()
self.re_numbers = re.compile("^\d+$")
def reject_all_numbers(self, address):
if self.re_numbers.search(address):
logger.info("Rejected by reject_all_numbers")
return True
return False
def reject_short_words(self, address, length=4):
if len(address) <= length:
logger.info("Rejected by reject_short_words")
return True
return False
def reject_large_main_places(self, address, threshold=15000):
if address in main_places:
population = main_places[address]["Population"]
if int(population) >= threshold:
return True
return False
def reject_resolution_to_main_place(self, address, threshold=15000):
parts = address.split(",")
first = parts[0].strip().lower()
return self.reject_large_main_places(first, threshold)
def reject_partial_match(self, result):
if "partial_match" in result and result["partial_match"]:
return True
return False
def resolve_address_google(self, address, **kwargs):
encoded_address = encode(address)
address = urllib.quote(encoded_address)
url = "https://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false®ion=za&key=%s" % (
address, GOOGLE_API_KEY)
response = urllib2.urlopen(url)
js = response.read()
try:
js = json.loads(js)
except ValueError as e:
logger.exception("Error trying to resolve %s" % address)
raise StandardError("Couldn't resolve %s: %s" %
(address, e.message))
results = []
if "status" in js and js["status"] not in ("OK", "ZERO_RESULTS"):
logger.error(
"Error trying to resolve %s - %s (%s)" %
(address, js.get("error_message", "Generic Error"), js))
raise StandardError("Couldn't resolve %s: %s" %
(address, js.get("error_message")))
if "results" in js and len(js["results"]) > 0:
for result in js["results"]:
res = self.reject_partial_match(result)
if res: continue
if "reject_resolution_to_main_place" in kwargs:
try:
res = self.reject_resolution_to_main_place(
result["formatted_address"],
int(kwargs["reject_resolution_to_main_place"][0]))
except (ValueError, TypeError):
res = self.resolution_to_main_place(
result["formatted_address"])
if res: continue
geom = result["geometry"]["location"]
results.append({
"lat": geom["lat"],
"lng": geom["lng"],
"formatted_address": result["formatted_address"],
"source": "Google Geocoding API",
})
if len(results) == 0: return None
return results
def resolve_address_esri(self, address):
result = self.geocoder.geocode(address + ", South Africa")
if not result: return None
if not "candidates" in result: return None
if len(result["candidates"]) == 0: return None
candidate = result["candidates"][0]
address = candidate.match_addr
latitude = candidate.y
longitude = candidate.x
return address, latitude, longitude
def resolve_address_nominatim(self, address, **kwargs):
encoded_address = encode(address)
results = self.nominatim.query(encoded_address)
return [{
"lat": r["lat"],
"lng": r["lon"],
"formatted_address": r["display_name"],
"source": "Nominatim",
} for r in results]
def convert_address(self, address, **kwargs):
address = address.strip()
if address == "": return None
if re_is_latlng.match(address):
return self.resolve_coords(address)
if "reject_numbers" in kwargs and self.reject_all_numbers(address):
return None
if "reject_short_words" in kwargs:
try:
val = self.reject_short_words(
address, int(kwargs["reject_short_words"][0]))
except (TypeError, ValueError):
val = self.reject_short_words(address)
if val: return None
if "reject_large_main_places" in kwargs:
try:
val = self.reject_large_main_places(
address, int(kwargs["reject_large_main_places"][0]))
except (TypeError, ValueError):
val = self.reject_large_main_places(address)
if val: return None
if not "south africa" in address.lower():
address = address + ", South Africa"
if "enable_nominatim" in kwargs:
results = self.resolve_address_nominatim(address, **kwargs)
if len(results) > 0:
return results
return self.resolve_address_google(address, **kwargs)
def convert_to_geography(self, sql, latitude, longitude):
poi = (longitude, latitude)
self.curs.execute(sql, poi)
return self.curs.fetchall()
def resolve_coords(self, coords):
try:
lat, lng = [float(s) for s in coords.split(',', 1)]
return [{
"lat": lat,
"lng": lng,
"formatted_address": "%s, %s" % (lat, lng),
"source": "Direct",
}]
except ValueError:
return None
"zip" : str(s.zip5),
"Proper Address" : paddress,
"Latitude" : '',
"Longitude" : ''}
students.append(student)
print 'records now exist in "students" list'
#Looping through list of dictionaries and adding geocoded results
## Default geocoding service is EsriWGS
i = len(students)
print '%s records remaining' % str(i)
for s in students:
i -= 1
gcode = g.geocode(s["Proper Address"])
try:
s["geocode response"] = gcode["candidates"][0]
s["Latitude"] = gcode["candidates"][0].y
s["Longitude"] = gcode["candidates"][0].x
print 'Good: %s records remaining' % str(i)
except KeyboardInterrupt:
break
except:
errors.append(s)
print 'Bad: %s records remaining' % str(i)
continue
print 'Geocoding Complete'
class GeocoderTest(OmgeoTestCase):
g = None # not set until set up
def setUp(self):
# Viewbox objects
vb = {}
vb['callowhill'] = Viewbox(-75.162628, 39.962769, -75.150963, 39.956322)
# PlaceQuery objects
self.pq = {}
# North American Addresses:
self.pq['azavea'] = PlaceQuery('340 N 12th St Ste 402 Philadelphia PA')
self.pq['ambiguous_azavea'] = PlaceQuery('340 12th St Ste 402 Philadelphia PA')
self.pq['wolf'] = PlaceQuery('Wolf Building')
self.pq['wolf_philly'] = PlaceQuery('Wolf Building, Philadelphia PA')
self.pq['wolf_bounded'] = PlaceQuery('Wolf Building', viewbox=vb['callowhill'])
self.pq['alpha_774R_W_Central_Ave'] = PlaceQuery('774R W Central Ave Alpha NJ')
self.pq['alpha_774_W_Central_Ave_Rear'] = PlaceQuery('774 W Central Ave Rear Alpha NJ')
self.pq['pine_needles_dr'] = PlaceQuery('11761 pine needles providence forge')
self.pq['pine_needles_ct'] = PlaceQuery('5328 pine needles providence forge')
self.pq['pine_needles_terr'] = PlaceQuery('5359 pine needles providence forge')
self.pq['moorestown_hyphenated'] = PlaceQuery('111-113 W Main St Moorestown NJ')
self.pq['willow_street'] = PlaceQuery('2819F Willow Street Pike Willow Street PA')
self.pq['quebec'] = PlaceQuery('756 Rue Berri Montreal QC', country='CA')
self.pq['quebec_bang'] = PlaceQuery('177 Rue Commerciale, Saint-Louis-du-Ha! Ha! QC')
self.pq['quebec_accent'] = PlaceQuery('527 Ch. Beauséjour, Saint-Elzéar-de-Témiscouata QC')
self.pq['quebec_hyphenated'] = PlaceQuery('227-227A Rue Commerciale, Saint-Louis-du-Ha! Ha! QC')
# European Addresses:
self.pq['london_pieces'] = PlaceQuery(address='31 Maiden Lane', city='London', country='UK')
self.pq['london_one_line'] = PlaceQuery('31 Maiden Lane, London WC2E', country='UK')
self.pq['london_pieces_hyphenated'] = PlaceQuery(address='31-32 Maiden Lane', city='London', country='UK')
self.pq['london_one_line_hyphenated'] = PlaceQuery('31-32 Maiden Lane London WC2E', country='UK')
# Oceanic Addresses:
self.pq['karori'] = PlaceQuery('102 Karori Road Karori Wellington', country='NZ')
# Geocoder objects
self.g = Geocoder()
if ESRI_MAPS_API_KEY is not None:
self.g_esri_na = Geocoder([['omgeo.services.EsriNA',
{'settings':{'api_key':ESRI_MAPS_API_KEY}}]])
self.g_esri_eu = Geocoder([['omgeo.services.EsriEU',
{'settings':{'api_key':ESRI_MAPS_API_KEY}}]])
else:
self.g_esri_na = Geocoder([['omgeo.services.EsriNA', {}]])
self.g_esri_eu = Geocoder([['omgeo.services.EsriEU', {}]])
self.g_bing = Geocoder([['omgeo.services.Bing', {'settings':{'api_key':BING_MAPS_API_KEY}}]])
self.g_nom = Geocoder([['omgeo.services.Nominatim',{}]])
self.g_dc = Geocoder([['omgeo.services.CitizenAtlas', {}]])
def tearDown(self):
pass
def test_geocode_azavea(self):
candidates = self.g.geocode(self.pq['azavea'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.')
def test_geocode_esri_na_us(self):
candidates = self.g_esri_na.geocode(self.pq['alpha_774_W_Central_Ave_Rear'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_esri_na_nz(self):
candidates = self.g_esri_na.geocode(self.pq['karori'])
self.assertEqual(len(candidates) > 0, False, 'Found New Zealand address when this should only be using the North American ESRI geocoder.')
def test_nom_bounded(self):
candidates = self.g.geocode(PlaceQuery('8 Kirkbride Rd'))
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
@unittest.skipIf(BING_MAPS_API_KEY is None, 'Enter a Bing Maps API key to run the bing tests')
def test_geocode_bing(self):
candidates = self.g_bing.geocode(self.pq['azavea'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def test_geocode_dc_address(self):
candidates = self.g_dc.geocode(PlaceQuery('1600 pennsylvania'))
self.assertTrue(len(candidates) > 0, 'No candidates returned.')
self.assertTrue(candidates[0].locator == 'DC Address', 'Expected 1600 pennsylvania to be an address match')
def test_geocode_dc_intersection(self):
candidates = self.g_dc.geocode(PlaceQuery('h and 15th'))
self.assertTrue(len(candidates) > 0, 'No candidates returned.')
self.assertTrue(candidates[0].locator == 'DC Intersection', 'h and 15th to be an intersection match')
def test_geocode_dupepicker(self):
candidates = self.g.geocode(self.pq['ambiguous_azavea'])
self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
def _test_geocode_results_all(self, verbosity=0, geocoder=Geocoder()):
queries_with_results = 0
for place in self.pq:
if verbosity > 1: print '\n%s' % place
if verbosity > 1: print len(place) * '-'
candidates = geocoder.geocode(self.pq[place])
if len(candidates) == 0:
if verbosity > 1: print 'Input: %s\n(no results)' % self.pq[place].query
else:
queries_with_results += 1
if verbosity > 1: print 'Input: %s' % self.pq[place].query
for x in ['Output: %r (%s %s)' % (c.match_addr, c.geoservice, [c.locator, c.score, c.confidence, c.entity]) for c in candidates]:
if verbosity > 1: print x
self.assertEqual(len(self.pq), queries_with_results, 'Got results for %d of %d queries.' % (queries_with_results, len(self.pq)))
def test_geocode_results_all(self):
if BING_MAPS_API_KEY is not None:
self.g.add_source(['omgeo.services.Bing',
{'settings':{'api_key':BING_MAPS_API_KEY}}])
self._test_geocode_results_all(geocoder=self.g)
def getBranches(provinceId,
localidadId,
localidadName,
provinceName,
extracash=False):
try:
fixpath = root + paths['fix'] + '/rapipago_fix.csv'
toFix = []
g = Geocoder()
extracash = 'true' if extracash else 'false'
res = requests.get(
'https://www.rapipago.com.ar/rapipagoWeb/index.php/resultado_sucursales?prov='
+ provinceId + '&loc=' + localidadId + '&extracash=' + extracash)
if res.status_code != 200:
raise ValueError('Error solicitando sucursales.')
soup = BeautifulSoup(res.text, 'lxml')
content = soup.find_all('div', class_="w-row accordion_content")
branches = []
#chequeo si el sitio posee sucursales
if content != None:
for c in content:
try:
global toPendingFixCount
withoutError = True
title = c.select('.text_expand_drop')[0].get_text()
address = c.select('.text_expand_drop')[1].get_text()
attentionHour = c.select(
'.text_horarios_drop')[0].get_text().replace(
'Horarios de atención:',
'').replace('\n', '').replace(' ', '')
data = [localidadName, title, address, attentionHour]
latlngExt = c.find_all('a')
#chequeo si encontro etiquetas de latitud y longitud
if len(latlngExt) > 0:
latlngExt = str(latlngExt[0])
lat = findBetweenR(latlngExt, 'data-lat="',
'" data-lng')
lng = findBetweenR(latlngExt, 'data-lng="',
'" data-marker')
data.append(lat)
data.append(lng)
#al no encontrar o no poseer con exactitud, controla si ya fueron definidas manualmente
if len(latlngExt) == 0 or (lat == '0.00000000'
or lng == '0.00000000'):
fixNotFound = True
with open(fixpath, 'r') as f:
reader = csv.reader(f)
fixList = list(reader)
search = [provinceName, localidadName, title, address]
for i, lc in enumerate(fixList):
concurrence = 0
for co in search:
if co in lc:
concurrence = concurrence + 1
if concurrence == len(search):
data = lc[1:len(lc)]
fixNotFound = False
#no esta la sucursal definida manualmente e intenta geolocalizarla
if fixNotFound:
try:
result = g.geocode(address + ', ' +
localidadName +
', Argentina')
lat = result['candidates'][0].x
lng = result['candidates'][0].y
except Exception as e:
#no logro geolocalizarla y sera puesta a la lista manual
withoutError = False
try:
mode = 'a' if os.path.exists(
fixpath) else 'w'
listString = ','.join(data)
if len(latlngExt) == 0:
listString = listString + ',0.00000000,0.00000000'
with open(fixpath, mode) as _file:
_file.write('\n' + provinceName + ',' +
(listString))
toPendingFixCount = toPendingFixCount + 1
except Exception as e:
print(e)
if withoutError:
branches.append(data)
except Exception as e:
print(e)
return branches
return []
except Exception as e:
print(e)
