def handle(self, *args, **options): # from omgeo import Geocoder g = Geocoder([ [ 'omgeo.services.Bing', { 'settings': { 'api_key': 'AnoU4IZjQyL3BLIVJNmfpUgqh86Z8jVEk4n2uQNRzSWAu1Suu5I1NvmjGslFS1Dk' } } ], # ['omgeo.services.Nominatim',{}], # ['omgeo.services.MapQuest',{ # 'settings':{ # 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6' # } # }] ]) for s in Location.objects.all(): #get geocoded address for this address address_string = '%s, %s' % (s.name, 'Champaign, IL') geocode_results = g.geocode(address_string, True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if c.confidence == 'High': s.point_location = GEOSGeometry( 'POINT(%s %s)' % ( c.x, c.y, ), 4326) s.save() self.stdout.write("Location found: %s %s\n" % ( c.x, c.y, )) if not s.point_location: address_string = '%s, %s' % (s.name, 'Urbana, IL') geocode_results = g.geocode(address_string, True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if c.confidence == 'High': s.point_location = GEOSGeometry( 'POINT(%s %s)' % ( c.x, c.y, ), 4326) s.save() self.stdout.write("Location found: %s %s\n" % ( c.x, c.y, )) if not s.point_location: self.stdout.write("No good geocode found\n")
def geocode(address): """Given an address, returns candidates.""" pq = PlaceQuery(address) global _geocoder if _geocoder is None: _geocoder = Geocoder() return _geocoder.geocode(pq)
def handle(self, *args, **options): # from omgeo import Geocoder g = Geocoder([ ['omgeo.services.Bing',{ 'settings':{ 'api_key':'AnoU4IZjQyL3BLIVJNmfpUgqh86Z8jVEk4n2uQNRzSWAu1Suu5I1NvmjGslFS1Dk' } }], # ['omgeo.services.Nominatim',{}], # ['omgeo.services.MapQuest',{ # 'settings':{ # 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6' # } # }] ]) for s in BPStation.objects.filter(address__isnull=False): #get geocoded address for this address address_string = '%s, %s, %s, %s' % (s.address, s.city, s.state, s.zip,) geocode_results = g.geocode(address_string,True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if c.confidence == 'High': s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326) s.save() self.stdout.write("Location found: %s %s\n" % (c.x, c.y,)) if not s.point_location: self.stdout.write("No good geocode found\n") #manual override of locations that don't geocode well try: if len(args) > 0: self.stdout.write('Using manual CSV location ...\n') file_path = args[0] else: self.stdout.write('Using default CSV location ...\n') working_dir = os.path.join(SITE_ROOT, '../data') file_path = os.path.join(working_dir, 'geocodestragglers20120830.csv') handle = csv.reader(open(file_path, 'rU'), delimiter=',', quotechar='"') header = handle.next() for h in handle: print h try: station = BPStation.objects.get(name=h[1].strip()) station.point_location = GEOSGeometry('POINT(%s %s)' % (h[7].strip(), h[8].strip(),),4326) station.save() self.stdout.write('%s: %s %s\n' % (h[0].strip(), h[7].strip(), h[8].strip(),)) except: raise self.stdout.write('Successfully loaded manual Border Patrol geocodes.\n') except AttributeError: raise
def load_data(inPath, outPath): outFile = open(outPath, 'w') counter = 0 with open(inPath,'Ur') as inFile: for line in inFile: GarageID,Zip,Street,City,State,cnt = line.split(",") address = Street+" "+City+" "+State+" "+"USA" g = Geocoder() result = g.geocode(address) outFile.write('%s\n' % (result))
def handle(self, *args, **options): # from omgeo import Geocoder g = Geocoder([ ['omgeo.services.Bing',{ 'settings':{ 'api_key':'AnoU4IZjQyL3BLIVJNmfpUgqh86Z8jVEk4n2uQNRzSWAu1Suu5I1NvmjGslFS1Dk' } }], # ['omgeo.services.Nominatim',{}], # ['omgeo.services.MapQuest',{ # 'settings':{ # 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6' # } # }] ]) for s in Location.objects.all(): #get geocoded address for this address address_string = '%s, %s' % (s.name, 'Champaign, IL') geocode_results = g.geocode(address_string,True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if c.confidence == 'High': s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326) s.save() self.stdout.write("Location found: %s %s\n" % (c.x, c.y,)) if not s.point_location: address_string = '%s, %s' % (s.name, 'Urbana, IL') geocode_results = g.geocode(address_string,True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if c.confidence == 'High': s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326) s.save() self.stdout.write("Location found: %s %s\n" % (c.x, c.y,)) if not s.point_location: self.stdout.write("No good geocode found\n")
def __init__(self, address): """Geocode given address and set self.coords to (x,y) tuple.""" geocoder = Geocoder() georesult = geocoder.geocode(address) geocandidates = georesult['candidates'] coord_pairs = [(c.x, c.y) for c in geocandidates] num_coord_pairs = len(coord_pairs) if num_coord_pairs == 0: raise Exception('No geocoding results for %s' % address) if num_coord_pairs > 1: choices = [c.match_addr for c in geocandidates] raise Exception('Ambiguous address. Choices are:\n%s' % choices) self.coords = coord_pairs[0]
def handle(self, *args, **options): parser = StreetAddressParser() #try: g = Geocoder([ ['omgeo.services.Bing',{ 'settings':{ 'api_key':'AufKQbvq8uGM3qsQWwMwJNtlf6LLxe5bPvOAVcxi-79Qp0tDl0T2qScdOQiBNKkE' } }], ]) for location in Location.objects.filter(point_location__isnull=True): string = location.address.strip() if location.city is not None: string = string + ", " + location.city else: if location.agency: string = string + ", " + location.agency.name + " IL" else: string = string + ", Champaign, IL" block_pattern = re.compile('BLOCK OF') cleaned_address = re.sub(block_pattern,'', string) print cleaned_address if location.intersection_indicator: slash = re.compile('/') string = re.sub(slash,'at', string) print string if len(location.address.strip()) > 0: geocode_results = g.geocode(cleaned_address,True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if math.floor(c.x) == -89.0 and math.floor(c.y) == 40.0: print "within acceptable ranges" if c.confidence == 'High': location.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326) location.save() self.stdout.write("Location found: %s %s\n" % (c.x, c.y,)) if not location.point_location: self.stdout.write("No good geocode found for %s \n" % (cleaned_address)) #location.attempted = True location.save() #except (RuntimeError, TypeError, NameError): # print "whoopsie"
def geocode_address(request, address): def result_in_bounding_box(result): x = float(result.x) y = float(result.y) left = float(settings.BOUNDING_BOX['left']) top = float(settings.BOUNDING_BOX['top']) right = float(settings.BOUNDING_BOX['right']) bottom = float(settings.BOUNDING_BOX['bottom']) return x > left and x < right and y > bottom and y < top if address is None or len(address) == 0: raise HttpBadRequestException("No address specfified") query = PlaceQuery(address, viewbox=Viewbox( settings.BOUNDING_BOX['left'], settings.BOUNDING_BOX['top'], settings.BOUNDING_BOX['right'], settings.BOUNDING_BOX['bottom']) ) if (('OMGEO_GEOCODER_SOURCES' in dir(settings) and settings.OMGEO_GEOCODER_SOURCES is not None)): geocoder = Geocoder(settings.OMGEO_GEOCODER_SOURCES) else: geocoder = Geocoder() results = geocoder.geocode(query) if results is not False: response = [] for result in results: # some geocoders do not support passing a bounding box filter if result_in_bounding_box(result): response.append({ "match_addr": result.match_addr, "x": result.x, "y": result.y, "score": result.score, "locator": result.locator, "geoservice": result.geoservice, "wkid": result.wkid, }) return response else: # This is not a very helpful error message, but omgeo as of # v1.2 does not report failure details. return {"error": "The geocoder failed to generate a list of results."}
def handle(self, *args, **options): try: g = Geocoder([ ['omgeo.services.Bing',{ 'settings':{ 'api_key':'AufKQbvq8uGM3qsQWwMwJNtlf6LLxe5bPvOAVcxi-79Qp0tDl0T2qScdOQiBNKkE' } }], # ['omgeo.services.Nominatim',{}], # ['omgeo.services.MapQuest',{ # 'settings':{ # 'api_key':'Fmjtd%7Cluua2501lu%2C2a%3Do5-962gq6' # } # }] ]) for s in Address.objects.all(): #get geocoded address for this address address_string = s.string print address_string homeless = address_string.find('HOMELESS') print homeless if homeless == -1: if len(address_string) > 0: geocode_results = g.geocode(address_string,True) geocode_candidates = geocode_results['candidates'] for c in geocode_candidates: if c.confidence == 'High': s.point_location = GEOSGeometry('POINT(%s %s)' % (c.x, c.y,),4326) s.save() self.stdout.write(address_string) self.stdout.write("Location found: %s %s\n" % (c.x, c.y,)) if not s.point_location: self.stdout.write("No good geocode found for %s \n" % (address_string)) s.attempted = True s.save() except: print "whoopsie for ", address_string raise
class GeocodingFilter(filters.Filter): """ Geocode contributions based on address. """ def __init__(self, *args, **kwargs): super(GeocodingFilter, self).__init__(*args, **kwargs) self.geocoder = Geocoder() self.cache = {} with open(os.path.join(PWD, 'data', 'geocache.csv')) as infile: for record in csv.reader(infile): self.cache[record[0]] = (record[1], record[2]) def process_record(self, record): if not record.get('lat') and not record.get('lon'): vals = (record['address'], record['city'], record['state'], record['zip']) addr = "%s %s %s %s" % vals if addr in self.cache: ll = self.cache[addr] record['lat'] = ll[0] record['lon'] = ll[1] else: result = self.geocoder.geocode(addr) candidates = result.get('candidates', None) if candidates: c = candidates[0] record['lat'] = c.y record['lon'] = c.x return record
class GeocoderTest(OmgeoTestCase): g = None # not set until set up BING_KEY_REQUIRED_MSG = 'Enter a Bing Maps API key to run the bing tests' def setUp(self): # Viewbox objects vb = {} vb['callowhill'] = Viewbox(-75.162628, 39.962769, -75.150963, 39.956322) # PlaceQuery objects self.pq = {} # North American Addresses: self.pq['azavea'] = PlaceQuery('340 N 12th St Ste 402 Philadelphia PA') self.pq['ambiguous_azavea'] = PlaceQuery('340 12th St Ste 402 Philadelphia PA') self.pq['wolf'] = PlaceQuery('Wolf Building') self.pq['wolf_philly'] = PlaceQuery('Wolf Building, Philadelphia PA') self.pq['wolf_bounded'] = PlaceQuery('Wolf Building', viewbox=vb['callowhill']) self.pq['alpha_774R_W_Central_Ave'] = PlaceQuery('774R W Central Ave Alpha NJ') self.pq['alpha_774_W_Central_Ave_Rear'] = PlaceQuery('774 W Central Ave Rear Alpha NJ') self.pq['8_kirkbride'] = PlaceQuery('8 Kirkbride Rd 08822') self.pq['george_washington'] = PlaceQuery('201 W Montmorency Blvd, George, Washington') self.pq['pine_needles_dr'] = PlaceQuery('11761 pine needles providence forge') self.pq['pine_needles_ct'] = PlaceQuery('5328 pine needles providence forge') self.pq['pine_needles_terr'] = PlaceQuery('5359 pine needles providence forge') self.pq['moorestown_hyphenated'] = PlaceQuery('111-113 W Main St Moorestown NJ') self.pq['willow_street'] = PlaceQuery('2819F Willow Street Pike Willow Street PA') self.pq['quebec'] = PlaceQuery('756 Rue Berri Montreal QC', country='CA') self.pq['quebec_accent'] = PlaceQuery('527 Ch. Beauséjour, Saint-Elzéar-de-Témiscouata QC') self.pq['quebec_hyphenated'] = PlaceQuery('227-227A Rue Commerciale, Saint-Louis-du-Ha! Ha! QC') # European Addresses: self.pq['london_pieces'] = PlaceQuery(address='31 Maiden Lane', city='London', country='UK') self.pq['london_one_line'] = PlaceQuery('31 Maiden Lane, London WC2E', country='UK') self.pq['london_pieces_hyphenated'] = PlaceQuery(address='31-32 Maiden Lane', city='London', country='UK') self.pq['london_one_line_hyphenated'] = PlaceQuery('31-32 Maiden Lane London WC2E', country='UK') # Oceanian Addresses: self.pq['karori'] = PlaceQuery('102 Karori Road Karori Wellington', country='NZ') # Set up geocoders if ESRI_MAPS_API_KEY is None: esri_settings = {} else: esri_settings = dict(api_key=ESRI_MAPS_API_KEY) self.service_esri_na = ['omgeo.services.EsriNA', dict(settings=esri_settings)] self.service_esri_eu = ['omgeo.services.EsriEU', dict(settings=esri_settings)] g_sources = [self.service_esri_na, self.service_esri_eu] self.g_esri_na = Geocoder([self.service_esri_na]) self.g_esri_eu = Geocoder([self.service_esri_eu]) if BING_MAPS_API_KEY is not None: bing_settings = dict(api_key=BING_MAPS_API_KEY) self.service_bing = ['omgeo.services.Bing', dict(settings=bing_settings)] g_sources.append(self.service_bing) self.g_bing = Geocoder([self.service_bing]) self.service_nom = ['omgeo.services.Nominatim', {}] g_sources.append(self.service_nom) self.g_nom = Geocoder([self.service_nom]) self.g = Geocoder(sources=g_sources) # main geocoder used for tests # other geocoders for services not used in self.g self.g_dc = Geocoder([['omgeo.services.CitizenAtlas', {}]]) self.g_esri_na_soap = Geocoder([['omgeo.services.EsriNASoap', {}]]) self.g_esri_eu_soap = Geocoder([['omgeo.services.EsriEUSoap', {}]]) def tearDown(self): pass def test_geocode_azavea(self): candidates = self.g.geocode(self.pq['azavea']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.') def test_geocode_snap_points_1(self): candidates = self.g.geocode(self.pq['8_kirkbride']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.') @unittest.skipIf(BING_MAPS_API_KEY is None, BING_KEY_REQUIRED_MSG) def test_geocode_snap_points_2(self): candidates = self.g.geocode(self.pq['alpha_774_W_Central_Ave_Rear']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.') def test_geocode_esri_na_us_soap(self): candidates = self.g_esri_na_soap.geocode(PlaceQuery('340 N 12th St., Philadelphia, PA, US')) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_esri_na_us(self): candidates = self.g_esri_na.geocode(self.pq['alpha_774_W_Central_Ave_Rear']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_esri_eu_soap(self): candidates = self.g_esri_eu_soap.geocode(PlaceQuery( address='31 Maiden Lane', city='London', country='UK')) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_esri_na_nz(self): candidates = self.g_esri_na.geocode(self.pq['karori']) self.assertEqual(len(candidates) > 0, False, 'Found New Zealand address when this should only' 'be using the North American ESRI geocoder.') @unittest.skipIf(BING_MAPS_API_KEY is None, BING_KEY_REQUIRED_MSG) def test_geocode_bing(self): candidates = self.g_bing.geocode(self.pq['azavea']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_nom(self): candidates = self.g_nom.geocode(PlaceQuery('1200 Callowhill St, Philadelphia, PA, 19123')) x_type = type(candidates[0].x) y_type = type(candidates[0].y) self.assertEqual(x_type == float, True, 'x coord is of type %s instead of float' % x_type) self.assertEqual(y_type == float, True, 'y coord is of type %s instead of float' % y_type) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_dc_address(self): candidates = self.g_dc.geocode(PlaceQuery('1600 pennsylvania')) self.assertTrue(len(candidates) > 0, 'No candidates returned.') self.assertTrue(candidates[0].locator == 'DC Address', 'Expected 1600 pennsylvania to be an address match') def test_geocode_dc_intersection(self): candidates = self.g_dc.geocode(PlaceQuery('h and 15th')) self.assertTrue(len(candidates) > 0, 'No candidates returned.') self.assertTrue(candidates[0].locator == 'DC Intersection', 'h and 15th to be an intersection match') def test_geocode_dupepicker(self): candidates = self.g.geocode(self.pq['ambiguous_azavea']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') @unittest.skipIf(BING_MAPS_API_KEY is None, BING_KEY_REQUIRED_MSG) def test_geocode_karori(self): def bldg_no_and_postal_in_addr(c): return ('102' in c.match_addr and '6012' in c.match_addr) candidates = self.g.geocode(self.pq['karori']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') self.assertEqual(any([bldg_no_and_postal_in_addr(c) for c in candidates]), True, 'Could not find bldg. no. "102" and postcode "6012" in any address.') def _test_geocode_results_all_(self, verbosity=0, geocoder=Geocoder(), expected_results=16): """ Geocode a list of addresses. Some of these only work with Bing so fewer results are expected when Bing is not used as a geocoder """ if verbosity > 1: logger.setLevel(logging.INFO) queries_with_results = 0 for place in self.pq: logger.info(place) logger.info(len(place) * '-') candidates = geocoder.geocode(self.pq[place]) if len(candidates) == 0: logger.info('Input: %s\n(no results)' % self.pq[place].query) else: queries_with_results += 1 logger.info('Input: %s' % self.pq[place].query) logger.info(map(lambda c: 'Output: %r (%s %s)\n' % (c.match_addr, c.geoservice, [c.locator, c.score, c.confidence, c.entity]), candidates)) self.assertEqual(expected_results, queries_with_results, 'Got results for %d of %d queries.' % (queries_with_results, len(self.pq))) def _test_geocode_results_all(self): if BING_MAPS_API_KEY is None: expected_results=16 else: self.g.add_source(['omgeo.services.Bing', {'settings':{'api_key':BING_MAPS_API_KEY}}]) expected_results=len(self.pq) self._test_geocode_results_all_(geocoder=self.g, expected_results=len(self.pq)) def test_esri_geocoder_na_default_override(self): # Bug in 3.1 - the EsriNA and EsriEU append processors rather than # replace geocoder = Geocoder([['omgeo.services.EsriNA', {'postprocessors': [AttrFilter([ 'rooftop', 'interpolation', 'postal_specific'], 'locator')]}]]) self.assertEqual(1, len(geocoder._sources[0]._postprocessors), 'EsriNA geocoder incorrectly processed defaults') self.assertEqual('AttrFilter', geocoder._sources[0]._postprocessors[0].__class__.__name__, 'EsriNA geocoder incorrectly processed defaults') def test_esri_geocoder_eu_default_override(self): # Bug in 3.1 - the EsriNA and EsriEU append processors rather than # replace geocoder = Geocoder([['omgeo.services.EsriEU', {'postprocessors': [AttrFilter([ 'rooftop', 'interpolation', 'postal_specific'], 'locator')]}]]) self.assertEqual(1, len(geocoder._sources[0]._postprocessors), 'EsriEU geocoder incorrectly processed defaults') self.assertEqual('AttrFilter', geocoder._sources[0]._postprocessors[0].__class__.__name__, 'EsriEU geocoder incorrectly processed defaults')
class AddressConverter(object): def __init__(self, curs): self.curs = curs self.geolocator = GoogleV3() self.nominatim = nominatim.Geocoder() self.geocoder = Geocoder() self.re_numbers = re.compile("^\d+$") def reject_all_numbers(self, address): if self.re_numbers.search(address): logger.info("Rejected by reject_all_numbers") return True return False def reject_short_words(self, address, length=4): if len(address) <= length: logger.info("Rejected by reject_short_words") return True return False def reject_large_main_places(self, address, threshold=15000): if address in main_places: population = main_places[address]["Population"] if int(population) >= threshold: return True return False def reject_resolution_to_main_place(self, address, threshold=15000): parts = address.split(",") first = parts[0].strip().lower() return self.reject_large_main_places(first, threshold) def reject_partial_match(self, result): if "partial_match" in result and result["partial_match"]: return True return False def resolve_address_google(self, address, **kwargs): try: encoded_address = encode(address) address = urllib.quote(encoded_address) url = "https://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false®ion=za&key=%s" % (address, configuration["environment"]["google_key"]) response = urllib2.urlopen(url) js = response.read() try: js = json.loads(js) except ValueError: logger.exception("Error trying to resolve %s" % address) return None results = [] if "results" in js and len(js["results"]) > 0: for result in js["results"]: res = self.reject_partial_match(result) if res: continue if "reject_resolution_to_main_place" in kwargs: try: res = self.reject_resolution_to_main_place(result["formatted_address"], int(kwargs["reject_resolution_to_main_place"][0])) except (ValueError, TypeError): res = self.resolution_to_main_place(result["formatted_address"]) if res: continue geom = result["geometry"]["location"] results.append({ "lat" : geom["lat"], "lng" : geom["lng"], "formatted_address" : result["formatted_address"], "source" : "Google Geocoding API", }) if len(results) == 0: return None return results except Exception: logger.exception("Error trying to resolve %s" % address) return None def resolve_address_esri(self, address): result = self.geocoder.geocode(address + ", South Africa") if not result: return None if not "candidates" in result: return None if len(result["candidates"]) == 0: return None candidate = result["candidates"][0] address = candidate.match_addr latitude = candidate.y longitude = candidate.x return address, latitude, longitude def resolve_address_nominatim(self, address, **kwargs): encoded_address = encode(address) results = self.nominatim.geocode(encoded_address) return [ { "lat" : r["lat"], "lng" : r["lon"], "formatted_address" : r["display_name"], "source" : "Nominatim", } for r in results ] def convert_address(self, address, **kwargs): address = address.strip() if address == "": return None if "reject_numbers" in kwargs and self.reject_all_numbers(address): return None if "reject_short_words" in kwargs: try: val = self.reject_short_words(address, int(kwargs["reject_short_words"][0])) except (TypeError, ValueError): val = self.reject_short_words(address) if val: return None if "reject_large_main_places" in kwargs: try: val = self.reject_large_main_places(address, int(kwargs["reject_large_main_places"][0])) except (TypeError, ValueError): val = self.reject_large_main_places(address) if val: return None if not "south africa" in address.lower(): address = address + ", South Africa" if "enable_nominatim" in kwargs: results = self.resolve_address_nominatim(address, **kwargs) if len(results) > 0: return results return self.resolve_address_google(address, **kwargs) def convert_to_geography(self, sql, latitude, longitude): poi = (longitude, latitude) self.curs.execute(sql, poi) return self.curs.fetchall()
class AddressConverter(object): def __init__(self, curs): self.curs = curs self.geolocator = GoogleV3() self.nominatim = nominatim.Nominatim() self.geocoder = Geocoder() self.re_numbers = re.compile("^\d+$") def reject_all_numbers(self, address): if self.re_numbers.search(address): logger.info("Rejected by reject_all_numbers") return True return False def reject_short_words(self, address, length=4): if len(address) <= length: logger.info("Rejected by reject_short_words") return True return False def reject_large_main_places(self, address, threshold=15000): if address in main_places: population = main_places[address]["Population"] if int(population) >= threshold: return True return False def reject_resolution_to_main_place(self, address, threshold=15000): parts = address.split(",") first = parts[0].strip().lower() return self.reject_large_main_places(first, threshold) def reject_partial_match(self, result): if "partial_match" in result and result["partial_match"]: return True return False def resolve_address_google(self, address, **kwargs): encoded_address = encode(address) address = urllib.quote(encoded_address) url = "https://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false®ion=za&key=%s" % ( address, GOOGLE_API_KEY) response = urllib2.urlopen(url) js = response.read() try: js = json.loads(js) except ValueError as e: logger.exception("Error trying to resolve %s" % address) raise StandardError("Couldn't resolve %s: %s" % (address, e.message)) results = [] if "status" in js and js["status"] not in ("OK", "ZERO_RESULTS"): logger.error( "Error trying to resolve %s - %s (%s)" % (address, js.get("error_message", "Generic Error"), js)) raise StandardError("Couldn't resolve %s: %s" % (address, js.get("error_message"))) if "results" in js and len(js["results"]) > 0: for result in js["results"]: res = self.reject_partial_match(result) if res: continue if "reject_resolution_to_main_place" in kwargs: try: res = self.reject_resolution_to_main_place( result["formatted_address"], int(kwargs["reject_resolution_to_main_place"][0])) except (ValueError, TypeError): res = self.resolution_to_main_place( result["formatted_address"]) if res: continue geom = result["geometry"]["location"] results.append({ "lat": geom["lat"], "lng": geom["lng"], "formatted_address": result["formatted_address"], "source": "Google Geocoding API", }) if len(results) == 0: return None return results def resolve_address_esri(self, address): result = self.geocoder.geocode(address + ", South Africa") if not result: return None if not "candidates" in result: return None if len(result["candidates"]) == 0: return None candidate = result["candidates"][0] address = candidate.match_addr latitude = candidate.y longitude = candidate.x return address, latitude, longitude def resolve_address_nominatim(self, address, **kwargs): encoded_address = encode(address) results = self.nominatim.query(encoded_address) return [{ "lat": r["lat"], "lng": r["lon"], "formatted_address": r["display_name"], "source": "Nominatim", } for r in results] def convert_address(self, address, **kwargs): address = address.strip() if address == "": return None if re_is_latlng.match(address): return self.resolve_coords(address) if "reject_numbers" in kwargs and self.reject_all_numbers(address): return None if "reject_short_words" in kwargs: try: val = self.reject_short_words( address, int(kwargs["reject_short_words"][0])) except (TypeError, ValueError): val = self.reject_short_words(address) if val: return None if "reject_large_main_places" in kwargs: try: val = self.reject_large_main_places( address, int(kwargs["reject_large_main_places"][0])) except (TypeError, ValueError): val = self.reject_large_main_places(address) if val: return None if not "south africa" in address.lower(): address = address + ", South Africa" if "enable_nominatim" in kwargs: results = self.resolve_address_nominatim(address, **kwargs) if len(results) > 0: return results return self.resolve_address_google(address, **kwargs) def convert_to_geography(self, sql, latitude, longitude): poi = (longitude, latitude) self.curs.execute(sql, poi) return self.curs.fetchall() def resolve_coords(self, coords): try: lat, lng = [float(s) for s in coords.split(',', 1)] return [{ "lat": lat, "lng": lng, "formatted_address": "%s, %s" % (lat, lng), "source": "Direct", }] except ValueError: return None
"zip" : str(s.zip5), "Proper Address" : paddress, "Latitude" : '', "Longitude" : ''} students.append(student) print 'records now exist in "students" list' #Looping through list of dictionaries and adding geocoded results ## Default geocoding service is EsriWGS i = len(students) print '%s records remaining' % str(i) for s in students: i -= 1 gcode = g.geocode(s["Proper Address"]) try: s["geocode response"] = gcode["candidates"][0] s["Latitude"] = gcode["candidates"][0].y s["Longitude"] = gcode["candidates"][0].x print 'Good: %s records remaining' % str(i) except KeyboardInterrupt: break except: errors.append(s) print 'Bad: %s records remaining' % str(i) continue print 'Geocoding Complete'
class GeocoderTest(OmgeoTestCase): g = None # not set until set up def setUp(self): # Viewbox objects vb = {} vb['callowhill'] = Viewbox(-75.162628, 39.962769, -75.150963, 39.956322) # PlaceQuery objects self.pq = {} # North American Addresses: self.pq['azavea'] = PlaceQuery('340 N 12th St Ste 402 Philadelphia PA') self.pq['ambiguous_azavea'] = PlaceQuery('340 12th St Ste 402 Philadelphia PA') self.pq['wolf'] = PlaceQuery('Wolf Building') self.pq['wolf_philly'] = PlaceQuery('Wolf Building, Philadelphia PA') self.pq['wolf_bounded'] = PlaceQuery('Wolf Building', viewbox=vb['callowhill']) self.pq['alpha_774R_W_Central_Ave'] = PlaceQuery('774R W Central Ave Alpha NJ') self.pq['alpha_774_W_Central_Ave_Rear'] = PlaceQuery('774 W Central Ave Rear Alpha NJ') self.pq['pine_needles_dr'] = PlaceQuery('11761 pine needles providence forge') self.pq['pine_needles_ct'] = PlaceQuery('5328 pine needles providence forge') self.pq['pine_needles_terr'] = PlaceQuery('5359 pine needles providence forge') self.pq['moorestown_hyphenated'] = PlaceQuery('111-113 W Main St Moorestown NJ') self.pq['willow_street'] = PlaceQuery('2819F Willow Street Pike Willow Street PA') self.pq['quebec'] = PlaceQuery('756 Rue Berri Montreal QC', country='CA') self.pq['quebec_bang'] = PlaceQuery('177 Rue Commerciale, Saint-Louis-du-Ha! Ha! QC') self.pq['quebec_accent'] = PlaceQuery('527 Ch. Beauséjour, Saint-Elzéar-de-Témiscouata QC') self.pq['quebec_hyphenated'] = PlaceQuery('227-227A Rue Commerciale, Saint-Louis-du-Ha! Ha! QC') # European Addresses: self.pq['london_pieces'] = PlaceQuery(address='31 Maiden Lane', city='London', country='UK') self.pq['london_one_line'] = PlaceQuery('31 Maiden Lane, London WC2E', country='UK') self.pq['london_pieces_hyphenated'] = PlaceQuery(address='31-32 Maiden Lane', city='London', country='UK') self.pq['london_one_line_hyphenated'] = PlaceQuery('31-32 Maiden Lane London WC2E', country='UK') # Oceanic Addresses: self.pq['karori'] = PlaceQuery('102 Karori Road Karori Wellington', country='NZ') # Geocoder objects self.g = Geocoder() if ESRI_MAPS_API_KEY is not None: self.g_esri_na = Geocoder([['omgeo.services.EsriNA', {'settings':{'api_key':ESRI_MAPS_API_KEY}}]]) self.g_esri_eu = Geocoder([['omgeo.services.EsriEU', {'settings':{'api_key':ESRI_MAPS_API_KEY}}]]) else: self.g_esri_na = Geocoder([['omgeo.services.EsriNA', {}]]) self.g_esri_eu = Geocoder([['omgeo.services.EsriEU', {}]]) self.g_bing = Geocoder([['omgeo.services.Bing', {'settings':{'api_key':BING_MAPS_API_KEY}}]]) self.g_nom = Geocoder([['omgeo.services.Nominatim',{}]]) self.g_dc = Geocoder([['omgeo.services.CitizenAtlas', {}]]) def tearDown(self): pass def test_geocode_azavea(self): candidates = self.g.geocode(self.pq['azavea']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') self.assertEqual(len(candidates) > 1, False, 'More than one candidate returned.') def test_geocode_esri_na_us(self): candidates = self.g_esri_na.geocode(self.pq['alpha_774_W_Central_Ave_Rear']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_esri_na_nz(self): candidates = self.g_esri_na.geocode(self.pq['karori']) self.assertEqual(len(candidates) > 0, False, 'Found New Zealand address when this should only be using the North American ESRI geocoder.') def test_nom_bounded(self): candidates = self.g.geocode(PlaceQuery('8 Kirkbride Rd')) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') @unittest.skipIf(BING_MAPS_API_KEY is None, 'Enter a Bing Maps API key to run the bing tests') def test_geocode_bing(self): candidates = self.g_bing.geocode(self.pq['azavea']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def test_geocode_dc_address(self): candidates = self.g_dc.geocode(PlaceQuery('1600 pennsylvania')) self.assertTrue(len(candidates) > 0, 'No candidates returned.') self.assertTrue(candidates[0].locator == 'DC Address', 'Expected 1600 pennsylvania to be an address match') def test_geocode_dc_intersection(self): candidates = self.g_dc.geocode(PlaceQuery('h and 15th')) self.assertTrue(len(candidates) > 0, 'No candidates returned.') self.assertTrue(candidates[0].locator == 'DC Intersection', 'h and 15th to be an intersection match') def test_geocode_dupepicker(self): candidates = self.g.geocode(self.pq['ambiguous_azavea']) self.assertEqual(len(candidates) > 0, True, 'No candidates returned.') def _test_geocode_results_all(self, verbosity=0, geocoder=Geocoder()): queries_with_results = 0 for place in self.pq: if verbosity > 1: print '\n%s' % place if verbosity > 1: print len(place) * '-' candidates = geocoder.geocode(self.pq[place]) if len(candidates) == 0: if verbosity > 1: print 'Input: %s\n(no results)' % self.pq[place].query else: queries_with_results += 1 if verbosity > 1: print 'Input: %s' % self.pq[place].query for x in ['Output: %r (%s %s)' % (c.match_addr, c.geoservice, [c.locator, c.score, c.confidence, c.entity]) for c in candidates]: if verbosity > 1: print x self.assertEqual(len(self.pq), queries_with_results, 'Got results for %d of %d queries.' % (queries_with_results, len(self.pq))) def test_geocode_results_all(self): if BING_MAPS_API_KEY is not None: self.g.add_source(['omgeo.services.Bing', {'settings':{'api_key':BING_MAPS_API_KEY}}]) self._test_geocode_results_all(geocoder=self.g)
def getBranches(provinceId, localidadId, localidadName, provinceName, extracash=False): try: fixpath = root + paths['fix'] + '/rapipago_fix.csv' toFix = [] g = Geocoder() extracash = 'true' if extracash else 'false' res = requests.get( 'https://www.rapipago.com.ar/rapipagoWeb/index.php/resultado_sucursales?prov=' + provinceId + '&loc=' + localidadId + '&extracash=' + extracash) if res.status_code != 200: raise ValueError('Error solicitando sucursales.') soup = BeautifulSoup(res.text, 'lxml') content = soup.find_all('div', class_="w-row accordion_content") branches = [] #chequeo si el sitio posee sucursales if content != None: for c in content: try: global toPendingFixCount withoutError = True title = c.select('.text_expand_drop')[0].get_text() address = c.select('.text_expand_drop')[1].get_text() attentionHour = c.select( '.text_horarios_drop')[0].get_text().replace( 'Horarios de atención:', '').replace('\n', '').replace(' ', '') data = [localidadName, title, address, attentionHour] latlngExt = c.find_all('a') #chequeo si encontro etiquetas de latitud y longitud if len(latlngExt) > 0: latlngExt = str(latlngExt[0]) lat = findBetweenR(latlngExt, 'data-lat="', '" data-lng') lng = findBetweenR(latlngExt, 'data-lng="', '" data-marker') data.append(lat) data.append(lng) #al no encontrar o no poseer con exactitud, controla si ya fueron definidas manualmente if len(latlngExt) == 0 or (lat == '0.00000000' or lng == '0.00000000'): fixNotFound = True with open(fixpath, 'r') as f: reader = csv.reader(f) fixList = list(reader) search = [provinceName, localidadName, title, address] for i, lc in enumerate(fixList): concurrence = 0 for co in search: if co in lc: concurrence = concurrence + 1 if concurrence == len(search): data = lc[1:len(lc)] fixNotFound = False #no esta la sucursal definida manualmente e intenta geolocalizarla if fixNotFound: try: result = g.geocode(address + ', ' + localidadName + ', Argentina') lat = result['candidates'][0].x lng = result['candidates'][0].y except Exception as e: #no logro geolocalizarla y sera puesta a la lista manual withoutError = False try: mode = 'a' if os.path.exists( fixpath) else 'w' listString = ','.join(data) if len(latlngExt) == 0: listString = listString + ',0.00000000,0.00000000' with open(fixpath, mode) as _file: _file.write('\n' + provinceName + ',' + (listString)) toPendingFixCount = toPendingFixCount + 1 except Exception as e: print(e) if withoutError: branches.append(data) except Exception as e: print(e) return branches return [] except Exception as e: print(e)