class GeocoderEntityProxy(AEntityProxy): def __init__(self, source): AEntityProxy.__init__(self, source) self._geocoder = Geocoder() def _processItems(self, items): utils.log("[%s] processing %d items" % (self, utils.count(items))) AEntityProxy._processItems(self, items) def _transform(self, entity): try: # if entity is already populated with lat/lng, then return it as-is if entity.lat is not None and entity.lng is not None: return entity except KeyError: pass try: # entity has an address and only needs the corresponding geocoded lat/lng address = entity.address latLng = self._geocoder.addressToLatLng(address) if latLng is not None: entity.lat = latLng[0] entity.lng = latLng[1] else: # we weren't able to geocode the given address to a valid lat/lng, so # filter this entity from the proxy's output return None except KeyError: # entity is not a place pass return entity
class GooglePlaces(AKeyBasedAPI): BASE_URL = 'https://maps.googleapis.com/maps/api/place' FORMAT = 'json' DEFAULT_RADIUS = 500 # meters NAME = 'GooglePlaces' TYPES = set([ 'restaurant' ]) API_KEYS = get_api_key('googleplaces', 'api_keys') _googleTypeToSubcategoryMap = { "food" : "restaurant", "restaurant" : "restaurant", "grocery_or_supermarket" : "market", } google_subcategory_whitelist = set([ "amusement_park", "aquarium", "art_gallery", "bakery", "bar", "beauty_salon", "book_store", "bowling_alley", "cafe", "campground", "casino", "clothing_store", "department_store", "florist", "food", "grocery_or_supermarket", "market", "gym", "home_goods_store", "jewelry_store", "library", "liquor_store", "lodging", "movie_theater", "museum", "night_club", "park", "restaurant", "school", "shoe_store", "shopping_mall", "spa", "stadium", "store", "university", "zoo", "other", "establishment", ]) def __init__(self): AKeyBasedAPI.__init__(self, self.API_KEYS) self._geocoder = Geocoder() def _run(self): pass def getPlaceDetails(self, reference, params=None, priority='low'): (offset, count) = self._initAPIKeyIndices() while True: # try a different API key for each attempt apiKey = self._getAPIKey(offset, count) if apiKey is None: return None response = self.getPlaceDetailsResponse(reference, apiKey, params, priority) if response is None: return None # ensure that we received a valid response if response['status'] != 'OK': #utils.log('[GooglePlaces] error searching "' + reference + '"\n' + # 'ErrorCode: ' + response['status'] + '\n') if response['status'] == 'OVER_QUERY_LIMIT': count += 1 continue else: return None return response['result'] def getSearchResultsByAddress(self, address, params=None, priority='low'): latLng = self.addressToLatLng(address) if latLng is None: # geocoding translation from address to lat/lng failed, so we will # be unable to cross-reference this address with google places. return None return self.getSearchResultsByLatLng(latLng, params, priority) def getEntityResultsByLatLng(self, latLng, params=None, detailed=False, priority='low'): results = self.getSearchResultsByLatLng(latLng, params, priority) output = [] if results is None: return None for result in results: entity = self.getEntityFromResult(result, detailed, priority) if entity is not None: output.append(entity) return output def getEntityFromResult(self, result, detailed=False, priority='low'): if result is None: return None entity = self.parseEntity(result) if entity is None: return None if detailed: # fetch a google places details request to fill in any missing data details = self.getPlaceDetails(result['reference'], priority=priority) self.parseEntityDetail(details, entity) return entity def parseEntity(self, result, valid=False): subcategory = self.getSubcategoryFromTypes(result['types']) if not valid and subcategory not in self.google_subcategory_whitelist: return None entity = PlaceEntity() entity.title = result['name'] coordinates = Coordinates() coordinates.lat = result['geometry']['location']['lat'] coordinates.lng = result['geometry']['location']['lng'] entity.coordinates = coordinates entity.sources.googleplaces_id = result['id'] entity.sources.googleplaces_reference = result['reference'] # TODO: TYPE types = set(entity.types) types.add(subcategory) entity.types = list(types) """ if result['icon'] != u'http://maps.gstatic.com/mapfiles/place_api/icons/restaurant-71.png' and \ result['icon'] != u'http://maps.gstatic.com/mapfiles/place_api/icons/generic_business-71.png': entity.image = result['icon'] """ if 'vicinity' in result: entity.neighborhood = result['vicinity'] return entity def parseEntityDetail(self, result, entity): if result is not None: if 'formatted_phone_number' in result: entity.phone = result['formatted_phone_number'] if 'formatted_address' in result: entity.address = result['formatted_address'] if 'address_components' in result: entity.address_components = result['address_components'] return entity # note: these decorators add tiered caching to this function, such that # results will be cached locally with a very small LRU cache of 64 items # and also cached remotely via memcached with a TTL of 7 days @countedFn(name='GooglePlaces (before caching)') @lru_cache(maxsize=64) @cachedFn() @countedFn(name='GooglePlaces (after caching)') def getSearchResultsByLatLng(self, latLng, params=None, priority="low"): (offset, count) = self._initAPIKeyIndices() while True: # try a different API key for each attempt apiKey = self._getAPIKey(offset, count) if apiKey is None: return None response = self._getSearchResponseByLatLng(latLng, apiKey, params, priority) if response is None: return None #utils.log(json.dumps(response, sort_keys=True, indent=2)) # ensure that we received a valid response if response['status'] != 'OK' or len(response['results']) <= 0: utils.log('[GooglePlaces] error searching "' + str(latLng) + '"\n' + 'ErrorCode: ' + str(response['status']) + '\n') import pprint pprint.pprint(response) if response['status'] == 'OVER_QUERY_LIMIT': count += 1 continue else: return None return response['results'] # note: these decorators add tiered caching to this function, such that # results will be cached locally with a very small LRU cache of 64 items # and also cached remotely via memcached with a TTL of 7 days @countedFn(name='GooglePlaces (before caching)') @lru_cache(maxsize=64) @cachedFn() @countedFn(name='GooglePlaces (after caching)') def getAutocompleteResults(self, latLng, query, params=None, priority='low'): (offset, count) = self._initAPIKeyIndices() while True: # try a different API key for each attempt apiKey = self._getAPIKey(offset, count) if apiKey is None: return None print apiKey response = self._getAutocompleteResponse(latLng, query, apiKey, params, priority) if response is None: return None #utils.log(json.dumps(response, sort_keys=True, indent=2)) # ensure that we received a valid response if response['status'] != 'OK' or len(response['predictions']) <= 0: utils.log('[GooglePlaces] error autocompleting "%s" (ErrorCode: %s)\n' % (query, response['status'])) if response['status'] == 'OVER_QUERY_LIMIT': count += 1 continue else: return None return response['predictions'] def getEntityAutocompleteResults(self, query, latLng=None, params=None, priority='low'): results = self.getAutocompleteResults(latLng, query, params, priority) output = [] if results is not None: for result in results: entity = PlaceEntity() entity.sources.googleplaces_id = result['id'] entity.sources.googleplaces_reference = result['reference'] try: terms = result['terms'] entity.title = terms[0]['value'] if len(terms) > 1: if terms[-1]['value'] == "United States": terms = terms[:-1] entity.formatted_address = string.joinfields((v['value'] for v in terms[1:]), ', ') except: entity.title = result['description'] output.append(entity) return output def _getSearchResponseByLatLng(self, latLng, apiKey, optionalParams=None, priority="low"): params = { 'location' : self._geocoder.getEncodedLatLng(latLng), 'radius' : self.DEFAULT_RADIUS, 'sensor' : 'false', 'key' : apiKey, } self._handleParams(params, optionalParams) # example URL: # https://maps.googleapis.com/maps/api/place/search/json?location=-33.8670522,151.1957362&radius=500&types=food&name=harbour&sensor=false&key=AIzaSyAxgU3LPU-m5PI7Jh7YTYYKAz6lV6bz2ok url = self._getAPIURL('search') utils.log('[GooglePlaces] ' + url) try: # GET the data and parse the response as json response, content = service_request('googleplaces', 'GET', url, query_params=params, priority=priority) return json.loads(content) except: utils.log('[GooglePlaces] unexpected error searching "' + url + '"') raise # note: these decorators add tiered caching to this function, such that # results will be cached locally with a very small LRU cache of 64 items # and also cached remotely via memcached with a TTL of 7 days @countedFn(name='GooglePlaces (before caching)') @lru_cache(maxsize=64) @cachedFn() @countedFn(name='GooglePlaces (after caching)') def getPlaceDetailsResponse(self, reference, apiKey, optionalParams=None, priority='low'): params = { 'reference' : reference, 'sensor' : 'false', 'key' : apiKey, } self._handleParams(params, optionalParams) # example URL: # https://maps.googleapis.com/maps/api/place/details/json?reference=...&sensor=false&key=AIzaSyAxgU3LPU-m5PI7Jh7YTYYKAz6lV6bz2ok url = self._getAPIURL('details') utils.log('[GooglePlaces] ' + url) try: response, content = service_request('googleplaces', 'GET', url, query_params=params, priority=priority) return json.loads(content) except: utils.log('[GooglePlaces] unexpected error searching "' + url + '"') raise def addPlaceReport(self, entity, priority='low'): params = { 'sensor' : 'false', 'key' : self._getAPIKey(0, 0), } post_params = { 'location' : { 'lat' : entity.lat, 'lng' : entity.lng, }, 'name' : entity.title, 'accuracy' : 50, 'types' : [ ], 'language' : 'en-US', } url = self._getAPIURL('add') #utils.log(url) response, content = service_request('googleplaces', 'POST', url, query_params=params, body=post_params, priority='low') return content def _getAutocompleteResponse(self, latLng, query, apiKey, optionalParams=None, priority='low'): params = { 'input' : query, 'sensor' : 'false', 'types' : 'establishment', 'key' : apiKey, } if latLng is not None: params['location'] = self._geocoder.getEncodedLatLng(latLng) self._handleParams(params, optionalParams) # example URL: # https://maps.googleapis.com/maps/api/place/autocomplete/json?input=test&sensor=false&key=AIzaSyAxgU3LPU-m5PI7Jh7YTYYKAz6lV6bz2ok url = self._getAPIURL('autocomplete') utils.log('[GooglePlaces] ' + url) response, content = service_request('googleplaces', 'GET', url, query_params=params, priority=priority) return json.loads(content) def _handleParams(self, params, optionalParams): if optionalParams is not None: for key in optionalParams: params[key] = optionalParams[key] for k in params: v = params[k] if isinstance(v, unicode): params[k] = v.encode('utf-8') def addressToLatLng(self, address): latLng = self._geocoder.addressToLatLng(address) if latLng is None or latLng[0] is None or latLng[1] is None: # geocoding translation from address to lat/lng failed return None return latLng def _getAPIURL(self, method): return "%s/%s/%s" % (self.BASE_URL, method, self.FORMAT) def getSubcategoryFromTypes(self, types): establishment = False for t in types: if t == "establishment": establishment = True else: try: return self._googleTypeToSubcategoryMap[t] except KeyError: return t if establishment: return "establishment" else: return "other"