class GeocoderEntityProxy(AEntityProxy):
def __init__(self, source):
AEntityProxy.__init__(self, source)
self._geocoder = Geocoder()
def _processItems(self, items):
utils.log("[%s] processing %d items" % (self, utils.count(items)))
AEntityProxy._processItems(self, items)
def _transform(self, entity):
try:
# if entity is already populated with lat/lng, then return it as-is
if entity.lat is not None and entity.lng is not None:
return entity
except KeyError:
pass
try:
# entity has an address and only needs the corresponding geocoded lat/lng
address = entity.address
latLng = self._geocoder.addressToLatLng(address)
if latLng is not None:
entity.lat = latLng[0]
entity.lng = latLng[1]
else:
# we weren't able to geocode the given address to a valid lat/lng, so
# filter this entity from the proxy's output
return None
except KeyError:
# entity is not a place
pass
return entity
def __init__(self):
AExternalServiceEntitySource.__init__(self, self.NAME, self.TYPES)
self._geocoder = Geocoder()
class GoogleLocal(AExternalServiceEntitySource):
NAME = 'GoogleLocal'
TYPES = set([ 'restaurant' ])
def __init__(self):
AExternalServiceEntitySource.__init__(self, self.NAME, self.TYPES)
self._geocoder = Geocoder()
def _run(self):
pass
def getLocalSearchResults(self, query, latLng=None, params=None, transform=True):
response, url = self._getLocalSearchResponse(query, latLng, params)
if response is None:
return None
if 200 != response["responseStatus"]:
utils.log('[GoogleLocal] unexpected return status "' + \
response["responseStatus"] + ' (' + url + ')"')
return None
if not transform:
return response
output = []
try:
results = response['responseData']['results']
for result in results:
output.append(self._parseEntity(result))
except:
utils.printException()
raise
return output
def _parseEntity(self, result):
entity = Entity()
entity.subcategory = 'other'
if 'titleNoFormatting' in result:
entity.title = result['titleNoFormatting']
if 'addressLines' in result:
entity.address = string.joinfields(result['addressLines'], ', ')
entity.subtitle = entity.address
if 'lat' in result and 'lng' in result:
entity.lat = float(result['lat'])
entity.lng = float(result['lng'])
if 'region' in result:
entity.vicinity = result['region']
if 'phoneNumbers' in result:
phoneNumbers = result['phoneNumbers']
if len(phoneNumbers) > 0:
entity.phone = phoneNumbers[0]['number']
entity.googleLocal = {}
entity.titlel = entity.title.lower()
return entity
def _getLocalSearchResponse(self, query, latLng=None, optionalParams=None):
params = {
'v' : '1.0',
'q' : query,
'rsz' : 8,
'mrt' : 'localonly',
'key' : 'ABQIAAAAwHbLTrUsG9ibtIA3QrujsRRB6mhcr2m5Q6fm3mUuDbLfyI5H4xTNn-E18G_3Zu-sDQ3-BTh9hK2BeQ',
}
if latLng is not None:
params['sll'] = self._geocoder.getEncodedLatLng(latLng)
self._handleParams(params, optionalParams)
url = "http://ajax.googleapis.com/ajax/services/search/local?%s" % urllib.urlencode(params)
utils.log('[GoogleLocal] ' + url)
try:
# GET the data and parse the response as json
request = urllib2.Request(url, None, {'Referer' : 'http://www.stamped.com' })
return json.loads(utils.getFile(url, request)), url
except:
utils.log('[GoogleLocal] unexpected error searching "' + url + '"')
utils.printException()
return None, url
return None, url
def _handleParams(self, params, optionalParams):
if optionalParams is not None:
for key in optionalParams:
params[key] = optionalParams[key]
for k in params:
v = params[k]
if isinstance(v, unicode):
params[k] = v.encode("ascii", "xmlcharrefreplace")
def getSubcategoryFromTypes(self, types):
for t in types:
if t != "establishment":
try:
return self._googleTypeToSubcategoryMap[t]
except KeyError:
return t
return "other"
def __init__(self):
AKeyBasedAPI.__init__(self, self.API_KEYS)
self._geocoder = Geocoder()
class GooglePlaces(AKeyBasedAPI):
BASE_URL = 'https://maps.googleapis.com/maps/api/place'
FORMAT = 'json'
DEFAULT_RADIUS = 500 # meters
NAME = 'GooglePlaces'
TYPES = set([ 'restaurant' ])
API_KEYS = get_api_key('googleplaces', 'api_keys')
_googleTypeToSubcategoryMap = {
"food" : "restaurant",
"restaurant" : "restaurant",
"grocery_or_supermarket" : "market",
}
google_subcategory_whitelist = set([
"amusement_park",
"aquarium",
"art_gallery",
"bakery",
"bar",
"beauty_salon",
"book_store",
"bowling_alley",
"cafe",
"campground",
"casino",
"clothing_store",
"department_store",
"florist",
"food",
"grocery_or_supermarket",
"market",
"gym",
"home_goods_store",
"jewelry_store",
"library",
"liquor_store",
"lodging",
"movie_theater",
"museum",
"night_club",
"park",
"restaurant",
"school",
"shoe_store",
"shopping_mall",
"spa",
"stadium",
"store",
"university",
"zoo",
"other",
"establishment",
])
def __init__(self):
AKeyBasedAPI.__init__(self, self.API_KEYS)
self._geocoder = Geocoder()
def _run(self):
pass
def getPlaceDetails(self, reference, params=None, priority='low'):
(offset, count) = self._initAPIKeyIndices()
while True:
# try a different API key for each attempt
apiKey = self._getAPIKey(offset, count)
if apiKey is None:
return None
response = self.getPlaceDetailsResponse(reference, apiKey, params, priority)
if response is None:
return None
# ensure that we received a valid response
if response['status'] != 'OK':
#utils.log('[GooglePlaces] error searching "' + reference + '"\n' +
# 'ErrorCode: ' + response['status'] + '\n')
if response['status'] == 'OVER_QUERY_LIMIT':
count += 1
continue
else:
return None
return response['result']
def getSearchResultsByAddress(self, address, params=None, priority='low'):
latLng = self.addressToLatLng(address)
if latLng is None:
# geocoding translation from address to lat/lng failed, so we will
# be unable to cross-reference this address with google places.
return None
return self.getSearchResultsByLatLng(latLng, params, priority)
def getEntityResultsByLatLng(self, latLng, params=None, detailed=False, priority='low'):
results = self.getSearchResultsByLatLng(latLng, params, priority)
output = []
if results is None:
return None
for result in results:
entity = self.getEntityFromResult(result, detailed, priority)
if entity is not None:
output.append(entity)
return output
def getEntityFromResult(self, result, detailed=False, priority='low'):
if result is None:
return None
entity = self.parseEntity(result)
if entity is None:
return None
if detailed:
# fetch a google places details request to fill in any missing data
details = self.getPlaceDetails(result['reference'], priority=priority)
self.parseEntityDetail(details, entity)
return entity
def parseEntity(self, result, valid=False):
subcategory = self.getSubcategoryFromTypes(result['types'])
if not valid and subcategory not in self.google_subcategory_whitelist:
return None
entity = PlaceEntity()
entity.title = result['name']
coordinates = Coordinates()
coordinates.lat = result['geometry']['location']['lat']
coordinates.lng = result['geometry']['location']['lng']
entity.coordinates = coordinates
entity.sources.googleplaces_id = result['id']
entity.sources.googleplaces_reference = result['reference']
# TODO: TYPE
types = set(entity.types)
types.add(subcategory)
entity.types = list(types)
"""
if result['icon'] != u'http://maps.gstatic.com/mapfiles/place_api/icons/restaurant-71.png' and \
result['icon'] != u'http://maps.gstatic.com/mapfiles/place_api/icons/generic_business-71.png':
entity.image = result['icon']
"""
if 'vicinity' in result:
entity.neighborhood = result['vicinity']
return entity
def parseEntityDetail(self, result, entity):
if result is not None:
if 'formatted_phone_number' in result:
entity.phone = result['formatted_phone_number']
if 'formatted_address' in result:
entity.address = result['formatted_address']
if 'address_components' in result:
entity.address_components = result['address_components']
return entity
# note: these decorators add tiered caching to this function, such that
# results will be cached locally with a very small LRU cache of 64 items
# and also cached remotely via memcached with a TTL of 7 days
@countedFn(name='GooglePlaces (before caching)')
@lru_cache(maxsize=64)
@cachedFn()
@countedFn(name='GooglePlaces (after caching)')
def getSearchResultsByLatLng(self, latLng, params=None, priority="low"):
(offset, count) = self._initAPIKeyIndices()
while True:
# try a different API key for each attempt
apiKey = self._getAPIKey(offset, count)
if apiKey is None:
return None
response = self._getSearchResponseByLatLng(latLng, apiKey, params, priority)
if response is None:
return None
#utils.log(json.dumps(response, sort_keys=True, indent=2))
# ensure that we received a valid response
if response['status'] != 'OK' or len(response['results']) <= 0:
utils.log('[GooglePlaces] error searching "' + str(latLng) + '"\n' +
'ErrorCode: ' + str(response['status']) + '\n')
import pprint
pprint.pprint(response)
if response['status'] == 'OVER_QUERY_LIMIT':
count += 1
continue
else:
return None
return response['results']
# note: these decorators add tiered caching to this function, such that
# results will be cached locally with a very small LRU cache of 64 items
# and also cached remotely via memcached with a TTL of 7 days
@countedFn(name='GooglePlaces (before caching)')
@lru_cache(maxsize=64)
@cachedFn()
@countedFn(name='GooglePlaces (after caching)')
def getAutocompleteResults(self, latLng, query, params=None, priority='low'):
(offset, count) = self._initAPIKeyIndices()
while True:
# try a different API key for each attempt
apiKey = self._getAPIKey(offset, count)
if apiKey is None:
return None
print apiKey
response = self._getAutocompleteResponse(latLng, query, apiKey, params, priority)
if response is None:
return None
#utils.log(json.dumps(response, sort_keys=True, indent=2))
# ensure that we received a valid response
if response['status'] != 'OK' or len(response['predictions']) <= 0:
utils.log('[GooglePlaces] error autocompleting "%s" (ErrorCode: %s)\n' %
(query, response['status']))
if response['status'] == 'OVER_QUERY_LIMIT':
count += 1
continue
else:
return None
return response['predictions']
def getEntityAutocompleteResults(self, query, latLng=None, params=None, priority='low'):
results = self.getAutocompleteResults(latLng, query, params, priority)
output = []
if results is not None:
for result in results:
entity = PlaceEntity()
entity.sources.googleplaces_id = result['id']
entity.sources.googleplaces_reference = result['reference']
try:
terms = result['terms']
entity.title = terms[0]['value']
if len(terms) > 1:
if terms[-1]['value'] == "United States":
terms = terms[:-1]
entity.formatted_address = string.joinfields((v['value'] for v in terms[1:]), ', ')
except:
entity.title = result['description']
output.append(entity)
return output
def _getSearchResponseByLatLng(self, latLng, apiKey, optionalParams=None, priority="low"):
params = {
'location' : self._geocoder.getEncodedLatLng(latLng),
'radius' : self.DEFAULT_RADIUS,
'sensor' : 'false',
'key' : apiKey,
}
self._handleParams(params, optionalParams)
# example URL:
# https://maps.googleapis.com/maps/api/place/search/json?location=-33.8670522,151.1957362&radius=500&types=food&name=harbour&sensor=false&key=AIzaSyAxgU3LPU-m5PI7Jh7YTYYKAz6lV6bz2ok
url = self._getAPIURL('search')
utils.log('[GooglePlaces] ' + url)
try:
# GET the data and parse the response as json
response, content = service_request('googleplaces', 'GET', url, query_params=params, priority=priority)
return json.loads(content)
except:
utils.log('[GooglePlaces] unexpected error searching "' + url + '"')
raise
# note: these decorators add tiered caching to this function, such that
# results will be cached locally with a very small LRU cache of 64 items
# and also cached remotely via memcached with a TTL of 7 days
@countedFn(name='GooglePlaces (before caching)')
@lru_cache(maxsize=64)
@cachedFn()
@countedFn(name='GooglePlaces (after caching)')
def getPlaceDetailsResponse(self, reference, apiKey, optionalParams=None, priority='low'):
params = {
'reference' : reference,
'sensor' : 'false',
'key' : apiKey,
}
self._handleParams(params, optionalParams)
# example URL:
# https://maps.googleapis.com/maps/api/place/details/json?reference=...&sensor=false&key=AIzaSyAxgU3LPU-m5PI7Jh7YTYYKAz6lV6bz2ok
url = self._getAPIURL('details')
utils.log('[GooglePlaces] ' + url)
try:
response, content = service_request('googleplaces', 'GET', url, query_params=params, priority=priority)
return json.loads(content)
except:
utils.log('[GooglePlaces] unexpected error searching "' + url + '"')
raise
def addPlaceReport(self, entity, priority='low'):
params = {
'sensor' : 'false',
'key' : self._getAPIKey(0, 0),
}
post_params = {
'location' : {
'lat' : entity.lat,
'lng' : entity.lng,
},
'name' : entity.title,
'accuracy' : 50,
'types' : [ ],
'language' : 'en-US',
}
url = self._getAPIURL('add')
#utils.log(url)
response, content = service_request('googleplaces', 'POST', url, query_params=params, body=post_params, priority='low')
return content
def _getAutocompleteResponse(self, latLng, query, apiKey, optionalParams=None, priority='low'):
params = {
'input' : query,
'sensor' : 'false',
'types' : 'establishment',
'key' : apiKey,
}
if latLng is not None:
params['location'] = self._geocoder.getEncodedLatLng(latLng)
self._handleParams(params, optionalParams)
# example URL:
# https://maps.googleapis.com/maps/api/place/autocomplete/json?input=test&sensor=false&key=AIzaSyAxgU3LPU-m5PI7Jh7YTYYKAz6lV6bz2ok
url = self._getAPIURL('autocomplete')
utils.log('[GooglePlaces] ' + url)
response, content = service_request('googleplaces', 'GET', url, query_params=params, priority=priority)
return json.loads(content)
def _handleParams(self, params, optionalParams):
if optionalParams is not None:
for key in optionalParams:
params[key] = optionalParams[key]
for k in params:
v = params[k]
if isinstance(v, unicode):
params[k] = v.encode('utf-8')
def addressToLatLng(self, address):
latLng = self._geocoder.addressToLatLng(address)
if latLng is None or latLng[0] is None or latLng[1] is None:
# geocoding translation from address to lat/lng failed
return None
return latLng
def _getAPIURL(self, method):
return "%s/%s/%s" % (self.BASE_URL, method, self.FORMAT)
def getSubcategoryFromTypes(self, types):
establishment = False
for t in types:
if t == "establishment":
establishment = True
else:
try:
return self._googleTypeToSubcategoryMap[t]
except KeyError:
return t
if establishment:
return "establishment"
else:
return "other"
def __init__(self, source):
AEntityProxy.__init__(self, source)
self._geocoder = Geocoder()
