def _getVenuesFromIndex(lat, lon, radius, sortOrder, maxNum):
total = 1
offset = 0
yelpVenues = []
while offset < total:
try:
locality = _singleSearchAPIQuery(lat, lon, offset, radius,
sortOrder)
total = min(locality.total, maxNum)
yelpVenues += locality.businesses
offset += 20
except KeyboardInterrupt:
raise KeyboardInterrupt()
except Exception:
log.exception("Exception searching with Yelp API")
# give up if this is the first try – we've seen
# occasional failures where the location doesn't
# have any existing support (e.g. Vanuatu)
# but also, just randon 500 errors, which we can recover from.
if total == 1 and offset == 0:
return yelpVenues
else:
offset += 20
return yelpVenues
def researchVenue(biz):
try:
yelpID = representation.createKey(biz)
cache = readCachedVenueDetails(yelpID)
venueIdentifiers = readCachedVenueIdentifiers(cache)
# This gets the identifiers from Factual. It's two HTTP requests
# per venue.
crosswalkedFoundInCached = venueIdentifiers is None
if crosswalkedFoundInCached:
venueIdentifiers, crosswalkAvailable = crosswalk.getVenueIdentifiers(
yelpID)
# This then uses the identifiers to look up (resolve) details.
# We'll fan out these as much as possible.
venueDetails = search._getVenueDetails(venueIdentifiers, cache)
# Once we've got the details, we should stash it in
# Firebase.
shouldCacheCrosswalk = (
not crosswalkedFoundInCached) or crosswalkAvailable
if shouldCacheCrosswalk:
writeVenueRecord(biz, venueDetails, venueIdentifiers)
else:
writeVenueRecord(biz, venueDetails)
return yelpID
except KeyboardInterrupt:
return False
except Exception as err:
log.exception("Error researching venue")
return False
def getVenueIdentifiers(yelpID):
yelpURL = "https://yelp.com/biz/%s" % yelpID
mapping = {
"id": yelpID,
"version": CROSSWALK_CACHE_VERSION,
"yelp": {
"url": yelpURL
}
}
try:
obj = factualClient.crosswalk().filters({"url": yelpURL}).data()
if len(obj) == 0:
return mapping, True
factualID = obj[0]["factual_id"]
mapping["factualID"] = factualID
mapping["factual"] = { "id": factualID }
idList = factualClient.crosswalk().filters({"factual_id": factualID}).data()
for idObj in idList:
namespace = idObj["namespace"]
del idObj["factual_id"]
del idObj["namespace"]
mapping[namespace] = idObj
return mapping, True
except APIException:
log.error("Factual API failed again")
except Exception:
log.exception("Factual problem " + yelpID)
return mapping, False
def updateFromGcals():
try:
loadCalendarEvents(datetime.timedelta(days=1))
scheduler.enter(calendarInfo["calRefreshSec"], 1, updateFromGcals, ())
except Exception as err:
from app.util import log
log.exception("Error running scheduled calendar fetch")
scheduler.enter(calendarInfo["calRefreshSec"], 1, updateFromGcals, ())
def researchPlace(keyID, placeProviderIDs):
try:
venueDetails = search.getVenueDetails(placeProviderIDs)
writeVenueProviderRecord(keyID, venueDetails)
return list(venueDetails.keys())
except Exception as e:
log.exception("Error researching venue {}: {}".format(keyID, e))
return []
def searchLocationWithErrorRecovery(lat, lng, radius=None):
try:
searchLocation(lat, lng, radius=radius)
except KeyboardInterrupt:
log.info("GOODBYE")
sys.exit(1)
except Exception:
from app.util import log
log.exception("Unknown exception")
def researchEvent(eventfulObj):
try:
eventObj = getEventfulEventObj(eventfulObj)
if eventObj:
writeEventRecord(eventObj)
return eventObj["id"]
except KeyboardInterrupt:
return False
except Exception as err:
log.exception("Error researching venue")
return False
def searchLocationWithErrorRecovery(lat, lng, radius=None):
try:
if radius is None:
# If the radius is unset, then we should set sensible
# defaults
radius = venueSearchRadius
searchLocation(lat, lng, radius)
except KeyboardInterrupt:
log.exception("GOODBYE")
sys.exit(1)
except Exception:
log.exception("Unknown exception")
def placeUser(latitude, longitude):
try:
latitude, longitude = float(latitude), float(longitude)
jobCount = searchLocation(latitude, longitude)
return "OK %.8f,%.8f / %d" % (latitude, longitude, jobCount)
except ValueError:
abort(404)
except KeyboardInterrupt:
log.info("GOODBYE")
sys.exit(1)
except Exception:
from app.util import log
log.exception("Unknown exception")
abort(500)
def getVenueDetailsFromProvider(namespace, idObj, cached):
venueDetails = {}
if namespace not in resolvers:
return venueDetails
if cached is not None:
venueDetails[namespace] = cached
return venueDetails
resolve = resolvers[namespace]
try:
info = resolve(idObj)
if info is not None:
venueDetails[namespace] = info
except Exception as err:
log.exception("Exeption hitting " + namespace)
return venueDetails
def writeVenueRecord(biz, details, idObj=None):
key = representation.createKey(biz)
try:
venue = representation.venueRecord(biz, **details)
geo = representation.geoRecord(biz)
record = {"details/" + key: venue, "locations/" + key: geo}
except Exception as err:
log.exception("Exception preparing to write to firebase for key " +
key)
record = {}
record["cache/" + key] = details
if idObj is not None:
# we're now going to cache the details, as well as the crosswalk
# identifiers that we used to look them up.
details["identifiers"] = idObj
db.child(venuesTable).update(record)
def getVenueDetailsFromProvider(namespace, idObj, cached):
venueDetails = {}
if namespace not in resolvers:
return venueDetails
if cached is not None:
venueDetails[namespace] = cached
return venueDetails
resolve = resolvers[namespace]
try:
# TODO: Handle parsing Crosswalk id out of url in addition to Proxwalk provider id
info = resolve(idObj)
if info is not None:
venueDetails[namespace] = info
except Exception as err:
log.exception("Exception hitting {}:{}".format(namespace, idObj))
return venueDetails
def getGcalEventObj(event):
if ("dateTime" not in event["start"]) or ("dateTime" not in event["end"]) or ("location" not in event):
return None
eventLoc = event["location"]
name, address = events.getNameAndAddress(eventLoc)
mapping = search._getAddressIdentifiers(eventLoc)
if mapping:
try:
location = mapping['location']
placeName = '%s, %s' % (mapping['name'], mapping['zipcode'])
yelpId = _guessYelpId(eventLoc, location['lat'], location['lng'])
if yelpId:
optUrl = event["description"] if "description" in event else None
eventObj = representation.eventRecord(yelpId, location['lat'], location['lng'], event['summary'], event['start']['dateTime'], event['end']['dateTime'], optUrl)
return eventObj
except Exception as err:
log.exception("getGcalEventObj")
log.info("Unable to find corresponding location for %s" % eventLoc)
def getVenueIdentifiers(yelpID):
yelpURL = "https://yelp.com/biz/%s" % yelpID
mapping = {
"id": yelpID,
"version": CROSSWALK_CACHE_VERSION,
"yelp": {
"url": yelpURL
}
}
try:
if deployment == "production/":
crosswalk = factualClient.table("crosswalk-us")
else:
crosswalk = factualClient.crosswalk()
obj = crosswalk.filters({"url": yelpURL}).data()
if len(obj) == 0:
log.debug("Crosswalk empty for Yelp -> Factual " + yelpID)
return mapping, True
factualID = obj[0]["factual_id"]
mapping["factualID"] = factualID
mapping["factual"] = {"id": factualID}
idList = crosswalk.filters({"factual_id": factualID}).data()
if len(idList) == 0:
log.warn("Crosswalk empty for Factual -> * " + yelpID + " " +
factualID)
for idObj in idList:
namespace = idObj["namespace"]
del idObj["factual_id"]
del idObj["namespace"]
mapping[namespace] = idObj
return mapping, True
except APIException:
log.error("Factual API failed again")
except Exception:
log.exception("Factual problem " + yelpID)
return mapping, False
