def _getVenuesFromIndex(lat, lon, radius, sortOrder, maxNum): total = 1 offset = 0 yelpVenues = [] while offset < total: try: locality = _singleSearchAPIQuery(lat, lon, offset, radius, sortOrder) total = min(locality.total, maxNum) yelpVenues += locality.businesses offset += 20 except KeyboardInterrupt: raise KeyboardInterrupt() except Exception: log.exception("Exception searching with Yelp API") # give up if this is the first try – we've seen # occasional failures where the location doesn't # have any existing support (e.g. Vanuatu) # but also, just randon 500 errors, which we can recover from. if total == 1 and offset == 0: return yelpVenues else: offset += 20 return yelpVenues
def researchVenue(biz): try: yelpID = representation.createKey(biz) cache = readCachedVenueDetails(yelpID) venueIdentifiers = readCachedVenueIdentifiers(cache) # This gets the identifiers from Factual. It's two HTTP requests # per venue. crosswalkedFoundInCached = venueIdentifiers is None if crosswalkedFoundInCached: venueIdentifiers, crosswalkAvailable = crosswalk.getVenueIdentifiers( yelpID) # This then uses the identifiers to look up (resolve) details. # We'll fan out these as much as possible. venueDetails = search._getVenueDetails(venueIdentifiers, cache) # Once we've got the details, we should stash it in # Firebase. shouldCacheCrosswalk = ( not crosswalkedFoundInCached) or crosswalkAvailable if shouldCacheCrosswalk: writeVenueRecord(biz, venueDetails, venueIdentifiers) else: writeVenueRecord(biz, venueDetails) return yelpID except KeyboardInterrupt: return False except Exception as err: log.exception("Error researching venue") return False
def getVenueIdentifiers(yelpID): yelpURL = "https://yelp.com/biz/%s" % yelpID mapping = { "id": yelpID, "version": CROSSWALK_CACHE_VERSION, "yelp": { "url": yelpURL } } try: obj = factualClient.crosswalk().filters({"url": yelpURL}).data() if len(obj) == 0: return mapping, True factualID = obj[0]["factual_id"] mapping["factualID"] = factualID mapping["factual"] = { "id": factualID } idList = factualClient.crosswalk().filters({"factual_id": factualID}).data() for idObj in idList: namespace = idObj["namespace"] del idObj["factual_id"] del idObj["namespace"] mapping[namespace] = idObj return mapping, True except APIException: log.error("Factual API failed again") except Exception: log.exception("Factual problem " + yelpID) return mapping, False
def updateFromGcals(): try: loadCalendarEvents(datetime.timedelta(days=1)) scheduler.enter(calendarInfo["calRefreshSec"], 1, updateFromGcals, ()) except Exception as err: from app.util import log log.exception("Error running scheduled calendar fetch") scheduler.enter(calendarInfo["calRefreshSec"], 1, updateFromGcals, ())
def researchPlace(keyID, placeProviderIDs): try: venueDetails = search.getVenueDetails(placeProviderIDs) writeVenueProviderRecord(keyID, venueDetails) return list(venueDetails.keys()) except Exception as e: log.exception("Error researching venue {}: {}".format(keyID, e)) return []
def searchLocationWithErrorRecovery(lat, lng, radius=None): try: searchLocation(lat, lng, radius=radius) except KeyboardInterrupt: log.info("GOODBYE") sys.exit(1) except Exception: from app.util import log log.exception("Unknown exception")
def researchEvent(eventfulObj): try: eventObj = getEventfulEventObj(eventfulObj) if eventObj: writeEventRecord(eventObj) return eventObj["id"] except KeyboardInterrupt: return False except Exception as err: log.exception("Error researching venue") return False
def searchLocationWithErrorRecovery(lat, lng, radius=None): try: if radius is None: # If the radius is unset, then we should set sensible # defaults radius = venueSearchRadius searchLocation(lat, lng, radius) except KeyboardInterrupt: log.exception("GOODBYE") sys.exit(1) except Exception: log.exception("Unknown exception")
def placeUser(latitude, longitude): try: latitude, longitude = float(latitude), float(longitude) jobCount = searchLocation(latitude, longitude) return "OK %.8f,%.8f / %d" % (latitude, longitude, jobCount) except ValueError: abort(404) except KeyboardInterrupt: log.info("GOODBYE") sys.exit(1) except Exception: from app.util import log log.exception("Unknown exception") abort(500)
def getVenueDetailsFromProvider(namespace, idObj, cached): venueDetails = {} if namespace not in resolvers: return venueDetails if cached is not None: venueDetails[namespace] = cached return venueDetails resolve = resolvers[namespace] try: info = resolve(idObj) if info is not None: venueDetails[namespace] = info except Exception as err: log.exception("Exeption hitting " + namespace) return venueDetails
def writeVenueRecord(biz, details, idObj=None): key = representation.createKey(biz) try: venue = representation.venueRecord(biz, **details) geo = representation.geoRecord(biz) record = {"details/" + key: venue, "locations/" + key: geo} except Exception as err: log.exception("Exception preparing to write to firebase for key " + key) record = {} record["cache/" + key] = details if idObj is not None: # we're now going to cache the details, as well as the crosswalk # identifiers that we used to look them up. details["identifiers"] = idObj db.child(venuesTable).update(record)
def getVenueDetailsFromProvider(namespace, idObj, cached): venueDetails = {} if namespace not in resolvers: return venueDetails if cached is not None: venueDetails[namespace] = cached return venueDetails resolve = resolvers[namespace] try: # TODO: Handle parsing Crosswalk id out of url in addition to Proxwalk provider id info = resolve(idObj) if info is not None: venueDetails[namespace] = info except Exception as err: log.exception("Exception hitting {}:{}".format(namespace, idObj)) return venueDetails
def getGcalEventObj(event): if ("dateTime" not in event["start"]) or ("dateTime" not in event["end"]) or ("location" not in event): return None eventLoc = event["location"] name, address = events.getNameAndAddress(eventLoc) mapping = search._getAddressIdentifiers(eventLoc) if mapping: try: location = mapping['location'] placeName = '%s, %s' % (mapping['name'], mapping['zipcode']) yelpId = _guessYelpId(eventLoc, location['lat'], location['lng']) if yelpId: optUrl = event["description"] if "description" in event else None eventObj = representation.eventRecord(yelpId, location['lat'], location['lng'], event['summary'], event['start']['dateTime'], event['end']['dateTime'], optUrl) return eventObj except Exception as err: log.exception("getGcalEventObj") log.info("Unable to find corresponding location for %s" % eventLoc)
def getVenueIdentifiers(yelpID): yelpURL = "https://yelp.com/biz/%s" % yelpID mapping = { "id": yelpID, "version": CROSSWALK_CACHE_VERSION, "yelp": { "url": yelpURL } } try: if deployment == "production/": crosswalk = factualClient.table("crosswalk-us") else: crosswalk = factualClient.crosswalk() obj = crosswalk.filters({"url": yelpURL}).data() if len(obj) == 0: log.debug("Crosswalk empty for Yelp -> Factual " + yelpID) return mapping, True factualID = obj[0]["factual_id"] mapping["factualID"] = factualID mapping["factual"] = {"id": factualID} idList = crosswalk.filters({"factual_id": factualID}).data() if len(idList) == 0: log.warn("Crosswalk empty for Factual -> * " + yelpID + " " + factualID) for idObj in idList: namespace = idObj["namespace"] del idObj["factual_id"] del idObj["namespace"] mapping[namespace] = idObj return mapping, True except APIException: log.error("Factual API failed again") except Exception: log.exception("Factual problem " + yelpID) return mapping, False