def fetchDetails(placeID):
placeStatus = statusTable[placeID]
# Get a list of (src, version) pairs that could be updated, skip searched places
# TODO: Gracefully handle if TripAdvisor-mapper runs out of API calls (25k)
newProviders = [
src for src in config if src not in placeStatus or (
config[src] > placeStatus[src]
and placeStatus[src] != Status.NOT_FOUND.value)
]
if not newProviders:
# log.info("No new sources for {}".format(placeID))
return
try:
placeProviderIDs = proxwalk.getAndCacheProviderIDs(
placeID, newProviders, placeStatus["identifiers"])
except Exception as e:
log.error("Error fetching or caching provider id: {}".format(e))
return
updatedProviders = request_handler.researchPlace(
placeID, placeProviderIDs)
# Write updated sources to /status
newStatus = makeNewStatusTable(config, updatedProviders,
placeProviderIDs, newProviders)
try:
placeStatus.update(newStatus)
db().child(venuesTable, "status", placeID).update(placeStatus)
except Exception as e:
log.error("Error accessing status table for {}: {}".format(
placeID, e))
log.info("{} done: {}".format(placeID, str(updatedProviders)))
def getVenueIdentifiers(yelpID):
yelpURL = "https://yelp.com/biz/%s" % yelpID
mapping = {
"id": yelpID,
"version": CROSSWALK_CACHE_VERSION,
"yelp": {
"url": yelpURL
}
}
try:
obj = factualClient.crosswalk().filters({"url": yelpURL}).data()
if len(obj) == 0:
return mapping, True
factualID = obj[0]["factual_id"]
mapping["factualID"] = factualID
mapping["factual"] = { "id": factualID }
idList = factualClient.crosswalk().filters({"factual_id": factualID}).data()
for idObj in idList:
namespace = idObj["namespace"]
del idObj["factual_id"]
del idObj["namespace"]
mapping[namespace] = idObj
return mapping, True
except APIException:
log.error("Factual API failed again")
except Exception:
log.exception("Factual problem " + yelpID)
return mapping, False
def writeVenueProviderRecord(yelpID, details):
try:
venue = representation.updateRecord(yelpID, **details)
for provider, data in list(venue["providers"].items()):
db().child(venuesTable, "details", yelpID, "providers").update({provider: data})
except Exception as e:
log.error("Error writing record: {}\n{}".format(details, e))
def readCachedVenueIterableDetails(place_ids):
"""Retrieves the cache objects matching the given place IDs.
This method retrieves the whole cache/ child when making a call so call sparingly.
We do this because it's slow to make network requests for each child individually.
To pull down less data, you can use `readCacheVenueDetails`.
:param place_ids: Iterable of place_ids
:return: a list of cache objects. If a place_id is not in the cache, it will be dropped from the results.
"""
out = []
try:
cache = db().child(cacheTable).get().val()
for place_id in place_ids:
if place_id not in cache: continue
out.append(cache[place_id])
except Exception:
log.error("Error fetching cached venue details for " + place_id)
return out
def getVenueIdentifiers(yelpID):
yelpURL = "https://yelp.com/biz/%s" % yelpID
mapping = {
"id": yelpID,
"version": CROSSWALK_CACHE_VERSION,
"yelp": {
"url": yelpURL
}
}
try:
if deployment == "production/":
crosswalk = factualClient.table("crosswalk-us")
else:
crosswalk = factualClient.crosswalk()
obj = crosswalk.filters({"url": yelpURL}).data()
if len(obj) == 0:
log.debug("Crosswalk empty for Yelp -> Factual " + yelpID)
return mapping, True
factualID = obj[0]["factual_id"]
mapping["factualID"] = factualID
mapping["factual"] = {"id": factualID}
idList = crosswalk.filters({"factual_id": factualID}).data()
if len(idList) == 0:
log.warn("Crosswalk empty for Factual -> * " + yelpID + " " +
factualID)
for idObj in idList:
namespace = idObj["namespace"]
del idObj["factual_id"]
del idObj["namespace"]
mapping[namespace] = idObj
return mapping, True
except APIException:
log.error("Factual API failed again")
except Exception:
log.exception("Factual problem " + yelpID)
return mapping, False
def readCachedVenueDetails(key):
try:
cache = db.child(venuesTable).child("cache/" + key).get().val()
return cache
except Exception:
log.error("Error fetching cached venue details for " + key)
