def execute(self, options, args):
q = db.session.query(model.Ride)
# TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
q = q.filter(Ride.private == False)
if not options.rewrite:
q = q.filter(Ride.detail_fetched == False)
if options.athlete_id:
self.logger.info("Filtering activity details for {}".format(options.athlete_id))
q = q.filter(Ride.athlete_id == options.athlete_id)
if options.max_records:
self.logger.info("Limiting to {} records".format(options.max_records))
q = q.limit(options.max_records)
use_cache = options.use_cache or options.only_cache
self.logger.info("Fetching details for {} activities".format(q.count()))
if options.progress:
i = 0
bar = progressbar.ProgressBar(max_value=q.count())
for ride in q:
try:
client = data.StravaClientForAthlete(ride.athlete)
# TODO: Make it configurable to force refresh of data.
activity_json = self.get_cached_activity_json(ride) if use_cache else None
if options.progress:
i += 1
bar.update(i)
if activity_json is None:
if options.only_cache:
self.logger.info("[CACHE-MISS] Skipping ride {} since there is no cached version.")
continue
self.logger.info("[CACHE-MISS] Fetching activity detail for {!r}".format(ride))
# We do this manually, so that we can cache the JSON for later use.
activity_json = client.protocol.get("/activities/{id}", id=ride.id, include_all_efforts=True)
strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client)
try:
self.logger.info("Caching activity {!r}".format(ride))
self.cache_activity(strava_activity, activity_json)
except:
log.error(
"Error caching activity {} (ignoring)".format(strava_activity),
exc_info=self.logger.isEnabledFor(logging.DEBUG),
)
else:
strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client)
self.logger.info("[CACHE-HIT] Using cached activity detail for {!r}".format(ride))
# try:
# self.logger.info("Writing out GPS track for {!r}".format(ride))
# data.write_ride_track(strava_activity, ride)
# except:
# self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete),
# exc_info=self.logger.isEnabledFor(logging.DEBUG))
# raise
# We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities.
data.update_ride_from_activity(strava_activity=strava_activity, ride=ride)
try:
self.logger.info("Writing out efforts for {!r}".format(ride))
data.write_ride_efforts(strava_activity, ride)
except:
self.logger.error(
"Error writing efforts for activity {0}, athlete {1}".format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG),
)
raise
try:
self.logger.info("Writing out primary photo for {!r}".format(ride))
if strava_activity.total_photo_count > 0:
data.write_ride_photo_primary(strava_activity, ride)
else:
self.logger.debug("No photos for {!r}".format(ride))
except:
self.logger.error(
"Error writing primary photo for activity {}, athlete {}".format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG),
)
raise
ride.detail_fetched = True
db.session.commit()
except:
self.logger.exception(
"Error fetching/writing activity detail {}, athlete {}".format(ride.id, ride.athlete)
)
db.session.rollback()
def execute(self, options, args):
q = db.session.query(model.Ride)
# TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
q = q.filter(Ride.private == False)
if not options.rewrite:
q = q.filter(Ride.detail_fetched == False)
if options.athlete_id:
self.logger.info("Filtering activity details for {}".format(
options.athlete_id))
q = q.filter(Ride.athlete_id == options.athlete_id)
if options.max_records:
self.logger.info("Limiting to {} records".format(
options.max_records))
q = q.limit(options.max_records)
use_cache = options.use_cache or options.only_cache
self.logger.info("Fetching details for {} activities".format(
q.count()))
if options.progress:
i = 0
bar = progressbar.ProgressBar(max_value=q.count())
for ride in q:
try:
client = data.StravaClientForAthlete(ride.athlete)
# TODO: Make it configurable to force refresh of data.
activity_json = self.get_cached_activity_json(
ride) if use_cache else None
if options.progress:
i += 1
bar.update(i)
if activity_json is None:
if options.only_cache:
self.logger.info(
"[CACHE-MISS] Skipping ride {} since there is no cached version."
)
continue
self.logger.info(
"[CACHE-MISS] Fetching activity detail for {!r}".
format(ride))
# We do this manually, so that we can cache the JSON for later use.
activity_json = client.protocol.get(
'/activities/{id}',
id=ride.id,
include_all_efforts=True)
strava_activity = stravamodel.Activity.deserialize(
activity_json, bind_client=client)
try:
self.logger.info("Caching activity {!r}".format(ride))
self.cache_activity(strava_activity, activity_json)
except:
log.error(
"Error caching activity {} (ignoring)".format(
strava_activity),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
else:
strava_activity = stravamodel.Activity.deserialize(
activity_json, bind_client=client)
self.logger.info(
"[CACHE-HIT] Using cached activity detail for {!r}".
format(ride))
# try:
# self.logger.info("Writing out GPS track for {!r}".format(ride))
# data.write_ride_track(strava_activity, ride)
# except:
# self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete),
# exc_info=self.logger.isEnabledFor(logging.DEBUG))
# raise
# We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities.
data.update_ride_from_activity(strava_activity=strava_activity,
ride=ride)
try:
self.logger.info(
"Writing out efforts for {!r}".format(ride))
data.write_ride_efforts(strava_activity, ride)
except:
self.logger.error(
"Error writing efforts for activity {0}, athlete {1}".
format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
raise
try:
self.logger.info(
"Writing out primary photo for {!r}".format(ride))
if strava_activity.total_photo_count > 0:
data.write_ride_photo_primary(strava_activity, ride)
else:
self.logger.debug("No photos for {!r}".format(ride))
except:
self.logger.error(
"Error writing primary photo for activity {}, athlete {}"
.format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
raise
ride.detail_fetched = True
db.session.commit()
except:
self.logger.exception(
"Error fetching/writing activity detail {}, athlete {}".
format(ride.id, ride.athlete))
db.session.rollback()
def _write_rides(start, team=None, athlete_id=None, rewrite=False):
logger = logging.getLogger('sync')
if team and athlete_id:
raise ValueError("team and athlete params are mutually exclusive")
elif team is None and athlete_id is None:
raise ValueError("either team or athlete_id param is required")
sess = db.session
if team:
api_ride_entries = data.list_rides(club_id=team.id, start_date=start, exclude_keywords=app.config.get('BAFS_EXCLUDE_KEYWORDS'))
q = sess.query(model.Ride)
q = q.filter(and_(model.Ride.athlete_id.in_(sess.query(model.Athlete.id).filter(model.Athlete.team_id==team.id)),
model.Ride.start_date >= start))
db_rides = q.all()
else:
api_ride_entries = data.list_rides(athlete_id=athlete_id, start_date=start, exclude_keywords=app.config.get('BAFS_EXCLUDE_KEYWORDS'))
q = sess.query(model.Ride)
q = q.filter(and_(model.Ride.athlete_id == athlete_id,
model.Ride.start_date >= start))
db_rides = q.all()
# Quickly filter out only the rides that are not in the database.
returned_ride_ids = set([r.id for r in api_ride_entries])
stored_ride_ids = set([r.id for r in db_rides])
new_ride_ids = list(returned_ride_ids - stored_ride_ids)
removed_ride_ids = list(stored_ride_ids - returned_ride_ids)
if rewrite:
num_rides = len(api_ride_entries)
else:
num_rides = len(new_ride_ids)
# If we are "clearing" the system, then we'll just use all the returned rides as the "new" rides.
for (i, ri_entry) in enumerate(api_ride_entries):
logger.info("Processing ride: {0} ({1}/{2})".format(ri_entry.id, i, num_rides))
if rewrite or not ri_entry.id in stored_ride_ids:
ride = data.write_ride(ri_entry.id, team=team)
logger.debug("Wrote ride: %r" % (ride,))
else:
logger.debug("Skipping existing ride: {id} - {name!r}".format(name=ri_entry.name,id=ri_entry.id))
# Remove any rides that are in the database for this team that were not in the returned list.
if removed_ride_ids:
q = sess.query(model.Ride)
q = q.filter(model.Ride.id.in_(removed_ride_ids))
deleted = q.delete(synchronize_session=False)
if team:
logger.info("Removed {0} no longer present rides for team {1}.".format(deleted, team.id))
else:
logger.info("Removed {0} no longer present rides for athlete {1}.".format(deleted, athlete_id))
else:
if team:
logger.info("(No removed rides for team {0!r}.)".format(team))
else:
logger.info("(No removed rides for athlete {0}.)".format(athlete_id))
sess.commit()
# Write out any efforts associated with these rides (not already in database)
q = sess.query(model.Ride).filter_by(efforts_fetched=False)
for ride in q.all():
logger.info("Writing out efforts for {0!r}".format(ride))
data.write_ride_efforts(ride)