def execute(self, options, args):
q = db.session.query(model.Ride)
# TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
q = q.filter(Ride.private == False)
if not options.rewrite:
q = q.filter(Ride.detail_fetched == False)
if options.athlete_id:
self.logger.info("Filtering activity details for {}".format(options.athlete_id))
q = q.filter(Ride.athlete_id == options.athlete_id)
if options.max_records:
self.logger.info("Limiting to {} records".format(options.max_records))
q = q.limit(options.max_records)
use_cache = options.use_cache or options.only_cache
self.logger.info("Fetching details for {} activities".format(q.count()))
if options.progress:
i = 0
bar = progressbar.ProgressBar(max_value=q.count())
for ride in q:
try:
client = data.StravaClientForAthlete(ride.athlete)
# TODO: Make it configurable to force refresh of data.
activity_json = self.get_cached_activity_json(ride) if use_cache else None
if options.progress:
i += 1
bar.update(i)
if activity_json is None:
if options.only_cache:
self.logger.info("[CACHE-MISS] Skipping ride {} since there is no cached version.")
continue
self.logger.info("[CACHE-MISS] Fetching activity detail for {!r}".format(ride))
# We do this manually, so that we can cache the JSON for later use.
activity_json = client.protocol.get("/activities/{id}", id=ride.id, include_all_efforts=True)
strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client)
try:
self.logger.info("Caching activity {!r}".format(ride))
self.cache_activity(strava_activity, activity_json)
except:
log.error(
"Error caching activity {} (ignoring)".format(strava_activity),
exc_info=self.logger.isEnabledFor(logging.DEBUG),
)
else:
strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client)
self.logger.info("[CACHE-HIT] Using cached activity detail for {!r}".format(ride))
# try:
# self.logger.info("Writing out GPS track for {!r}".format(ride))
# data.write_ride_track(strava_activity, ride)
# except:
# self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete),
# exc_info=self.logger.isEnabledFor(logging.DEBUG))
# raise
# We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities.
data.update_ride_from_activity(strava_activity=strava_activity, ride=ride)
try:
self.logger.info("Writing out efforts for {!r}".format(ride))
data.write_ride_efforts(strava_activity, ride)
except:
self.logger.error(
"Error writing efforts for activity {0}, athlete {1}".format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG),
)
raise
try:
self.logger.info("Writing out primary photo for {!r}".format(ride))
if strava_activity.total_photo_count > 0:
data.write_ride_photo_primary(strava_activity, ride)
else:
self.logger.debug("No photos for {!r}".format(ride))
except:
self.logger.error(
"Error writing primary photo for activity {}, athlete {}".format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG),
)
raise
ride.detail_fetched = True
db.session.commit()
except:
self.logger.exception(
"Error fetching/writing activity detail {}, athlete {}".format(ride.id, ride.athlete)
)
db.session.rollback()
def execute(self, options, args):
q = db.session.query(model.Ride)
# TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
q = q.filter(Ride.private == False)
if not options.rewrite:
q = q.filter(Ride.detail_fetched == False)
if options.athlete_id:
self.logger.info("Filtering activity details for {}".format(
options.athlete_id))
q = q.filter(Ride.athlete_id == options.athlete_id)
if options.max_records:
self.logger.info("Limiting to {} records".format(
options.max_records))
q = q.limit(options.max_records)
use_cache = options.use_cache or options.only_cache
self.logger.info("Fetching details for {} activities".format(
q.count()))
if options.progress:
i = 0
bar = progressbar.ProgressBar(max_value=q.count())
for ride in q:
try:
client = data.StravaClientForAthlete(ride.athlete)
# TODO: Make it configurable to force refresh of data.
activity_json = self.get_cached_activity_json(
ride) if use_cache else None
if options.progress:
i += 1
bar.update(i)
if activity_json is None:
if options.only_cache:
self.logger.info(
"[CACHE-MISS] Skipping ride {} since there is no cached version."
)
continue
self.logger.info(
"[CACHE-MISS] Fetching activity detail for {!r}".
format(ride))
# We do this manually, so that we can cache the JSON for later use.
activity_json = client.protocol.get(
'/activities/{id}',
id=ride.id,
include_all_efforts=True)
strava_activity = stravamodel.Activity.deserialize(
activity_json, bind_client=client)
try:
self.logger.info("Caching activity {!r}".format(ride))
self.cache_activity(strava_activity, activity_json)
except:
log.error(
"Error caching activity {} (ignoring)".format(
strava_activity),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
else:
strava_activity = stravamodel.Activity.deserialize(
activity_json, bind_client=client)
self.logger.info(
"[CACHE-HIT] Using cached activity detail for {!r}".
format(ride))
# try:
# self.logger.info("Writing out GPS track for {!r}".format(ride))
# data.write_ride_track(strava_activity, ride)
# except:
# self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete),
# exc_info=self.logger.isEnabledFor(logging.DEBUG))
# raise
# We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities.
data.update_ride_from_activity(strava_activity=strava_activity,
ride=ride)
try:
self.logger.info(
"Writing out efforts for {!r}".format(ride))
data.write_ride_efforts(strava_activity, ride)
except:
self.logger.error(
"Error writing efforts for activity {0}, athlete {1}".
format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
raise
try:
self.logger.info(
"Writing out primary photo for {!r}".format(ride))
if strava_activity.total_photo_count > 0:
data.write_ride_photo_primary(strava_activity, ride)
else:
self.logger.debug("No photos for {!r}".format(ride))
except:
self.logger.error(
"Error writing primary photo for activity {}, athlete {}"
.format(ride.id, ride.athlete),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
raise
ride.detail_fetched = True
db.session.commit()
except:
self.logger.exception(
"Error fetching/writing activity detail {}, athlete {}".
format(ride.id, ride.athlete))
db.session.rollback()
def execute(self, options, args):
q = db.session.query(model.Ride)
# TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
q = q.filter(and_(Ride.private==False,
Ride.manual==False))
if not options.rewrite:
q = q.filter(Ride.track_fetched==False,)
if options.athlete_id:
self.logger.info("Filtering activity details for {}".format(options.athlete_id))
q = q.filter(Ride.athlete_id == options.athlete_id)
if options.max_records:
self.logger.info("Limiting to {} records".format(options.max_records))
q = q.limit(options.max_records)
use_cache = options.use_cache or options.only_cache
self.logger.info("Fetching gps tracks for {} activities".format(q.count()))
for ride in q:
try:
client = data.StravaClientForAthlete(ride.athlete)
# TODO: Make it configurable to force refresh of data.
streams_json = self.get_cached_streams_json(ride) if use_cache else None
if streams_json is None:
if options.only_cache:
self.logger.info("[CACHE-MISS] Skipping ride {} since there is no cached stream.".format(ride))
continue
self.logger.info("[CACHE-MISS] Fetching streams for {!r}".format(ride))
# We do this manually, so that we can cache the JSON for later use.
streams_json = client.protocol.get(
'/activities/{id}/streams/{types}'.format(id=ride.id, types='latlng,time,altitude'),
resolution='low'
)
streams = [stravamodel.Stream.deserialize(stream_struct, bind_client=client) for stream_struct in streams_json]
try:
self.logger.info("Caching streams for {!r}".format(ride))
self.cache_stream(ride, streams_json)
except:
log.error("Error caching streams for activity {} (ignoring)".format(ride),
exc_info=self.logger.isEnabledFor(logging.DEBUG))
else:
streams = [stravamodel.Stream.deserialize(stream_struct, bind_client=client) for stream_struct in streams_json]
self.logger.info("[CACHE-HIT] Using cached streams detail for {!r}".format(ride))
data.write_ride_streams(streams, ride)
db.session.commit()
except:
self.logger.exception("Error fetching/writing activity streams for {}, athlete {}".format(ride, ride.athlete))
db.session.rollback()
