def execute(self, options, args): q = db.session.query(model.Ride) # TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc. q = q.filter(Ride.private == False) if not options.rewrite: q = q.filter(Ride.detail_fetched == False) if options.athlete_id: self.logger.info("Filtering activity details for {}".format(options.athlete_id)) q = q.filter(Ride.athlete_id == options.athlete_id) if options.max_records: self.logger.info("Limiting to {} records".format(options.max_records)) q = q.limit(options.max_records) use_cache = options.use_cache or options.only_cache self.logger.info("Fetching details for {} activities".format(q.count())) if options.progress: i = 0 bar = progressbar.ProgressBar(max_value=q.count()) for ride in q: try: client = data.StravaClientForAthlete(ride.athlete) # TODO: Make it configurable to force refresh of data. activity_json = self.get_cached_activity_json(ride) if use_cache else None if options.progress: i += 1 bar.update(i) if activity_json is None: if options.only_cache: self.logger.info("[CACHE-MISS] Skipping ride {} since there is no cached version.") continue self.logger.info("[CACHE-MISS] Fetching activity detail for {!r}".format(ride)) # We do this manually, so that we can cache the JSON for later use. activity_json = client.protocol.get("/activities/{id}", id=ride.id, include_all_efforts=True) strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client) try: self.logger.info("Caching activity {!r}".format(ride)) self.cache_activity(strava_activity, activity_json) except: log.error( "Error caching activity {} (ignoring)".format(strava_activity), exc_info=self.logger.isEnabledFor(logging.DEBUG), ) else: strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client) self.logger.info("[CACHE-HIT] Using cached activity detail for {!r}".format(ride)) # try: # self.logger.info("Writing out GPS track for {!r}".format(ride)) # data.write_ride_track(strava_activity, ride) # except: # self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete), # exc_info=self.logger.isEnabledFor(logging.DEBUG)) # raise # We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities. data.update_ride_from_activity(strava_activity=strava_activity, ride=ride) try: self.logger.info("Writing out efforts for {!r}".format(ride)) data.write_ride_efforts(strava_activity, ride) except: self.logger.error( "Error writing efforts for activity {0}, athlete {1}".format(ride.id, ride.athlete), exc_info=self.logger.isEnabledFor(logging.DEBUG), ) raise try: self.logger.info("Writing out primary photo for {!r}".format(ride)) if strava_activity.total_photo_count > 0: data.write_ride_photo_primary(strava_activity, ride) else: self.logger.debug("No photos for {!r}".format(ride)) except: self.logger.error( "Error writing primary photo for activity {}, athlete {}".format(ride.id, ride.athlete), exc_info=self.logger.isEnabledFor(logging.DEBUG), ) raise ride.detail_fetched = True db.session.commit() except: self.logger.exception( "Error fetching/writing activity detail {}, athlete {}".format(ride.id, ride.athlete) ) db.session.rollback()
def execute(self, options, args): q = db.session.query(model.Ride) # TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc. q = q.filter(Ride.private == False) if not options.rewrite: q = q.filter(Ride.detail_fetched == False) if options.athlete_id: self.logger.info("Filtering activity details for {}".format( options.athlete_id)) q = q.filter(Ride.athlete_id == options.athlete_id) if options.max_records: self.logger.info("Limiting to {} records".format( options.max_records)) q = q.limit(options.max_records) use_cache = options.use_cache or options.only_cache self.logger.info("Fetching details for {} activities".format( q.count())) if options.progress: i = 0 bar = progressbar.ProgressBar(max_value=q.count()) for ride in q: try: client = data.StravaClientForAthlete(ride.athlete) # TODO: Make it configurable to force refresh of data. activity_json = self.get_cached_activity_json( ride) if use_cache else None if options.progress: i += 1 bar.update(i) if activity_json is None: if options.only_cache: self.logger.info( "[CACHE-MISS] Skipping ride {} since there is no cached version." ) continue self.logger.info( "[CACHE-MISS] Fetching activity detail for {!r}". format(ride)) # We do this manually, so that we can cache the JSON for later use. activity_json = client.protocol.get( '/activities/{id}', id=ride.id, include_all_efforts=True) strava_activity = stravamodel.Activity.deserialize( activity_json, bind_client=client) try: self.logger.info("Caching activity {!r}".format(ride)) self.cache_activity(strava_activity, activity_json) except: log.error( "Error caching activity {} (ignoring)".format( strava_activity), exc_info=self.logger.isEnabledFor(logging.DEBUG)) else: strava_activity = stravamodel.Activity.deserialize( activity_json, bind_client=client) self.logger.info( "[CACHE-HIT] Using cached activity detail for {!r}". format(ride)) # try: # self.logger.info("Writing out GPS track for {!r}".format(ride)) # data.write_ride_track(strava_activity, ride) # except: # self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete), # exc_info=self.logger.isEnabledFor(logging.DEBUG)) # raise # We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities. data.update_ride_from_activity(strava_activity=strava_activity, ride=ride) try: self.logger.info( "Writing out efforts for {!r}".format(ride)) data.write_ride_efforts(strava_activity, ride) except: self.logger.error( "Error writing efforts for activity {0}, athlete {1}". format(ride.id, ride.athlete), exc_info=self.logger.isEnabledFor(logging.DEBUG)) raise try: self.logger.info( "Writing out primary photo for {!r}".format(ride)) if strava_activity.total_photo_count > 0: data.write_ride_photo_primary(strava_activity, ride) else: self.logger.debug("No photos for {!r}".format(ride)) except: self.logger.error( "Error writing primary photo for activity {}, athlete {}" .format(ride.id, ride.athlete), exc_info=self.logger.isEnabledFor(logging.DEBUG)) raise ride.detail_fetched = True db.session.commit() except: self.logger.exception( "Error fetching/writing activity detail {}, athlete {}". format(ride.id, ride.athlete)) db.session.rollback()
def execute(self, options, args): q = db.session.query(model.Ride) # TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc. q = q.filter(and_(Ride.private==False, Ride.manual==False)) if not options.rewrite: q = q.filter(Ride.track_fetched==False,) if options.athlete_id: self.logger.info("Filtering activity details for {}".format(options.athlete_id)) q = q.filter(Ride.athlete_id == options.athlete_id) if options.max_records: self.logger.info("Limiting to {} records".format(options.max_records)) q = q.limit(options.max_records) use_cache = options.use_cache or options.only_cache self.logger.info("Fetching gps tracks for {} activities".format(q.count())) for ride in q: try: client = data.StravaClientForAthlete(ride.athlete) # TODO: Make it configurable to force refresh of data. streams_json = self.get_cached_streams_json(ride) if use_cache else None if streams_json is None: if options.only_cache: self.logger.info("[CACHE-MISS] Skipping ride {} since there is no cached stream.".format(ride)) continue self.logger.info("[CACHE-MISS] Fetching streams for {!r}".format(ride)) # We do this manually, so that we can cache the JSON for later use. streams_json = client.protocol.get( '/activities/{id}/streams/{types}'.format(id=ride.id, types='latlng,time,altitude'), resolution='low' ) streams = [stravamodel.Stream.deserialize(stream_struct, bind_client=client) for stream_struct in streams_json] try: self.logger.info("Caching streams for {!r}".format(ride)) self.cache_stream(ride, streams_json) except: log.error("Error caching streams for activity {} (ignoring)".format(ride), exc_info=self.logger.isEnabledFor(logging.DEBUG)) else: streams = [stravamodel.Stream.deserialize(stream_struct, bind_client=client) for stream_struct in streams_json] self.logger.info("[CACHE-HIT] Using cached streams detail for {!r}".format(ride)) data.write_ride_streams(streams, ride) db.session.commit() except: self.logger.exception("Error fetching/writing activity streams for {}, athlete {}".format(ride, ride.athlete)) db.session.rollback()