예제 #1
0
    def execute(self, options, args):

        q = db.session.query(model.Ride)

        # TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
        q = q.filter(Ride.private == False)

        if not options.rewrite:
            q = q.filter(Ride.detail_fetched == False)

        if options.athlete_id:
            self.logger.info("Filtering activity details for {}".format(options.athlete_id))
            q = q.filter(Ride.athlete_id == options.athlete_id)

        if options.max_records:
            self.logger.info("Limiting to {} records".format(options.max_records))
            q = q.limit(options.max_records)

        use_cache = options.use_cache or options.only_cache

        self.logger.info("Fetching details for {} activities".format(q.count()))

        if options.progress:
            i = 0
            bar = progressbar.ProgressBar(max_value=q.count())

        for ride in q:
            try:
                client = data.StravaClientForAthlete(ride.athlete)

                # TODO: Make it configurable to force refresh of data.
                activity_json = self.get_cached_activity_json(ride) if use_cache else None

                if options.progress:
                    i += 1
                    bar.update(i)

                if activity_json is None:
                    if options.only_cache:
                        self.logger.info("[CACHE-MISS] Skipping ride {} since there is no cached version.")
                        continue

                    self.logger.info("[CACHE-MISS] Fetching activity detail for {!r}".format(ride))
                    # We do this manually, so that we can cache the JSON for later use.
                    activity_json = client.protocol.get("/activities/{id}", id=ride.id, include_all_efforts=True)
                    strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client)

                    try:
                        self.logger.info("Caching activity {!r}".format(ride))
                        self.cache_activity(strava_activity, activity_json)
                    except:
                        log.error(
                            "Error caching activity {} (ignoring)".format(strava_activity),
                            exc_info=self.logger.isEnabledFor(logging.DEBUG),
                        )

                else:
                    strava_activity = stravamodel.Activity.deserialize(activity_json, bind_client=client)
                    self.logger.info("[CACHE-HIT] Using cached activity detail for {!r}".format(ride))

                # try:
                #     self.logger.info("Writing out GPS track for {!r}".format(ride))
                #     data.write_ride_track(strava_activity, ride)
                # except:
                #     self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete),
                #                       exc_info=self.logger.isEnabledFor(logging.DEBUG))
                #     raise

                # We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities.
                data.update_ride_from_activity(strava_activity=strava_activity, ride=ride)

                try:
                    self.logger.info("Writing out efforts for {!r}".format(ride))
                    data.write_ride_efforts(strava_activity, ride)
                except:
                    self.logger.error(
                        "Error writing efforts for activity {0}, athlete {1}".format(ride.id, ride.athlete),
                        exc_info=self.logger.isEnabledFor(logging.DEBUG),
                    )
                    raise

                try:
                    self.logger.info("Writing out primary photo for {!r}".format(ride))
                    if strava_activity.total_photo_count > 0:
                        data.write_ride_photo_primary(strava_activity, ride)
                    else:
                        self.logger.debug("No photos for {!r}".format(ride))
                except:
                    self.logger.error(
                        "Error writing primary photo for activity {}, athlete {}".format(ride.id, ride.athlete),
                        exc_info=self.logger.isEnabledFor(logging.DEBUG),
                    )
                    raise

                ride.detail_fetched = True
                db.session.commit()

            except:
                self.logger.exception(
                    "Error fetching/writing activity detail {}, athlete {}".format(ride.id, ride.athlete)
                )
                db.session.rollback()
예제 #2
0
    def execute(self, options, args):

        q = db.session.query(model.Ride)

        # TODO: Construct a more complex query to catch photos_fetched=False, track_fetched=False, etc.
        q = q.filter(Ride.private == False)

        if not options.rewrite:
            q = q.filter(Ride.detail_fetched == False)

        if options.athlete_id:
            self.logger.info("Filtering activity details for {}".format(
                options.athlete_id))
            q = q.filter(Ride.athlete_id == options.athlete_id)

        if options.max_records:
            self.logger.info("Limiting to {} records".format(
                options.max_records))
            q = q.limit(options.max_records)

        use_cache = options.use_cache or options.only_cache

        self.logger.info("Fetching details for {} activities".format(
            q.count()))

        if options.progress:
            i = 0
            bar = progressbar.ProgressBar(max_value=q.count())

        for ride in q:
            try:
                client = data.StravaClientForAthlete(ride.athlete)

                # TODO: Make it configurable to force refresh of data.
                activity_json = self.get_cached_activity_json(
                    ride) if use_cache else None

                if options.progress:
                    i += 1
                    bar.update(i)

                if activity_json is None:
                    if options.only_cache:
                        self.logger.info(
                            "[CACHE-MISS] Skipping ride {} since there is no cached version."
                        )
                        continue

                    self.logger.info(
                        "[CACHE-MISS] Fetching activity detail for {!r}".
                        format(ride))
                    # We do this manually, so that we can cache the JSON for later use.
                    activity_json = client.protocol.get(
                        '/activities/{id}',
                        id=ride.id,
                        include_all_efforts=True)
                    strava_activity = stravamodel.Activity.deserialize(
                        activity_json, bind_client=client)

                    try:
                        self.logger.info("Caching activity {!r}".format(ride))
                        self.cache_activity(strava_activity, activity_json)
                    except:
                        log.error(
                            "Error caching activity {} (ignoring)".format(
                                strava_activity),
                            exc_info=self.logger.isEnabledFor(logging.DEBUG))

                else:
                    strava_activity = stravamodel.Activity.deserialize(
                        activity_json, bind_client=client)
                    self.logger.info(
                        "[CACHE-HIT] Using cached activity detail for {!r}".
                        format(ride))

                # try:
                #     self.logger.info("Writing out GPS track for {!r}".format(ride))
                #     data.write_ride_track(strava_activity, ride)
                # except:
                #     self.logger.error("Error writing track for activity {0}, athlete {1}".format(ride.id, ride.athlete),
                #                       exc_info=self.logger.isEnabledFor(logging.DEBUG))
                #     raise

                # We do this just to take advantage of the use-cache/only-cache feature for reprocessing activities.
                data.update_ride_from_activity(strava_activity=strava_activity,
                                               ride=ride)

                try:
                    self.logger.info(
                        "Writing out efforts for {!r}".format(ride))
                    data.write_ride_efforts(strava_activity, ride)
                except:
                    self.logger.error(
                        "Error writing efforts for activity {0}, athlete {1}".
                        format(ride.id, ride.athlete),
                        exc_info=self.logger.isEnabledFor(logging.DEBUG))
                    raise

                try:
                    self.logger.info(
                        "Writing out primary photo for {!r}".format(ride))
                    if strava_activity.total_photo_count > 0:
                        data.write_ride_photo_primary(strava_activity, ride)
                    else:
                        self.logger.debug("No photos for {!r}".format(ride))
                except:
                    self.logger.error(
                        "Error writing primary photo for activity {}, athlete {}"
                        .format(ride.id, ride.athlete),
                        exc_info=self.logger.isEnabledFor(logging.DEBUG))
                    raise

                ride.detail_fetched = True
                db.session.commit()

            except:
                self.logger.exception(
                    "Error fetching/writing activity detail {}, athlete {}".
                    format(ride.id, ride.athlete))
                db.session.rollback()
예제 #3
0
파일: scripts.py 프로젝트: jrenaut/bafs
def _write_rides(start, team=None, athlete_id=None, rewrite=False):
    
    logger = logging.getLogger('sync')
    
    if team and athlete_id:
        raise ValueError("team and athlete params are mutually exclusive")
    elif team is None and athlete_id is None:
        raise ValueError("either team or athlete_id param is required")
    
    sess = db.session
    
    if team:
        api_ride_entries = data.list_rides(club_id=team.id, start_date=start, exclude_keywords=app.config.get('BAFS_EXCLUDE_KEYWORDS'))
        q = sess.query(model.Ride)
        q = q.filter(and_(model.Ride.athlete_id.in_(sess.query(model.Athlete.id).filter(model.Athlete.team_id==team.id)),
                          model.Ride.start_date >= start))
        db_rides = q.all()
    else:
        api_ride_entries = data.list_rides(athlete_id=athlete_id, start_date=start, exclude_keywords=app.config.get('BAFS_EXCLUDE_KEYWORDS'))
        q = sess.query(model.Ride)
        q = q.filter(and_(model.Ride.athlete_id == athlete_id,
                          model.Ride.start_date >= start))
        db_rides = q.all()
    
    # Quickly filter out only the rides that are not in the database.
    returned_ride_ids = set([r.id for r in api_ride_entries])
    stored_ride_ids = set([r.id for r in db_rides])
    new_ride_ids = list(returned_ride_ids - stored_ride_ids)
    removed_ride_ids = list(stored_ride_ids - returned_ride_ids)
    
    if rewrite:
        num_rides = len(api_ride_entries)
    else:
        num_rides = len(new_ride_ids)
    
    # If we are "clearing" the system, then we'll just use all the returned rides as the "new" rides.
    
    for (i, ri_entry) in enumerate(api_ride_entries):
        logger.info("Processing ride: {0} ({1}/{2})".format(ri_entry.id, i, num_rides))
        if rewrite or not ri_entry.id in stored_ride_ids:
            ride = data.write_ride(ri_entry.id, team=team)
            logger.debug("Wrote ride: %r" % (ride,))
        else:
            logger.debug("Skipping existing ride: {id} - {name!r}".format(name=ri_entry.name,id=ri_entry.id))

    # Remove any rides that are in the database for this team that were not in the returned list.
    if removed_ride_ids:
        q = sess.query(model.Ride)
        q = q.filter(model.Ride.id.in_(removed_ride_ids))
        deleted = q.delete(synchronize_session=False)
        if team:
            logger.info("Removed {0} no longer present rides for team {1}.".format(deleted, team.id))
        else:
            logger.info("Removed {0} no longer present rides for athlete {1}.".format(deleted, athlete_id))
    else:
        if team:
            logger.info("(No removed rides for team {0!r}.)".format(team))
        else:
            logger.info("(No removed rides for athlete {0}.)".format(athlete_id))
    
    
    sess.commit() 
        
    # Write out any efforts associated with these rides (not already in database)
    q = sess.query(model.Ride).filter_by(efforts_fetched=False)
    for ride in q.all():
        logger.info("Writing out efforts for {0!r}".format(ride))
        data.write_ride_efforts(ride)