def execute(self, args): fetcher = ActivitySync(logger=self.logger) fetcher.sync_rides_detail(athlete_id=args.athlete_id, rewrite=args.rewrite, use_cache=args.use_cache, only_cache=args.only_cache, max_records=args.max_records)
def __init__(self, beanstalk_client: greenstalk.Client, shutdown_event: threading.Event): self.client = beanstalk_client self.shutdown_event = shutdown_event self.logger = logging.getLogger(__name__) self.activity_sync = ActivitySync(self.logger) self.streams_sync = StreamSync(self.logger)
def execute(self, args): fetcher = ActivitySync(logger=self.logger) fetcher.sync_rides( start_date=args.start_date, athlete_ids=args.athlete_id, rewrite=args.rewrite, force=args.force, )
def main(): init_logging() init_model(config.SQLALCHEMY_URL) shutdown_event = threading.Event() scheduler = BackgroundScheduler() # workflow_publisher = configured_publisher() activity_sync = ActivitySync() weather_sync = WeatherSync() athlete_sync = AthleteSync() # Every hour run a sync on the activities for athletes fall into the specified segment # athlete_id % total_segments == segment # TODO: Probably it would be more prudent to split into 15-minute segments, to match rate limits # Admittedly that will make the time-based segment calculation a little trickier. def segmented_sync_activities(): activity_sync.sync_rides_distributed(total_segments=4, segment=(arrow.now().hour % 4)) scheduler.add_job(segmented_sync_activities, 'cron', minute='50') # This should generally not pick up anytihng. scheduler.add_job(activity_sync.sync_rides_detail, 'cron', minute='20') # Sync weather at 8am UTC scheduler.add_job(weather_sync.sync_weather, 'cron', hour='8') # Sync athletes once a day at 6am UTC scheduler.add_job(athlete_sync.sync_athletes, 'cron', minute='30') scheduler.start() beanclient = Client(host=config.BEANSTALKD_HOST, port=config.BEANSTALKD_PORT, watch=[DefinedTubes.activity_update.value]) subscriber = ActivityUpdateSubscriber(beanstalk_client=beanclient, shutdown_event=shutdown_event) def shutdown_app(): shutdown_event.wait() scheduler.shutdown() shutdown_monitor = threading.Thread(target=shutdown_app) shutdown_monitor.start() try: # This is here to simulate application activity (which keeps the main thread alive). subscriber.run_forever() except (KeyboardInterrupt, SystemExit): log.info("Exiting on user request.") except: log.exception("Error running sync/listener.") finally: shutdown_event.set() shutdown_monitor.join()
class ActivityUpdateSubscriber: def __init__(self, beanstalk_client: greenstalk.Client, shutdown_event: threading.Event): self.client = beanstalk_client self.shutdown_event = shutdown_event self.logger = logging.getLogger(__name__) self.activity_sync = ActivitySync(self.logger) self.streams_sync = StreamSync(self.logger) def handle_message(self, message: ActivityUpdate): self.logger.info("Processing activity update {}".format(message)) with meta.transaction_context() as session: athlete: Athlete = session.query(Athlete).get(message.athlete_id) if not athlete: self.logger.warning( "Athlete {} not found in database, " "ignoring activity update message {}".format( message.athlete_id, message)) return # Makes the else a little unnecessary, but reads easier. try: if message.operation is AspectType.delete: statsd.increment('strava.activity.delete', tags=['team:{}'.format(athlete.team_id)]) self.activity_sync.delete_activity( athlete_id=message.athlete_id, activity_id=message.activity_id) elif message.operation is AspectType.update: statsd.increment('strava.activity.update', tags=['team:{}'.format(athlete.team_id)]) self.activity_sync.fetch_and_store_actvitiy_detail( athlete_id=message.athlete_id, activity_id=message.activity_id) # (We'll assume the stream doens't need re-fetching.) elif message.operation is AspectType.create: statsd.increment('strava.activity.create', tags=['team:{}'.format(athlete.team_id)]) self.activity_sync.fetch_and_store_actvitiy_detail( athlete_id=message.athlete_id, activity_id=message.activity_id) self.streams_sync.fetch_and_store_activity_streams( athlete_id=message.athlete_id, activity_id=message.activity_id) except (ActivityNotFound, IneligibleActivity) as x: log.info(str(x)) def run_forever(self): # This is expecting to run in the main thread. Needs a bit of redesign # if this is to be moved to a background thread. try: schema = ActivityUpdateSchema() while not self.shutdown_event.is_set(): try: job = self.client.reserve(timeout=30) except (KeyboardInterrupt, SystemExit): raise except greenstalk.TimedOutError: self.logger.debug( "Internal beanstalkd connection timeout; reconnecting." ) continue else: try: self.logger.info("Received message: {!r}".format( job.body)) update = schema.loads(job.body).data self.handle_message(update) except: self.logger.exception( "Error procesing message, will requeue w/ delay.") statsd.increment('strava.webhook.error') self.client.release( job, delay=20) # We put it back with a delay else: self.client.delete(job) except (KeyboardInterrupt, SystemExit): raise except: self.logger.exception( "Unhandled error in tube subscriber loop, exiting.") self.shutdown_event.set() raise