def execute(self, args):
fetcher = ActivitySync(logger=self.logger)
fetcher.sync_rides_detail(athlete_id=args.athlete_id,
rewrite=args.rewrite,
use_cache=args.use_cache,
only_cache=args.only_cache,
max_records=args.max_records)
def __init__(self, beanstalk_client: greenstalk.Client,
shutdown_event: threading.Event):
self.client = beanstalk_client
self.shutdown_event = shutdown_event
self.logger = logging.getLogger(__name__)
self.activity_sync = ActivitySync(self.logger)
self.streams_sync = StreamSync(self.logger)
def execute(self, args):
fetcher = ActivitySync(logger=self.logger)
fetcher.sync_rides(
start_date=args.start_date,
athlete_ids=args.athlete_id,
rewrite=args.rewrite,
force=args.force,
)
def main():
init_logging()
init_model(config.SQLALCHEMY_URL)
shutdown_event = threading.Event()
scheduler = BackgroundScheduler()
# workflow_publisher = configured_publisher()
activity_sync = ActivitySync()
weather_sync = WeatherSync()
athlete_sync = AthleteSync()
# Every hour run a sync on the activities for athletes fall into the specified segment
# athlete_id % total_segments == segment
# TODO: Probably it would be more prudent to split into 15-minute segments, to match rate limits
# Admittedly that will make the time-based segment calculation a little trickier.
def segmented_sync_activities():
activity_sync.sync_rides_distributed(total_segments=4,
segment=(arrow.now().hour % 4))
scheduler.add_job(segmented_sync_activities, 'cron', minute='50')
# This should generally not pick up anytihng.
scheduler.add_job(activity_sync.sync_rides_detail, 'cron', minute='20')
# Sync weather at 8am UTC
scheduler.add_job(weather_sync.sync_weather, 'cron', hour='8')
# Sync athletes once a day at 6am UTC
scheduler.add_job(athlete_sync.sync_athletes, 'cron', minute='30')
scheduler.start()
beanclient = Client(host=config.BEANSTALKD_HOST,
port=config.BEANSTALKD_PORT,
watch=[DefinedTubes.activity_update.value])
subscriber = ActivityUpdateSubscriber(beanstalk_client=beanclient,
shutdown_event=shutdown_event)
def shutdown_app():
shutdown_event.wait()
scheduler.shutdown()
shutdown_monitor = threading.Thread(target=shutdown_app)
shutdown_monitor.start()
try:
# This is here to simulate application activity (which keeps the main thread alive).
subscriber.run_forever()
except (KeyboardInterrupt, SystemExit):
log.info("Exiting on user request.")
except:
log.exception("Error running sync/listener.")
finally:
shutdown_event.set()
shutdown_monitor.join()
class ActivityUpdateSubscriber:
def __init__(self, beanstalk_client: greenstalk.Client,
shutdown_event: threading.Event):
self.client = beanstalk_client
self.shutdown_event = shutdown_event
self.logger = logging.getLogger(__name__)
self.activity_sync = ActivitySync(self.logger)
self.streams_sync = StreamSync(self.logger)
def handle_message(self, message: ActivityUpdate):
self.logger.info("Processing activity update {}".format(message))
with meta.transaction_context() as session:
athlete: Athlete = session.query(Athlete).get(message.athlete_id)
if not athlete:
self.logger.warning(
"Athlete {} not found in database, "
"ignoring activity update message {}".format(
message.athlete_id, message))
return # Makes the else a little unnecessary, but reads easier.
try:
if message.operation is AspectType.delete:
statsd.increment('strava.activity.delete',
tags=['team:{}'.format(athlete.team_id)])
self.activity_sync.delete_activity(
athlete_id=message.athlete_id,
activity_id=message.activity_id)
elif message.operation is AspectType.update:
statsd.increment('strava.activity.update',
tags=['team:{}'.format(athlete.team_id)])
self.activity_sync.fetch_and_store_actvitiy_detail(
athlete_id=message.athlete_id,
activity_id=message.activity_id)
# (We'll assume the stream doens't need re-fetching.)
elif message.operation is AspectType.create:
statsd.increment('strava.activity.create',
tags=['team:{}'.format(athlete.team_id)])
self.activity_sync.fetch_and_store_actvitiy_detail(
athlete_id=message.athlete_id,
activity_id=message.activity_id)
self.streams_sync.fetch_and_store_activity_streams(
athlete_id=message.athlete_id,
activity_id=message.activity_id)
except (ActivityNotFound, IneligibleActivity) as x:
log.info(str(x))
def run_forever(self):
# This is expecting to run in the main thread. Needs a bit of redesign
# if this is to be moved to a background thread.
try:
schema = ActivityUpdateSchema()
while not self.shutdown_event.is_set():
try:
job = self.client.reserve(timeout=30)
except (KeyboardInterrupt, SystemExit):
raise
except greenstalk.TimedOutError:
self.logger.debug(
"Internal beanstalkd connection timeout; reconnecting."
)
continue
else:
try:
self.logger.info("Received message: {!r}".format(
job.body))
update = schema.loads(job.body).data
self.handle_message(update)
except:
self.logger.exception(
"Error procesing message, will requeue w/ delay.")
statsd.increment('strava.webhook.error')
self.client.release(
job, delay=20) # We put it back with a delay
else:
self.client.delete(job)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.logger.exception(
"Unhandled error in tube subscriber loop, exiting.")
self.shutdown_event.set()
raise