def _getAPIUris(self, serviceRecord):
if hasattr(self, "_uris"): # cache these for the life of the batch job at least? hope so
return self._uris
else:
uris_json = redis.get(self._URI_CACHE_KEY)
if uris_json is not None:
uris = json.loads(uris_json.decode('utf-8'))
else:
response = self._rate_limit("user",
lambda: requests.get("https://api.runkeeper.com/user/",
headers=self._apiHeaders(serviceRecord)))
if response.status_code != 200:
if response.status_code == 401 or response.status_code == 403:
raise APIException("No authorization to retrieve user URLs", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True))
raise APIException("Unable to retrieve user URLs" + str(response))
uris = response.json()
for k in uris.keys():
if type(uris[k]) == str:
uris[k] = "https://api.runkeeper.com" + uris[k]
# Runkeeper wants you to request these on a per-user basis.
# In practice, the URIs are identical for ever user (only the userID key changes).
# So, only do it once every 24 hours, across the entire system.
redis.setex(self._URI_CACHE_KEY, json.dumps(uris), timedelta(hours=24))
self._uris = uris
return uris
def _getAPIUris(self, serviceRecord):
if hasattr(
self, "_uris"
): # cache these for the life of the batch job at least? hope so
return self._uris
else:
uris_json = redis.get(self._URI_CACHE_KEY)
if uris_json is not None:
uris = json.loads(uris_json.decode('utf-8'))
else:
response = requests.get(
"https://api.runkeeper.com/user/",
headers=self._apiHeaders(serviceRecord))
if response.status_code != 200:
if response.status_code == 401 or response.status_code == 403:
raise APIException(
"No authorization to retrieve user URLs",
block=True,
user_exception=UserException(
UserExceptionType.Authorization,
intervention_required=True))
raise APIException("Unable to retrieve user URLs" +
str(response))
uris = response.json()
for k in uris.keys():
if type(uris[k]) == str:
uris[k] = "https://api.runkeeper.com" + uris[k]
# Runkeeper wants you to request these on a per-user basis.
# In practice, the URIs are identical for ever user (only the userID key changes).
# So, only do it once every 24 hours, across the entire system.
redis.setex(self._URI_CACHE_KEY, json.dumps(uris),
timedelta(hours=24))
self._uris = uris
return uris
def _rate_limit(self, endpoint, req_lambda):
if redis.get(self._RATE_LIMIT_KEY) is not None:
raise APIException("RK global rate limit previously reached on %s" % endpoint, user_exception=UserException(UserExceptionType.RateLimited))
response = req_lambda()
if response.status_code == 429:
if "user" not in response.text:
# When we hit a limit we preemptively fail all future requests till we're sure
# than the limit is expired. The maximum period appears to be 1 day.
# This entire thing is an excercise in better-safe-than-sorry as it's unclear
# how their rate-limit logic works (when limits reset, etc).
# As it turns out, there are several parallel rate limits operating at once.
# Attempt to parse out how long we should wait - if we can't figure it out,
# default to the shortest time I've seen (15m). As long as the timer doesn't reset
# every time you request during the over-quota period, this should work.
timeout = timedelta(minutes=15)
timeout_match = re.search(r"(\d+) (second|minute|hour|day)", response.text)
if timeout_match:
# This line is too clever for its own good.
timeout = timedelta(**{"%ss" % timeout_match.group(2): float(timeout_match.group(1))})
redis.setex(self._RATE_LIMIT_KEY, response.text, timeout)
raise APIException("RK global rate limit reached on %s" % endpoint, user_exception=UserException(UserExceptionType.RateLimited))
else:
# Per-user limit hit: don't halt entire system, just bail for this user
# If a user has too many pages of activities, they will never sync as we'll keep hitting the limit.
# But that's a Very Hard Problem to Solve™ given that I can't do incremental listings...
raise APIException("RK user rate limit reached on %s" % endpoint, user_exception=UserException(UserExceptionType.RateLimited))
return response
def GenerateUserAuthorizationURL(self, level=None):
full = level == "full"
if full:
sess = session.DropboxSession(DROPBOX_FULL_APP_KEY, DROPBOX_FULL_APP_SECRET, "dropbox")
else:
sess = session.DropboxSession(DROPBOX_APP_KEY, DROPBOX_APP_SECRET, "app_folder")
reqToken = sess.obtain_request_token()
redis.setex("dropbox:oauth:%s" % reqToken.key, pickle.dumps(reqToken), timedelta(hours=24))
return sess.build_authorize_url(reqToken, oauth_callback=WEB_ROOT + reverse("oauth_return", kwargs={"service": "dropbox", "level": "full" if full else "normal"}))
def GenerateUserAuthorizationURL(self, level=None):
full = level == "full"
if full:
sess = session.DropboxSession(DROPBOX_FULL_APP_KEY, DROPBOX_FULL_APP_SECRET, "dropbox")
else:
sess = session.DropboxSession(DROPBOX_APP_KEY, DROPBOX_APP_SECRET, "app_folder")
reqToken = sess.obtain_request_token()
redis.setex("dropbox:oauth:%s" % reqToken.key, pickle.dumps(reqToken), timedelta(hours=24))
return sess.build_authorize_url(reqToken, oauth_callback=WEB_ROOT + reverse("oauth_return", kwargs={"service": "dropbox", "level": "full" if full else "normal"}))
def GenerateUserAuthorizationURL(self, session, level=None):
oauthSession = self._oauthSession(
callback_uri=WEB_ROOT +
reverse("oauth_return", kwargs={"service": "endomondo"}))
tokens = oauthSession.fetch_request_token(
"https://api.endomondo.com/oauth/request_token")
redis_token_key = 'endomondo:oauth:%s' % tokens["oauth_token"]
redis.setex(redis_token_key, tokens["oauth_token_secret"],
timedelta(hours=24))
return oauthSession.authorization_url(
"https://www.endomondo.com/oauth/authorize")
def _rate_limit(self, endpoint, req_lambda):
if redis.get(self._RATE_LIMIT_KEY % endpoint) is not None:
raise APIException("RK global rate limit previously reached on %s" % endpoint, user_exception=UserException(UserExceptionType.RateLimited))
response = req_lambda()
if response.status_code == 429:
# When we hit a limit we preemptively fail all future requests till we're sure
# than the limit is expired. The maximum period appears to be 1 day.
# This entire thing is an excercise in better-safe-than-sorry as it's unclear
# how their rate-limit logic works (when limits reset, etc).
redis.setex(self._RATE_LIMIT_KEY % endpoint, response.text, timedelta(hours=24))
raise APIException("RK global rate limit reached on %s" % endpoint, user_exception=UserException(UserExceptionType.RateLimited))
return response
def _rate_limit(self, endpoint, req_lambda):
if redis.get(self._RATE_LIMIT_KEY) is not None:
raise APIException(
"RK global rate limit previously reached on %s" % endpoint,
user_exception=UserException(UserExceptionType.RateLimited))
response = req_lambda()
if response.status_code == 429:
if "user" not in response.text:
# When we hit a limit we preemptively fail all future requests till we're sure
# than the limit is expired. The maximum period appears to be 1 day.
# This entire thing is an excercise in better-safe-than-sorry as it's unclear
# how their rate-limit logic works (when limits reset, etc).
# As it turns out, there are several parallel rate limits operating at once.
# Attempt to parse out how long we should wait - if we can't figure it out,
# default to the shortest time I've seen (15m). As long as the timer doesn't reset
# every time you request during the over-quota period, this should work.
timeout = timedelta(minutes=15)
timeout_match = re.search(r"(\d+) (second|minute|hour|day)",
response.text)
if timeout_match:
# This line is too clever for its own good.
timeout = timedelta(
**{
"%ss" % timeout_match.group(2):
float(timeout_match.group(1))
})
redis.setex(self._RATE_LIMIT_KEY, response.text, timeout)
raise APIException("RK global rate limit reached on %s" %
endpoint,
user_exception=UserException(
UserExceptionType.RateLimited))
else:
# Per-user limit hit: don't halt entire system, just bail for this user
# If a user has too many pages of activities, they will never sync as we'll keep hitting the limit.
# But that's a Very Hard Problem to Solve™ given that I can't do incremental listings...
raise APIException("RK user rate limit reached on %s" %
endpoint,
user_exception=UserException(
UserExceptionType.RateLimited))
return response
def GenerateUserAuthorizationURL(self, level=None):
oauthSession = self._oauthSession(callback_uri=WEB_ROOT + reverse("oauth_return", kwargs={"service": "endomondo"}))
tokens = oauthSession.fetch_request_token("https://api.endomondo.com/oauth/request_token")
redis_token_key = 'endomondo:oauth:%s' % tokens["oauth_token"]
redis.setex(redis_token_key, tokens["oauth_token_secret"], timedelta(hours=24))
return oauthSession.authorization_url("https://www.endomondo.com/oauth/authorize")
def Set(self, pk, value, lifetime=None):
lifetime = lifetime or self._lifetime
redis.setex(self._cacheKey % pk, pickle.dumps(value), lifetime)
def Set(self, pk, value, lifetime=None):
lifetime = lifetime or self._lifetime
redis.setex(self._cacheKey % pk, pickle.dumps(value), lifetime)
def Set(self, pk, value): redis.setex(self._cacheKey % pk, pickle.dumps(value), self._lifetime)
def Set(self, pk, value):
redis.setex(self._cacheKey % pk, pickle.dumps(value), self._lifetime)
