def test_to_age_barely_works(): now = utcnow() actual = to_age(now, dt_now=now) assert actual == "in just a moment" wait = timedelta(seconds=0.5) actual = to_age(now - wait, dt_now=now) assert actual == "just a moment ago"
def check_ratelimit_headers(self, response): """Emit log messages if we're running out of ratelimit. """ prefix = getattr(self, 'ratelimit_headers_prefix', None) if prefix: limit = response.headers.get(prefix + 'limit') remaining = response.headers.get(prefix + 'remaining') reset = response.headers.get(prefix + 'reset') try: limit, remaining, reset = int(limit), int(remaining), int( reset) except (TypeError, ValueError): limit, remaining, reset = None, None, None if None in (limit, remaining, reset): d = dict(limit=limit, remaining=remaining, reset=reset) log('Got weird rate headers from %s: %s' % (self.name, d)) else: percent_remaining = remaining / limit if percent_remaining < 0.5: reset = to_age(datetime.fromtimestamp(reset, tz=utc)) log_msg = ( '{0} API: {1:.1%} of ratelimit has been consumed, ' '{2} requests remaining, resets {3}.').format( self.name, 1 - percent_remaining, remaining, reset) log_lvl = logging.WARNING if percent_remaining < 0.2: log_lvl = logging.ERROR elif percent_remaining < 0.05: log_lvl = logging.CRITICAL log(log_msg, log_lvl)
def check_ratelimit_headers(self, response): """Emit log messages if we're running out of ratelimit. """ prefix = getattr(self, 'ratelimit_headers_prefix', None) if prefix: limit = response.headers.get(prefix+'limit') remaining = response.headers.get(prefix+'remaining') reset = response.headers.get(prefix+'reset') try: limit, remaining, reset = int(limit), int(remaining), int(reset) except (TypeError, ValueError): limit, remaining, reset = None, None, None if None in (limit, remaining, reset): d = dict(limit=limit, remaining=remaining, reset=reset) log('Got weird rate headers from %s: %s' % (self.name, d)) else: percent_remaining = remaining/limit if percent_remaining < 0.5: reset = to_age(datetime.fromtimestamp(reset, tz=utc)) log_msg = ( '{0} API: {1:.1%} of ratelimit has been consumed, ' '{2} requests remaining, resets {3}.' ).format(self.name, 1 - percent_remaining, remaining, reset) log_lvl = logging.WARNING if percent_remaining < 0.2: log_lvl = logging.ERROR elif percent_remaining < 0.05: log_lvl = logging.CRITICAL log(log_msg, log_lvl)
def get_user_info(screen_name): """Given a unicode, return a dict. """ typecheck(screen_name, unicode) try: rec = gittip.db.one( "SELECT user_info FROM elsewhere " "WHERE platform='twitter' " "AND user_info->'screen_name' = %s" , (screen_name,) ) except TooFew: rec = None if rec is not None: user_info = rec['user_info'] else: # Updated using Twython as a point of reference: # https://github.com/ryanmcgrath/twython/blob/master/twython/twython.py#L76 oauth = OAuth1( # we do not have access to the website obj, # so let's grab the details from the env environ['TWITTER_CONSUMER_KEY'], environ['TWITTER_CONSUMER_SECRET'], environ['TWITTER_ACCESS_TOKEN'], environ['TWITTER_ACCESS_TOKEN_SECRET'], ) url = "https://api.twitter.com/1.1/users/show.json?screen_name=%s" user_info = requests.get(url % screen_name, auth=oauth) # Keep an eye on our Twitter usage. # ================================= rate_limit = user_info.headers['X-Rate-Limit-Limit'] rate_limit_remaining = user_info.headers['X-Rate-Limit-Remaining'] rate_limit_reset = user_info.headers['X-Rate-Limit-Reset'] try: rate_limit = int(rate_limit) rate_limit_remaining = int(rate_limit_remaining) rate_limit_reset = int(rate_limit_reset) except (TypeError, ValueError): log( "Got weird rate headers from Twitter: %s %s %s" % (rate_limit, rate_limit_remaining, rate_limit_reset) ) else: reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) reset = to_age(reset) log( "Twitter API calls used: %d / %d. Resets %s." % (rate_limit - rate_limit_remaining, rate_limit, reset) ) if user_info.status_code == 200: user_info = json.loads(user_info.text) else: log("Twitter lookup failed with %d." % user_info.status_code) raise Response(404) return user_info
def get_user_info(screen_name): """Given a unicode, return a dict. """ typecheck(screen_name, (unicode, UnicodeWithParams)) rec = gittip.db.one( "SELECT user_info FROM elsewhere " "WHERE platform='twitter' " "AND user_info->'screen_name' = %s" , (screen_name,) ) if rec is not None: user_info = rec else: # Updated using Twython as a point of reference: # https://github.com/ryanmcgrath/twython/blob/master/twython/twython.py#L76 oauth = OAuth1( # we do not have access to the website obj, # so let's grab the details from the env environ['TWITTER_CONSUMER_KEY'], environ['TWITTER_CONSUMER_SECRET'], environ['TWITTER_ACCESS_TOKEN'], environ['TWITTER_ACCESS_TOKEN_SECRET'], ) url = "https://api.twitter.com/1.1/users/show.json?screen_name=%s" user_info = requests.get(url % screen_name, auth=oauth) # Keep an eye on our Twitter usage. # ================================= rate_limit = user_info.headers['X-Rate-Limit-Limit'] rate_limit_remaining = user_info.headers['X-Rate-Limit-Remaining'] rate_limit_reset = user_info.headers['X-Rate-Limit-Reset'] try: rate_limit = int(rate_limit) rate_limit_remaining = int(rate_limit_remaining) rate_limit_reset = int(rate_limit_reset) except (TypeError, ValueError): log( "Got weird rate headers from Twitter: %s %s %s" % (rate_limit, rate_limit_remaining, rate_limit_reset) ) else: reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) reset = to_age(reset) log( "Twitter API calls used: %d / %d. Resets %s." % (rate_limit - rate_limit_remaining, rate_limit, reset) ) if user_info.status_code == 200: user_info = json.loads(user_info.text) else: log("Twitter lookup failed with %d." % user_info.status_code) raise Response(404) return user_info
def msg(_, to_age): if remaining == 0 and reset: return _( "You've consumed your quota of requests, you can try again in {0}.", to_age(reset)) else: return _( "You're making requests too fast, please try again later." )
def get_user_info(screen_name): """Given a unicode, return a dict. """ typecheck(screen_name, unicode) rec = gittip.db.fetchone( "SELECT user_info FROM elsewhere " "WHERE platform='twitter' " "AND user_info->'screen_name' = %s" , (screen_name,) ) if rec is not None: user_info = rec['user_info'] else: oauth = OAuthHook( # we haven't got access to the website obj, # so let's grab the details from the env access_token=environ['TWITTER_ACCESS_TOKEN'], access_token_secret=environ['TWITTER_ACCESS_TOKEN_SECRET'], consumer_key=environ['TWITTER_CONSUMER_KEY'], consumer_secret=environ['TWITTER_CONSUMER_SECRET'], header_auth=True ) url = "https://api.twitter.com/1.1/users/show.json?screen_name=%s" user_info = requests.get(url % screen_name, hooks={'pre_request': oauth}) # Keep an eye on our Twitter usage. # ================================= rate_limit = user_info.headers['X-RateLimit-Limit'] rate_limit_remaining = user_info.headers['X-RateLimit-Remaining'] rate_limit_reset = user_info.headers['X-RateLimit-Reset'] try: rate_limit = int(rate_limit) rate_limit_remaining = int(rate_limit_remaining) rate_limit_reset = int(rate_limit_reset) except (TypeError, ValueError): log( "Got weird rate headers from Twitter: %s %s %s" % (rate_limit, rate_limit_remaining, rate_limit_reset) ) else: reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) reset = to_age(reset) log( "Twitter API calls used: %d / %d. Resets %s." % (rate_limit - rate_limit_remaining, rate_limit, reset) ) if user_info.status_code == 200: user_info = json.loads(user_info.text) else: log("Twitter lookup failed with %d." % user_info.status_code) raise Response(404) return user_info
def _to_age(participant): # XXX I can't believe I'm doing this. Evolve aspen.utils.to_age! age = to_age(participant.claimed_time, fmt_past="%(age)s") age = age.replace('just a moment', 'just now') age = age.replace('an ', '1 ').replace('a ', '1 ') if age.endswith(' seconds'): age = '1 minute' words = ('zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine') for i, word in enumerate(words): age = age.replace(word, str(i)) return age.replace(' ', ' <span class="unit">') + "</span>"
def _to_age(participant): # XXX I can't believe I'm doing this. Evolve aspen.utils.to_age! age = to_age(participant.claimed_time, fmt_past="%(age)s") age = age.replace('just a moment', 'just now') age = age.replace('an ', '1 ').replace('a ', '1 ') if age.endswith(' seconds'): age = '1 minute' words = ('zero', 'one', 'two','three', 'four', 'five', 'six', 'seven', 'eight', 'nine') for i, word in enumerate(words): age = age.replace(word, str(i)) return age.replace(' ', ' <span class="unit">') + "</span>"
def api_get(self, path, sess=None, **kw): """ Given a `path` (e.g. /users/foo), this function sends a GET request to the platform's API (e.g. https://api.github.com/users/foo). The response is returned, after checking its status code and ratelimit headers. """ if not sess: sess = self.get_auth_session() response = sess.get(self.api_url + path, **kw) # Check status status = response.status_code if status == 404: raise Response(404) elif status != 200: log('{} api responded with {}:\n{}'.format(self.name, status, response.text), level=logging.ERROR) raise Response( 500, '{} lookup failed with {}'.format(self.name, status)) # Check ratelimit headers prefix = getattr(self, 'ratelimit_headers_prefix', None) if prefix: limit = response.headers[prefix + 'limit'] remaining = response.headers[prefix + 'remaining'] reset = response.headers[prefix + 'reset'] try: limit, remaining, reset = int(limit), int(remaining), int( reset) except (TypeError, ValueError): d = dict(limit=limit, remaining=remaining, reset=reset) log('Got weird rate headers from %s: %s' % (self.name, d)) else: percent_remaining = remaining / limit if percent_remaining < 0.5: reset = to_age(datetime.fromtimestamp(reset, tz=utc)) log_msg = ( '{0} API: {1:.1%} of ratelimit has been consumed, ' '{2} requests remaining, resets {3}.').format( self.name, 1 - percent_remaining, remaining, reset) log_lvl = logging.WARNING if percent_remaining < 0.2: log_lvl = logging.ERROR elif percent_remaining < 0.05: log_lvl = logging.CRITICAL log(log_msg, log_lvl) return response
def get_user_info(screen_name): """Given a unicode, return a dict. """ typecheck(screen_name, unicode) rec = gittip.db.fetchone( "SELECT user_info FROM elsewhere " "WHERE platform='twitter' " "AND user_info->'screen_name' = %s", (screen_name, )) if rec is not None: user_info = rec['user_info'] else: oauth = OAuthHook( # we haven't got access to the website obj, # so let's grab the details from the env access_token=environ['TWITTER_ACCESS_TOKEN'], access_token_secret=environ['TWITTER_ACCESS_TOKEN_SECRET'], consumer_key=environ['TWITTER_CONSUMER_KEY'], consumer_secret=environ['TWITTER_CONSUMER_SECRET'], header_auth=True) url = "https://api.twitter.com/1.1/users/show.json?screen_name=%s" user_info = requests.get(url % screen_name, hooks={'pre_request': oauth}) # Keep an eye on our Twitter usage. # ================================= rate_limit = user_info.headers['X-RateLimit-Limit'] rate_limit_remaining = user_info.headers['X-RateLimit-Remaining'] rate_limit_reset = user_info.headers['X-RateLimit-Reset'] try: rate_limit = int(rate_limit) rate_limit_remaining = int(rate_limit_remaining) rate_limit_reset = int(rate_limit_reset) except (TypeError, ValueError): log("Got weird rate headers from Twitter: %s %s %s" % (rate_limit, rate_limit_remaining, rate_limit_reset)) else: reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) reset = to_age(reset) log("Twitter API calls used: %d / %d. Resets %s." % (rate_limit - rate_limit_remaining, rate_limit, reset)) if user_info.status_code == 200: user_info = json.loads(user_info.text) else: log("Twitter lookup failed with %d." % user_info.status_code) raise Response(404) return user_info
def api_get(self, path, sess=None, **kw): """ Given a `path` (e.g. /users/foo), this function sends a GET request to the platform's API (e.g. https://api.github.com/users/foo). The response is returned, after checking its status code and ratelimit headers. """ if not sess: sess = self.get_auth_session() response = sess.get(self.api_url+path, **kw) # Check status status = response.status_code if status == 404: raise Response(404) elif status != 200: log('{} api responded with {}:\n{}'.format(self.name, status, response.text) , level=logging.ERROR) raise Response(500, '{} lookup failed with {}'.format(self.name, status)) # Check ratelimit headers prefix = getattr(self, 'ratelimit_headers_prefix', None) if prefix: limit = response.headers[prefix+'limit'] remaining = response.headers[prefix+'remaining'] reset = response.headers[prefix+'reset'] try: limit, remaining, reset = int(limit), int(remaining), int(reset) except (TypeError, ValueError): d = dict(limit=limit, remaining=remaining, reset=reset) log('Got weird rate headers from %s: %s' % (self.name, d)) else: percent_remaining = remaining/limit if percent_remaining < 0.5: reset = to_age(datetime.fromtimestamp(reset, tz=utc)) log_msg = ( '{0} API: {1:.1%} of ratelimit has been consumed, ' '{2} requests remaining, resets {3}.' ).format(self.name, 1 - percent_remaining, remaining, reset) log_lvl = logging.WARNING if percent_remaining < 0.2: log_lvl = logging.ERROR elif percent_remaining < 0.05: log_lvl = logging.CRITICAL log(log_msg, log_lvl) return response
def log_ratelimit_headers(self, limit, remaining, reset): """Emit log messages if we're running out of ratelimit. """ if None in (limit, remaining, reset): return percent_remaining = remaining / limit if percent_remaining < 0.5: log_msg = ( "{0} API: {1:.1%} of ratelimit has been consumed, " "{2} requests remaining, resets {3}." ).format(self.name, 1 - percent_remaining, remaining, to_age(reset)) log_lvl = logging.WARNING if percent_remaining < 0.2: log_lvl = logging.ERROR elif percent_remaining < 0.05: log_lvl = logging.CRITICAL log(log_msg, log_lvl)
def get_user_info(screen_name): """Given a unicode, return a dict. """ typecheck(screen_name, unicode) rec = gittip.db.fetchone( "SELECT user_info FROM elsewhere " "WHERE platform='twitter' " "AND user_info->'screen_name' = %s" , (screen_name,) ) if rec is not None: user_info = rec['user_info'] else: url = "https://api.twitter.com/1/users/show.json?screen_name=%s" user_info = requests.get(url % screen_name) # Keep an eye on our Twitter usage. # ================================= rate_limit = user_info.headers['X-RateLimit-Limit'] rate_limit_remaining = user_info.headers['X-RateLimit-Remaining'] rate_limit_reset = user_info.headers['X-RateLimit-Reset'] try: rate_limit = int(rate_limit) rate_limit_remaining = int(rate_limit_remaining) rate_limit_reset = int(rate_limit_reset) except (TypeError, ValueError): log( "Got weird rate headers from Twitter: %s %s %s" % (rate_limit, rate_limit_remaining, rate_limit_reset) ) else: reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) reset = to_age(reset) log( "Twitter API calls used: %d / %d. Resets %s." % (rate_limit - rate_limit_remaining, rate_limit, reset) ) if user_info.status_code == 200: user_info = json.loads(user_info.text) else: log("Twitter lookup failed with %d." % user_info.status_code) raise Response(404) return user_info
def log_ratelimit_headers(self, limit, remaining, reset): """Emit log messages if we're running out of ratelimit. """ if None in (limit, remaining, reset): return percent_remaining = remaining / limit if percent_remaining < 0.5: log_msg = ('{0} API: {1:.1%} of ratelimit has been consumed, ' '{2} requests remaining, resets {3}.').format( self.name, 1 - percent_remaining, remaining, to_age(reset)) log_lvl = logging.WARNING if percent_remaining < 0.2: log_lvl = logging.ERROR elif percent_remaining < 0.05: log_lvl = logging.CRITICAL log(log_msg, log_lvl)
def get_user_info(screen_name): """Given a unicode, return a dict. """ typecheck(screen_name, unicode) rec = gittip.db.fetchone( "SELECT user_info FROM elsewhere " "WHERE platform='twitter' " "AND user_info->'screen_name' = %s", (screen_name, )) if rec is not None: user_info = rec['user_info'] else: url = "https://api.twitter.com/1/users/show.json?screen_name=%s" user_info = requests.get(url % screen_name) # Keep an eye on our Twitter usage. # ================================= rate_limit = user_info.headers['X-RateLimit-Limit'] rate_limit_remaining = user_info.headers['X-RateLimit-Remaining'] rate_limit_reset = user_info.headers['X-RateLimit-Reset'] try: rate_limit = int(rate_limit) rate_limit_remaining = int(rate_limit_remaining) rate_limit_reset = int(rate_limit_reset) except (TypeError, ValueError): log("Got weird rate headers from Twitter: %s %s %s" % (rate_limit, rate_limit_remaining, rate_limit_reset)) else: reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) reset = to_age(reset) log("Twitter API calls used: %d / %d. Resets %s." % (rate_limit - rate_limit_remaining, rate_limit, reset)) if user_info.status_code == 200: user_info = json.loads(user_info.text) else: log("Twitter lookup failed with %d." % user_info.status_code) raise Response(404) return user_info
def msg(_, to_age): if remaining == 0 and reset: return _("You've consumed your quota of requests, you can try again in {0}.", to_age(reset)) else: return _("You're making requests too fast, please try again later.")
def test_to_age_formatting_works(): now = utcnow() actual = to_age(now, fmt_future="Cheese, for %(age)s!", dt_now=now) assert actual == "Cheese, for just a moment!"
def test_to_age_barely_works(): actual = to_age(utcnow()) assert actual == "just a moment ago", actual
def test_to_age_formatting_works(): actual = to_age(utcnow(), fmt_past="Cheese, for {age}!") assert actual == "Cheese, for just a moment!", actual
def test_to_age_formatting_works(): actual = to_age(utcnow(), fmt_past="Cheese, for %(age)s!") assert actual == "Cheese, for just a moment!"
def test_to_age_barely_works(): actual = to_age(utcnow()) assert actual == "just a moment ago"