def _fetch_api_url(self, url): # fetch the url and parse to json ''' jdata = None try: resp = self.instance._requester.requestJson( 'GET', url ) data = resp[2] jdata = json.loads(data) except Exception as e: logging.error(e) ''' jdata = None while True: resp = self.instance._requester.requestJson( 'GET', url ) data = resp[2] jdata = json.loads(data) if isinstance(jdata, dict) and jdata.get('documentation_url'): if C.DEFAULT_BREAKPOINTS: import epdb; epdb.st() else: raise RateLimitError("rate limited") else: break return jdata
def get_request(self, url): '''Get an arbitrary API endpoint''' headers = { u'Accept': u','.join(self.accepts_headers), u'Authorization': u'Bearer %s' % self.token, } rr = requests.get(url, headers=headers) data = rr.json() # handle ratelimits ... if isinstance(data, dict) and data.get(u'message'): if data[u'message'].lower().startswith(u'api rate limit exceeded'): raise RateLimitError() # pagination if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'): _data = self.get_request(rr.links[u'next'][u'url']) try: if isinstance(data, list): data += _data elif isinstance(data, dict): data.update(_data) except TypeError as e: if C.DEFAULT_BREAKPOINTS: logging.error(u'breakpoint!') import epdb epdb.st() else: raise Exception(e) return data
def get_request(self, url): '''Get an arbitrary API endpoint''' accepts = [ u'application/json', u'application/vnd.github.mockingbird-preview', u'application/vnd.github.sailor-v-preview+json', u'application/vnd.github.starfox-preview+json', u'application/vnd.github.squirrel-girl-preview', u'application/vnd.github.v3+json', ] headers = { u'Accept': u','.join(accepts), u'Authorization': u'Bearer %s' % self.token, } rr = requests.get(url, headers=headers) data = rr.json() # handle ratelimits ... if isinstance(data, dict) and data.get(u'message'): if data[u'message'].lower().startswith(u'api rate limit exceeded'): raise RateLimitError() if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'): _data = self.get_request(rr.links[u'next'][u'url']) data += _data return data
def get_cached_request(self, url): '''Use a combination of sqlite and ondisk caching to GET an api resource''' url_parts = url.split('/') cdf = os.path.join(self.cached_requests_dir, url.replace('https://', '') + '.json.gz') cdd = os.path.dirname(cdf) if not os.path.exists(cdd): os.makedirs(cdd) # FIXME - commits are static and can always be used from cache. if url_parts[-2] == 'commits' and os.path.exists(cdf): return read_gzip_json_file(cdf) headers = { u'Accept': u','.join(self.accepts_headers), u'Authorization': u'Bearer %s' % self.token, } meta = ADB.get_github_api_request_meta(url, token=self.token) if meta is None: meta = {} # https://developer.github.com/v3/#conditional-requests etag = meta.get('etag') if etag and os.path.exists(cdf): headers['If-None-Match'] = etag rr = requests.get(url, headers=headers) if rr.status_code == 304: # not modified with open(cdf, 'r') as f: data = json.loads(f.read()) else: data = rr.json() # handle ratelimits ... if isinstance(data, dict) and data.get(u'message'): if data[u'message'].lower().startswith( u'api rate limit exceeded'): raise RateLimitError() # cache data to disk logging.debug('write %s' % cdf) write_gzip_json_file(cdf, data) # save the meta ADB.set_github_api_request_meta(url, rr.headers, cdf, token=self.token) # pagination if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'): _data = self.get_request(rr.links[u'next'][u'url']) if isinstance(data, list): data += _data else: data.update(_data) return data
def get_reviews(self): # https://developer.github.com/ # early-access/graphql/enum/pullrequestreviewstate/ # https://developer.github.com/v3/ # pulls/reviews/#list-reviews-on-a-pull-request reviews_url = self.pullrequest.url + u'/reviews' headers = { u'Accept': u'application/vnd.github.black-cat-preview+json', } status, hdrs, body = self.instance._requester.requestJson( u'GET', reviews_url, headers=headers) jdata = json.loads(body) # need to catch rate limit message here if isinstance(jdata, dict) and u'rate' in jdata[u'message']: raise RateLimitError("rate limited") if isinstance(jdata, dict): logging.error( u'get_reviews | pr_reviews.keys=%s | pr_reviews.len=%s | ' u'resp.headers=%s | resp.status=%s', jdata.keys(), len(jdata), hdrs, status, ) return jdata
def get_request(self, url): '''Get an arbitrary API endpoint''' headers = { 'Accept': ','.join(HEADERS), 'Authorization': 'Bearer %s' % self.token, } rr = requests.get(url, headers=headers) data = rr.json() # handle ratelimits ... if isinstance(data, dict) and data.get('message'): if data['message'].lower().startswith('api rate limit exceeded'): raise RateLimitError() # pagination if hasattr(rr, 'links') and rr.links and rr.links.get('next'): _data = self.get_request(rr.links['next']['url']) if isinstance(data, list): data += _data elif isinstance(data, dict): data.update(_data) return data
def get_reviews(self): # https://developer.github.com/ # early-access/graphql/enum/pullrequestreviewstate/ # https://developer.github.com/v3/ # pulls/reviews/#list-reviews-on-a-pull-request reviews_url = self.pullrequest.url + u'/reviews' headers = { u'Accept': u'application/vnd.github.black-cat-preview+json', } status, hdrs, body = self.instance._requester.requestJson( u'GET', reviews_url, headers=headers) jdata = json.loads(body) if isinstance(jdata, dict): logging.error( u'get_reviews | pr_reviews.keys=%s | pr_reviews.len=%s | ' u'resp.headers=%s | resp.status=%s', jdata.keys(), len(jdata), hdrs, status, ) is_rate_limited = u'rate' in jdata[u'message'] is_server_error = (u'Server Error' == jdata[u'message'] or 500 <= status < 600) if is_rate_limited: raise RateLimitError("rate limited") if is_server_error: raise RateLimitError("server error") raise RateLimitError("unknown error: GH responded with a dict " "while a list of reviews was expected") return jdata
def get_reviews(self): # https://developer.github.com/ # early-access/graphql/enum/pullrequestreviewstate/ # https://developer.github.com/v3/ # pulls/reviews/#list-reviews-on-a-pull-request reviews_url = self.pullrequest.url + '/reviews' headers = dict( Accept='application/vnd.github.black-cat-preview+json', ) resp = self.instance._requester.requestJson('GET', reviews_url, headers=headers) jdata = json.loads(resp[2]) # need to catch rate limit message here if isinstance(jdata, dict) and 'rate' in jdata: raise RateLimitError("rate limited") return jdata
def get_request(self, url): '''Get an arbitrary API endpoint''' headers = { u'Accept': u','.join(self.accepts_headers), u'Authorization': u'Bearer %s' % self.token, } rr = requests.get(url, headers=headers) data = rr.json() # handle ratelimits ... if isinstance(data, dict) and data.get(u'message'): if data[u'message'].lower().startswith(u'api rate limit exceeded'): raise RateLimitError() # pagination if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'): _data = self.get_request(rr.links[u'next'][u'url']) data += _data return data