def _fetch_api_url(self, url):
# fetch the url and parse to json
'''
jdata = None
try:
resp = self.instance._requester.requestJson(
'GET',
url
)
data = resp[2]
jdata = json.loads(data)
except Exception as e:
logging.error(e)
'''
jdata = None
while True:
resp = self.instance._requester.requestJson(
'GET',
url
)
data = resp[2]
jdata = json.loads(data)
if isinstance(jdata, dict) and jdata.get('documentation_url'):
if C.DEFAULT_BREAKPOINTS:
import epdb; epdb.st()
else:
raise RateLimitError("rate limited")
else:
break
return jdata
def get_request(self, url):
'''Get an arbitrary API endpoint'''
headers = {
u'Accept': u','.join(self.accepts_headers),
u'Authorization': u'Bearer %s' % self.token,
}
rr = requests.get(url, headers=headers)
data = rr.json()
# handle ratelimits ...
if isinstance(data, dict) and data.get(u'message'):
if data[u'message'].lower().startswith(u'api rate limit exceeded'):
raise RateLimitError()
# pagination
if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
_data = self.get_request(rr.links[u'next'][u'url'])
try:
if isinstance(data, list):
data += _data
elif isinstance(data, dict):
data.update(_data)
except TypeError as e:
if C.DEFAULT_BREAKPOINTS:
logging.error(u'breakpoint!')
import epdb
epdb.st()
else:
raise Exception(e)
return data
def get_request(self, url):
'''Get an arbitrary API endpoint'''
accepts = [
u'application/json',
u'application/vnd.github.mockingbird-preview',
u'application/vnd.github.sailor-v-preview+json',
u'application/vnd.github.starfox-preview+json',
u'application/vnd.github.squirrel-girl-preview',
u'application/vnd.github.v3+json',
]
headers = {
u'Accept': u','.join(accepts),
u'Authorization': u'Bearer %s' % self.token,
}
rr = requests.get(url, headers=headers)
data = rr.json()
# handle ratelimits ...
if isinstance(data, dict) and data.get(u'message'):
if data[u'message'].lower().startswith(u'api rate limit exceeded'):
raise RateLimitError()
if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
_data = self.get_request(rr.links[u'next'][u'url'])
data += _data
return data
def get_cached_request(self, url):
'''Use a combination of sqlite and ondisk caching to GET an api resource'''
url_parts = url.split('/')
cdf = os.path.join(self.cached_requests_dir,
url.replace('https://', '') + '.json.gz')
cdd = os.path.dirname(cdf)
if not os.path.exists(cdd):
os.makedirs(cdd)
# FIXME - commits are static and can always be used from cache.
if url_parts[-2] == 'commits' and os.path.exists(cdf):
return read_gzip_json_file(cdf)
headers = {
u'Accept': u','.join(self.accepts_headers),
u'Authorization': u'Bearer %s' % self.token,
}
meta = ADB.get_github_api_request_meta(url, token=self.token)
if meta is None:
meta = {}
# https://developer.github.com/v3/#conditional-requests
etag = meta.get('etag')
if etag and os.path.exists(cdf):
headers['If-None-Match'] = etag
rr = requests.get(url, headers=headers)
if rr.status_code == 304:
# not modified
with open(cdf, 'r') as f:
data = json.loads(f.read())
else:
data = rr.json()
# handle ratelimits ...
if isinstance(data, dict) and data.get(u'message'):
if data[u'message'].lower().startswith(
u'api rate limit exceeded'):
raise RateLimitError()
# cache data to disk
logging.debug('write %s' % cdf)
write_gzip_json_file(cdf, data)
# save the meta
ADB.set_github_api_request_meta(url, rr.headers, cdf, token=self.token)
# pagination
if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
_data = self.get_request(rr.links[u'next'][u'url'])
if isinstance(data, list):
data += _data
else:
data.update(_data)
return data
def get_reviews(self):
# https://developer.github.com/
# early-access/graphql/enum/pullrequestreviewstate/
# https://developer.github.com/v3/
# pulls/reviews/#list-reviews-on-a-pull-request
reviews_url = self.pullrequest.url + u'/reviews'
headers = {
u'Accept': u'application/vnd.github.black-cat-preview+json',
}
status, hdrs, body = self.instance._requester.requestJson(
u'GET', reviews_url, headers=headers)
jdata = json.loads(body)
# need to catch rate limit message here
if isinstance(jdata, dict) and u'rate' in jdata[u'message']:
raise RateLimitError("rate limited")
if isinstance(jdata, dict):
logging.error(
u'get_reviews | pr_reviews.keys=%s | pr_reviews.len=%s | '
u'resp.headers=%s | resp.status=%s',
jdata.keys(),
len(jdata),
hdrs,
status,
)
return jdata
def get_request(self, url):
'''Get an arbitrary API endpoint'''
headers = {
'Accept': ','.join(HEADERS),
'Authorization': 'Bearer %s' % self.token,
}
rr = requests.get(url, headers=headers)
data = rr.json()
# handle ratelimits ...
if isinstance(data, dict) and data.get('message'):
if data['message'].lower().startswith('api rate limit exceeded'):
raise RateLimitError()
# pagination
if hasattr(rr, 'links') and rr.links and rr.links.get('next'):
_data = self.get_request(rr.links['next']['url'])
if isinstance(data, list):
data += _data
elif isinstance(data, dict):
data.update(_data)
return data
def get_reviews(self):
# https://developer.github.com/
# early-access/graphql/enum/pullrequestreviewstate/
# https://developer.github.com/v3/
# pulls/reviews/#list-reviews-on-a-pull-request
reviews_url = self.pullrequest.url + u'/reviews'
headers = {
u'Accept': u'application/vnd.github.black-cat-preview+json',
}
status, hdrs, body = self.instance._requester.requestJson(
u'GET', reviews_url, headers=headers)
jdata = json.loads(body)
if isinstance(jdata, dict):
logging.error(
u'get_reviews | pr_reviews.keys=%s | pr_reviews.len=%s | '
u'resp.headers=%s | resp.status=%s',
jdata.keys(),
len(jdata),
hdrs,
status,
)
is_rate_limited = u'rate' in jdata[u'message']
is_server_error = (u'Server Error' == jdata[u'message']
or 500 <= status < 600)
if is_rate_limited:
raise RateLimitError("rate limited")
if is_server_error:
raise RateLimitError("server error")
raise RateLimitError("unknown error: GH responded with a dict "
"while a list of reviews was expected")
return jdata
def get_reviews(self):
# https://developer.github.com/
# early-access/graphql/enum/pullrequestreviewstate/
# https://developer.github.com/v3/
# pulls/reviews/#list-reviews-on-a-pull-request
reviews_url = self.pullrequest.url + '/reviews'
headers = dict(
Accept='application/vnd.github.black-cat-preview+json', )
resp = self.instance._requester.requestJson('GET',
reviews_url,
headers=headers)
jdata = json.loads(resp[2])
# need to catch rate limit message here
if isinstance(jdata, dict) and 'rate' in jdata:
raise RateLimitError("rate limited")
return jdata
def get_request(self, url):
'''Get an arbitrary API endpoint'''
headers = {
u'Accept': u','.join(self.accepts_headers),
u'Authorization': u'Bearer %s' % self.token,
}
rr = requests.get(url, headers=headers)
data = rr.json()
# handle ratelimits ...
if isinstance(data, dict) and data.get(u'message'):
if data[u'message'].lower().startswith(u'api rate limit exceeded'):
raise RateLimitError()
# pagination
if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
_data = self.get_request(rr.links[u'next'][u'url'])
data += _data
return data