def _walk_api(full_url, db_config, options):
"""
Call the github api at full_url, and continue to aggregate json
results as long as a "next" link exists.
Errors will raise an exception.
"""
all_jsons = []
resp = _raw_api(full_url, db_config, options)
if not resp.ok:
# for now, just bomb out totally, on the theory that we don't
# want to deal with stale caches etc. LATER in the future,
# make this a little more robust.
err = "Github API error: %s" % resp.text
log.error(err)
raise RuntimeError(err)
all_jsons.append(resp.text)
# walk through any "next" links
if "link" in resp.headers:
links = [h.strip() for h in resp.headers["link"].split(",")]
for link in links:
pieces = [piece.strip() for piece in link.split(";")]
if pieces[1] == 'rel="next"':
next_url = pieces[0].strip("<>")
log.verbose("Found a next link %s, following it." % next_url, options)
all_jsons.extend(_walk_api(next_url, db_config, options))
return all_jsons
def _raw_api(full_url, db_config, options):
"""
Make a single api call at full_url to the github api, and return
the json found there.
"""
log.verbose("Hitting github url: %s" % full_url, options)
return requests.get(full_url, auth=(db_config["api_user"], db_config["api_password"]))
def _all_issue_text(gh_issue, db_config, options):
"""
Extract all the issue text, meaning the body and the bodies of the
comments if there are such.
"""
txts = [gh_issue["body"]]
if gh_issue.get("comments", 0):
log.verbose("Github issue %s has comments" % gh_issue["number"], options)
txts.extend(_all_comment_bodies(gh_issue, db_config, options))
return "\n\n".join(txts)
def _slurp(cache_shelf, db_config, since, gh_now, options):
"""
Slurp down the issues, partially or fully, from a single github
repo.
"""
if options["refresh"] == "full":
since = None
log.verbose("Grabbing all github issues due to refresh=full", options)
log.verbose("Slurping github issues for %s since %s" % (db_config["repo"], since if since else "always"), options)
issues = _issues(db_config, since, options=options)
_record_issues(issues=issues, cache_shelf=cache_shelf, gh_now=gh_now, db_config=db_config, options=options)
def open(self):
"""
Open the bug db, refreshing from remote dbs as specified by
the refresh option.
"""
self.cache_shelf = _open_cache_shelf(self.cache_fname)
if not self.cache_shelf.get("version", None):
# new shelf
self.cache_shelf["version"] = VERSION
if int(self.cache_shelf["version"]) != int(VERSION):
raise RuntimeError("Wrong shelf version %d (code is %d" % (int(self.cache_shelf["version"]), int(VERSION)))
if self.options["refresh"] == "none":
log.verbose("Not refreshing any bug dbs due to refresh=none", self.options)
else:
self._refresh_from_remotes()
