def _walk_api(full_url, db_config, options): """ Call the github api at full_url, and continue to aggregate json results as long as a "next" link exists. Errors will raise an exception. """ all_jsons = [] resp = _raw_api(full_url, db_config, options) if not resp.ok: # for now, just bomb out totally, on the theory that we don't # want to deal with stale caches etc. LATER in the future, # make this a little more robust. err = "Github API error: %s" % resp.text log.error(err) raise RuntimeError(err) all_jsons.append(resp.text) # walk through any "next" links if "link" in resp.headers: links = [h.strip() for h in resp.headers["link"].split(",")] for link in links: pieces = [piece.strip() for piece in link.split(";")] if pieces[1] == 'rel="next"': next_url = pieces[0].strip("<>") log.verbose("Found a next link %s, following it." % next_url, options) all_jsons.extend(_walk_api(next_url, db_config, options)) return all_jsons
def _raw_api(full_url, db_config, options): """ Make a single api call at full_url to the github api, and return the json found there. """ log.verbose("Hitting github url: %s" % full_url, options) return requests.get(full_url, auth=(db_config["api_user"], db_config["api_password"]))
def _all_issue_text(gh_issue, db_config, options): """ Extract all the issue text, meaning the body and the bodies of the comments if there are such. """ txts = [gh_issue["body"]] if gh_issue.get("comments", 0): log.verbose("Github issue %s has comments" % gh_issue["number"], options) txts.extend(_all_comment_bodies(gh_issue, db_config, options)) return "\n\n".join(txts)
def _slurp(cache_shelf, db_config, since, gh_now, options): """ Slurp down the issues, partially or fully, from a single github repo. """ if options["refresh"] == "full": since = None log.verbose("Grabbing all github issues due to refresh=full", options) log.verbose("Slurping github issues for %s since %s" % (db_config["repo"], since if since else "always"), options) issues = _issues(db_config, since, options=options) _record_issues(issues=issues, cache_shelf=cache_shelf, gh_now=gh_now, db_config=db_config, options=options)
def open(self): """ Open the bug db, refreshing from remote dbs as specified by the refresh option. """ self.cache_shelf = _open_cache_shelf(self.cache_fname) if not self.cache_shelf.get("version", None): # new shelf self.cache_shelf["version"] = VERSION if int(self.cache_shelf["version"]) != int(VERSION): raise RuntimeError("Wrong shelf version %d (code is %d" % (int(self.cache_shelf["version"]), int(VERSION))) if self.options["refresh"] == "none": log.verbose("Not refreshing any bug dbs due to refresh=none", self.options) else: self._refresh_from_remotes()