def refresh(owner, repo, refresh_ref): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, owner) if not installation_id: # pragma: no cover flask.abort(400, "%s have not installed mergify_engine" % owner) token = integration.get_access_token(installation_id).token g = github.Github(token) r = g.get_repo("%s/%s" % (owner, repo)) try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: return "No .mergify.yml", 202 else: raise if refresh_ref == "full" or refresh_ref.startswith("branch/"): if refresh_ref.startswith("branch/"): branch = refresh_ref[7:] pulls = r.get_pulls(base=branch) else: branch = '*' pulls = r.get_pulls() key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(), repo.lower(), r.private, branch) utils.get_redis_for_cache().delete(key) else: try: pull_number = int(refresh_ref[5:]) except ValueError: # pragma: no cover return "Invalid PR ref", 400 pulls = [r.get_pull(pull_number)] subscription = utils.get_subscription(utils.get_redis_for_cache(), installation_id) if not subscription["token"]: # pragma: no cover return "", 202 if r.private and not subscription["subscribed"]: # pragma: no cover return "", 202 for p in pulls: # Mimic the github event format data = { 'repository': r.raw_data, 'installation': { 'id': installation_id }, 'pull_request': p.raw_data, } get_queue(r.full_name, subscription).enqueue(worker.event_handler, "refresh", subscription, data) return "", 202
def job_marketplace(event_type, event_id, data): owner = data["marketplace_purchase"]["account"]["login"] account_type = data["marketplace_purchase"]["account"]["type"] integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_id = utils.get_installation_id( integration, owner, account_type=account_type ) except github.GithubException as e: LOG.warning("%s: mergify not installed", owner, error=str(e)) installation_id = None subscription = { "subscription_active": "Unknown", "subscription_reason": "No", "tokens": None, } else: r = utils.get_redis_for_cache() r.delete("subscription-cache-%s" % installation_id) subscription = sub_utils.get_subscription(r, installation_id) LOG.info( "Marketplace event", event_type=event_type, event_id=event_id, install_id=installation_id, sender=data["sender"]["login"], subscription_active=subscription["subscription_active"], subscription_reason=subscription["subscription_reason"], )
def set_local_project(self): """ update self.local_project """ # TODO: ogr should have a method, something like this: # get_github_service(token, app_id, inst_id, cert_path) -> GithubService # the logic below should be the function # I want to leave this code here up the end of this sprint # TODO: in order to support any git forge here, ogr should also have a method like this: # get_github_service_from_url(url, **kwargs): # ogr should guess the forge based on the url; kwargs should be passed to the # constructor in order to support the above if (self.config.github_app_id and self.config.github_app_cert_path and self.config.github_app_installation_id): logger.info("Authenticating with Github using a Githab app.") private_key = Path(self.config.github_app_cert_path).read_text() integration = github.GithubIntegration(self.config.github_app_id, private_key) token = integration.get_access_token( self.config.github_app_installation_id) gh_service = GithubService(token=token) else: logger.debug("Authenticating with Github using a token.") gh_service = GithubService(token=self.config.github_token) self.local_project.git_service = gh_service if not self.local_project.repo_name: # will this ever happen? self.local_project.repo_name = self.package_name
def test(): import github from mergify_engine import gh_pr from mergify_engine import utils utils.setup_logging() config.log() gh_pr.monkeypatch_github() parts = sys.argv[1].split("/") LOG.info("Getting repo %s ..." % sys.argv[1]) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, parts[3]) token = integration.get_access_token(installation_id).token g = github.Github(token) user = g.get_user(parts[3]) repo = user.get_repo(parts[4]) LOG.info("Protecting repo %s branch %s ..." % (sys.argv[1], sys.argv[2])) rule = rules.get_branch_rule(repo, sys.argv[2]) configure_protection_if_needed(repo, sys.argv[2], rule)
def job_events(event_type, subscription, data): """Everything starts here.""" integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_token = integration.get_access_token( data["installation"]["id"]).token except github.UnknownObjectException: LOG.error("token for install %d does not exists anymore (%s)", data["installation"]["id"], data["repository"]["full_name"]) return g = github.Github(installation_token) try: user = g.get_user(data["repository"]["owner"]["login"]) repo = user.get_repo(data["repository"]["name"]) engine.MergifyEngine(g, data["installation"]["id"], installation_token, subscription, user, repo).handle(event_type, data) except github.BadCredentialsException: # pragma: no cover LOG.error("token for install %d is no longuer valid (%s)", data["installation"]["id"], data["repository"]["full_name"]) except github.RateLimitExceededException: # pragma: no cover LOG.error("rate limit reached for install %d (%s)", data["installation"]["id"], data["repository"]["full_name"])
def test(): from mergify_engine import gh_pr from mergify_engine import utils utils.setup_logging() config.log() gh_pr.monkeypatch_github() parts = sys.argv[1].split("/") LOG.info("Getting repo %s ..." % sys.argv[1]) if True: # With access_token got from oauth token = sys.argv[2] g = github.Github(token) user = g.get_user(parts[3]) repo = user.get_repo(parts[4]) pull = repo.get_pull(int(parts[6])) update_branch(pull, token) else: # With access_token got from integration integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, parts[3]) token = integration.get_access_token(installation_id).token update_branch(pull, "x-access-token:%s" % token)
def refresh_all(): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) for repo in i.get_repos(): counts[1] += 1 pulls = repo.get_pulls() branches = set([p.base.ref for p in pulls]) # Mimic the github event format for branch in branches: counts[2] += 1 get_queue().enqueue( worker.event_handler, "refresh", { 'repository': repo.raw_data, 'installation': { 'id': install['id'] }, 'refresh_ref': "branch/%s" % branch, }) return ("Updated %s installations, %s repositories, " "%s branches" % tuple(counts)), 202
def get_installation_token(installation_id): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: return integration.get_access_token(installation_id).token except github.UnknownObjectException: # pragma: no cover LOG.error("token for install %d does not exists anymore", installation_id) return
def refresh_all(): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) subscription = utils.get_subscription(utils.get_redis_for_cache(), install["id"]) if not subscription["token"]: # pragma: no cover continue for r in i.get_repos(): if r.private and not subscription["subscribed"]: continue counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': install["id"]}, 'pull_request': p.raw_data, } get_queue().enqueue(worker.event_handler, "refresh", subscription, data) return ("Updated %s installations, %s repositories, " "%s branches" % tuple(counts)), 202
def job_refresh_all(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) i = g.get_installation(install["id"]) for r in i.get_repos(): if r.archived: # pragma: no cover continue try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: continue else: raise counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': install["id"]}, 'pull_request': p.raw_data, } engine.run.s('refresh', data).apply_async() LOG.info("Refreshing %s installations, %s repositories, " "%s branches", *counts)
def status_repo(login, repo="*"): r = utils.get_redis_for_cache() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, login) return (_get_status(r, installation_id, login, repo), 200, {'Content-Type': 'application/json'})
def stream_repo(login, repo="*"): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, login) return flask.Response(flask.stream_with_context( stream_generate(installation_id, login, repo) ), mimetype="text/event-stream")
def report(url): path = url.replace("https://github.com/", "") owner, repo, _, pull_number = path.split("/") integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) install_id = utils.get_installation_id(integration, owner) installation_token = integration.get_access_token(install_id).token g = github.Github(installation_token) r = g.get_repo(owner + "/" + repo) p = r.get_pull(int(pull_number)) print("* CONFIGURATION:") print(r.get_contents(".mergify.yml").decoded_content.decode()) mp = mergify_pull.MergifyPull(p, installation_token) print("* PULL REQUEST:") pprint.pprint(mp.to_dict(), width=160) print("is_behind: %s" % mp.is_behind()) print("* MERGIFY STATUSES:") commit = p.base.repo.get_commit(p.head.sha) for s in commit.get_combined_status().statuses: if s.context.startswith("mergify"): print("[%s]: %s" % (s.context, s.state)) print("* MERGIFY CHECKS:") checks = list(check_api.get_checks(p)) for c in checks: if c.name.startswith("Mergify"): print("[%s]: %s | %s" % (c.name, c.conclusion, c.output.get("title"))) print("> " + "\n> ".join(c.output.get("summary").split("\n"))) return g, p
def refresh(owner, repo, branch): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, owner) if not installation_id: flask.abort(404, "%s have not installed pastamaker" % owner) # Mimic the github event format data = { 'repository': { 'name': repo, 'full_name': '%s/%s' % (owner, repo), 'owner': { 'login': owner }, }, 'installation': { 'id': installation_id }, "branch": branch, } get_queue().enqueue(worker.event_handler, "refresh", data) return "", 202
def app_auth(repo_name: str, app_id: int, private_key: str): gh = github.GithubIntegration(app_id, private_key) owner, repo = repo_name.split("/") inst = gh.get_installation(owner, repo) inst_auth = gh.get_access_token(inst.id.value) # type: ignore # error: "_ValuedAttribute" has no attribute "value" return github.Github(login_or_token=inst_auth.token)
def job_refresh(owner, repo, refresh_ref): LOG.info("%s/%s/%s: refreshing", owner, repo, refresh_ref) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_id = utils.get_installation_id(integration, owner, repo) except github.GithubException as e: LOG.warning("%s/%s/%s: mergify not installed", owner, repo, refresh_ref, error=str(e)) return token = integration.get_access_token(installation_id).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) r = g.get_repo("%s/%s" % (owner, repo)) if refresh_ref == "full" or refresh_ref.startswith("branch/"): if refresh_ref.startswith("branch/"): branch = refresh_ref[7:] pulls = r.get_pulls(base=branch) else: branch = '*' pulls = r.get_pulls() key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(), repo.lower(), r.private, branch) utils.get_redis_for_cache().delete(key) else: try: pull_number = int(refresh_ref[5:]) except ValueError: # pragma: no cover LOG.info("%s/%s/%s: Invalid PR ref", owner, repo, refresh_ref) return pulls = [r.get_pull(pull_number)] subscription = sub_utils.get_subscription(utils.get_redis_for_cache(), installation_id) if r.archived: # pragma: no cover LOG.warning("%s/%s/%s: repository archived", owner, repo, refresh_ref) return if not subscription["tokens"]: # pragma: no cover LOG.warning("%s/%s/%s: installation without token", owner, repo, refresh_ref) return for p in pulls: # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': installation_id}, 'pull_request': p.raw_data, } engine.run.s('refresh', data).apply_async()
def smart_strict_workflow_periodic_task(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) redis = utils.get_redis_for_cache() LOG.debug("smart strict workflow loop start") for key in redis.keys("strict-merge-queues~*"): _, installation_id, owner, reponame, branch = key.split("~") try: installation_token = integration.get_access_token( installation_id).token except github.UnknownObjectException: # pragma: no cover LOG.error("token for install %d does not exists anymore (%s/%s)", installation_id, owner, reponame) continue cur_key = "current-%s" % key redis = utils.get_redis_for_cache() pull_number = redis.get(cur_key) if pull_number and redis.sismember(key, pull_number): pull = mergify_pull.MergifyPull.from_number( installation_id, installation_token, owner, reponame, int(pull_number)) if pull.g_pull.state == "closed" or pull.is_behind(): # NOTE(sileht): Someone can have merged something manually in # base branch in the meantime, so we have to update it again. LOG.debug( "pull request needs to be updated again or " "has been closed", installation_id=installation_id, pull_number=pull_number, repo=owner + "/" + reponame, branch=branch) else: # NOTE(sileht): Pull request has not been merged or cancelled # yet wait next loop LOG.debug("pull request checks are still in progress", installation_id=installation_id, pull_number=pull_number, repo=owner + "/" + reponame, branch=branch) continue subscription = utils.get_subscription(redis, installation_id) if not subscription["token"]: # pragma: no cover LOG.error("no subscription token for updating base branch", installation_id=installation_id, repo=owner + "/" + reponame, branch=branch) continue # NOTE(sileht): Pick up the next pull request and rebase it update_next_pull(installation_id, installation_token, subscription, owner, reponame, branch, key, cur_key) LOG.debug("smart strict workflow loop end")
def event_handler(event_type, data): """Everything start here""" integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) token = integration.get_access_token(data["installation"]["id"]).token g = github.Github(token) user = g.get_user(data["repository"]["owner"]["login"]) repo = user.get_repo(data["repository"]["name"]) engine.PastaMakerEngine(g, user, repo).handle(event_type, data)
def _rpc_connection(self): installations = utils.find_installations(self.request) if installations.count() != 1: raise Exception( 'Cannot find GitHub App installation for tenant "%s"' % self.request.tenant.name) installation = installations.first() gh_app = github.GithubIntegration(settings.KIWI_GITHUB_APP_ID, settings.KIWI_GITHUB_APP_PRIVATE_KEY) token = utils.find_token_from_app_inst(gh_app, installation) return GithubKiwiTCMSBot(token)
def refresh(owner, repo, refresh_ref): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, owner) if not installation_id: flask.abort(404, "%s have not installed pastamaker" % owner) if refresh_ref == "full": token = integration.get_access_token(installation_id).token g = github.Github(token) r = g.get_repo("%s/%s" % (owner, repo)) pulls = r.get_pulls() branches = set([p.base.ref for p in pulls]) for branch in branches: # Mimic the github event format data = { 'repository': { 'name': repo, 'full_name': '%s/%s' % (owner, repo), 'owner': { 'login': owner }, }, 'installation': { 'id': installation_id }, "refresh_ref": "branch/%s" % branch, } get_queue().enqueue(worker.event_handler, "refresh", data) else: # Mimic the github event format data = { 'repository': { 'name': repo, 'full_name': '%s/%s' % (owner, repo), 'owner': { 'login': owner }, }, 'installation': { 'id': installation_id }, "refresh_ref": refresh_ref, } get_queue().enqueue(worker.event_handler, "refresh", data) return "", 202
def real_event_handler(event_type, data): """Everything start here""" integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) token = integration.get_access_token(data["installation"]["id"]).token g = github.Github(token) try: user = g.get_user(data["repository"]["owner"]["login"]) repo = user.get_repo(data["repository"]["name"]) engine.MergifyEngine(g, data["installation"]["id"], user, repo).handle(event_type, data) except github.RateLimitExceededException: LOG.error("rate limit reached")
def job_installations(installation_id, repositories): """Create the initial configuration on an repository.""" integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_token = integration.get_access_token( installation_id).token except github.UnknownObjectException: # pragma: no cover LOG.error("token for install %d does not exists anymore", installation_id) return g = github.Github(installation_token) try: if isinstance(repositories, str): installation = g.get_installation(installation_id) if repositories == "private": repositories = [ repo for repo in installation.get_repos() if repo.private ] elif repositories == "all": repositories = [repo for repo in installation.get_repos()] else: raise RuntimeError("Unexpected 'repositories' format: %s", type(repositories)) elif isinstance(repositories, list): # Some events return incomplete repository structure (like # installation event). Complete them in this case new_repos = [] for repository in repositories: user = g.get_user(repository["full_name"].split("/")[0]) repo = user.get_repo(repository["name"]) new_repos.append(repo) repositories = new_repos else: # pragma: no cover raise RuntimeError("Unexpected 'repositories' format: %s", type(repositories)) for repository in repositories: # NOTE(sileht): the installations event doesn't have this # attribute, so we keep it here. if repository.archived: # pragma: no cover continue initial_configuration.create_pull_request_if_needed( installation_token, repository) except github.RateLimitExceededException: # pragma: no cover LOG.error("rate limit reached for install %d", installation_id)
def __init__(self, bus, app_id, private_key, user_agent): super().__init__(bus) self._app_id = app_id self._private_key = private_key self._user_agent = user_agent self.installations = {} self.installation_clients = {} self.repo_to_installation = {} self._gh_integration = github.GithubIntegration( self._app_id, self._private_key, )
def event_handler(event_type, subscription, data): """Everything start here""" integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_token = integration.get_access_token( data["installation"]["id"]).token g = github.Github(installation_token) try: user = g.get_user(data["repository"]["owner"]["login"]) repo = user.get_repo(data["repository"]["name"]) engine.MergifyEngine(g, data["installation"]["id"], installation_token, subscription, user, repo).handle(event_type, data) except github.RateLimitExceededException: # pragma: no cover LOG.error("rate limit reached for install %d (%s)", data["installation"]["id"], data["repository"]["full_name"])
def setUp(self): super(TestEngineScenario, self).setUp() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) access_token = integration.get_access_token( config.INSTALLATION_ID).token g = github.Github(access_token) self.repo_as_app = g.get_repo("mergify-test1/" + self.name) # Used to access the cache with its helper self.processor = v1.Processor(self.subscription, self.repo_as_app, config.INSTALLATION_ID) self.setup_repo(CONFIG, ['stable', 'nostrict', 'disabled', 'enabling_label'])
def job_refresh(owner, repo, kind, ref=None): LOG.info("job refresh", kind=kind, ref=ref, gh_owner=owner, gh_repo=repo) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_id = utils.get_installation_id(integration, owner, repo) except github.GithubException as e: LOG.warning( "mergify not installed", kind=kind, ref=ref, gh_owner=owner, gh_repo=repo, error=str(e), ) return token = integration.get_access_token(installation_id).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) r = g.get_repo("%s/%s" % (owner, repo)) if kind == "repo": pulls = r.get_pulls() elif kind == "branch": pulls = r.get_pulls(base=ref) elif kind == "pull": pulls = [r.get_pull(ref)] else: raise RuntimeError("Invalid kind") for p in pulls: # Mimic the github event format data = { "repository": r.raw_data, "installation": { "id": installation_id }, "pull_request": p.raw_data, "sender": { "login": "******" }, } github_events.job_filter_and_dispatch.s("refresh", str(uuid.uuid4()), data).apply_async()
def job_refresh_all(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) subscription = utils.get_subscription(utils.get_redis_for_cache(), install["id"]) if not subscription["token"]: # pragma: no cover continue for r in i.get_repos(): if r.archived: # pragma: no cover continue if r.private and not subscription["subscribed"]: continue try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: continue else: raise counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': { 'id': install["id"] }, 'pull_request': p.raw_data, } queue.route(r.full_name, subscription, "events", "refresh", subscription, data) LOG.info("Refreshing %s installations, %s repositories, " "%s branches" % tuple(counts))
def PullRequestUrl(v): _, owner, repo, _, pull_number = urlsplit(v).path.split("/") pull_number = int(pull_number) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_id = utils.get_installation_id(integration, owner, repo) except github.GithubException: raise PullRequestUrlInvalid( message="Mergify not installed on repository '%s'" % owner) token = integration.get_access_token(installation_id).token try: return mergify_pull.MergifyPull.from_number(installation_id, token, owner, repo, pull_number) except github.UnknownObjectException: raise PullRequestUrlInvalid(message=("Pull request '%s' not found" % v))
def main(): # pragma: no cover integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installs = utils.get_installations(integration) r = utils.get_redis_for_cache() subscribed = 0 for i in installs: if utils.get_subscription(r, i["id"])["subscribed"]: subscribed += 1 print("installations: %d" % len(installs)) print("subscriptions: %d" % subscribed) active_slugs = set() for key in utils.get_redis_for_cache().keys("queues~*"): _, _, owner, repo, private, branch = key.split("~") active_slugs.add("%s/%s" % (owner, repo)) print("repositories with pending PRs: %d" % len(active_slugs)) print(" * %s" % "\n * ".join(sorted(active_slugs)))
def _get_next_pull_request(queue): _, installation_id, owner, reponame, branch = queue.split("~") integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) try: installation_token = integration.get_access_token( installation_id).token except github.UnknownObjectException: # pragma: no cover LOG.error("token for install %d does not exists anymore (%s/%s)", installation_id, owner, reponame) return redis = utils.get_redis_for_cache() pull_numbers = redis.zrange(queue, 0, 0) if pull_numbers: return mergify_pull.MergifyPull.from_number( installation_id, installation_token, owner, reponame, int(pull_numbers[0]))