def refresh(owner, repo, refresh_ref): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, owner) if not installation_id: # pragma: no cover flask.abort(400, "%s have not installed mergify_engine" % owner) token = integration.get_access_token(installation_id).token g = github.Github(token) r = g.get_repo("%s/%s" % (owner, repo)) try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: return "No .mergify.yml", 202 else: raise if refresh_ref == "full" or refresh_ref.startswith("branch/"): if refresh_ref.startswith("branch/"): branch = refresh_ref[7:] pulls = r.get_pulls(base=branch) else: branch = '*' pulls = r.get_pulls() key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(), repo.lower(), r.private, branch) utils.get_redis_for_cache().delete(key) else: try: pull_number = int(refresh_ref[5:]) except ValueError: # pragma: no cover return "Invalid PR ref", 400 pulls = [r.get_pull(pull_number)] subscription = utils.get_subscription(utils.get_redis_for_cache(), installation_id) if not subscription["token"]: # pragma: no cover return "", 202 if r.private and not subscription["subscribed"]: # pragma: no cover return "", 202 for p in pulls: # Mimic the github event format data = { 'repository': r.raw_data, 'installation': { 'id': installation_id }, 'pull_request': p.raw_data, } get_queue(r.full_name, subscription).enqueue(worker.event_handler, "refresh", subscription, data) return "", 202
def refresh_all(): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) subscription = utils.get_subscription(utils.get_redis_for_cache(), install["id"]) if not subscription["token"]: # pragma: no cover continue for r in i.get_repos(): if r.private and not subscription["subscribed"]: continue counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': install["id"]}, 'pull_request': p.raw_data, } get_queue().enqueue(worker.event_handler, "refresh", subscription, data) return ("Updated %s installations, %s repositories, " "%s branches" % tuple(counts)), 202
def smart_strict_workflow_periodic_task(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) redis = utils.get_redis_for_cache() LOG.debug("smart strict workflow loop start") for key in redis.keys("strict-merge-queues~*"): _, installation_id, owner, reponame, branch = key.split("~") try: installation_token = integration.get_access_token( installation_id).token except github.UnknownObjectException: # pragma: no cover LOG.error("token for install %d does not exists anymore (%s/%s)", installation_id, owner, reponame) continue cur_key = "current-%s" % key redis = utils.get_redis_for_cache() pull_number = redis.get(cur_key) if pull_number and redis.sismember(key, pull_number): pull = mergify_pull.MergifyPull.from_number( installation_id, installation_token, owner, reponame, int(pull_number)) if pull.g_pull.state == "closed" or pull.is_behind(): # NOTE(sileht): Someone can have merged something manually in # base branch in the meantime, so we have to update it again. LOG.debug( "pull request needs to be updated again or " "has been closed", installation_id=installation_id, pull_number=pull_number, repo=owner + "/" + reponame, branch=branch) else: # NOTE(sileht): Pull request has not been merged or cancelled # yet wait next loop LOG.debug("pull request checks are still in progress", installation_id=installation_id, pull_number=pull_number, repo=owner + "/" + reponame, branch=branch) continue subscription = utils.get_subscription(redis, installation_id) if not subscription["token"]: # pragma: no cover LOG.error("no subscription token for updating base branch", installation_id=installation_id, repo=owner + "/" + reponame, branch=branch) continue # NOTE(sileht): Pick up the next pull request and rebase it update_next_pull(installation_id, installation_token, subscription, owner, reponame, branch, key, cur_key) LOG.debug("smart strict workflow loop end")
def subscription_cache(installation_id): # pragma: no cover authentification() r = utils.get_redis_for_cache() r.delete("subscription-cache-%s" % installation_id) subscription = utils.get_subscription( utils.get_redis_for_cache(), installation_id) # New subscription, create initial configuration for private repo # public repository have already been done during the installation # event. if subscription["token"] and subscription["subscribed"]: get_queue().enqueue(worker.installation_handler, installation_id, "private") return "Cache cleaned", 200
def job_refresh_all(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) subscription = utils.get_subscription(utils.get_redis_for_cache(), install["id"]) if not subscription["token"]: # pragma: no cover continue for r in i.get_repos(): if r.archived: # pragma: no cover continue if r.private and not subscription["subscribed"]: continue try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: continue else: raise counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': { 'id': install["id"] }, 'pull_request': p.raw_data, } queue.route(r.full_name, subscription, "events", "refresh", subscription, data) LOG.info("Refreshing %s installations, %s repositories, " "%s branches" % tuple(counts))
def main(): # pragma: no cover integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installs = utils.get_installations(integration) r = utils.get_redis_for_cache() subscribed = 0 for i in installs: if utils.get_subscription(r, i["id"])["subscribed"]: subscribed += 1 print("installations: %d" % len(installs)) print("subscriptions: %d" % subscribed) active_slugs = set() for key in utils.get_redis_for_cache().keys("queues~*"): _, _, owner, repo, private, branch = key.split("~") active_slugs.add("%s/%s" % (owner, repo)) print("repositories with pending PRs: %d" % len(active_slugs)) print(" * %s" % "\n * ".join(sorted(active_slugs)))
def event_handler(): authentification() event_type = flask.request.headers.get("X-GitHub-Event") event_id = flask.request.headers.get("X-GitHub-Delivery") data = flask.request.get_json() subscription = utils.get_subscription( utils.get_redis_for_cache(), data["installation"]["id"]) if not subscription["token"]: msg_action = "ignored (no token)" elif event_type == "installation" and data["action"] == "created": for repository in data["repositories"]: if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover continue get_queue().enqueue(worker.installation_handler, data["installation"]["id"], [repository]) msg_action = "pushed to backend" elif event_type == "installation" and data["action"] == "deleted": key = "queues~%s~*~*~*~*" % data["installation"]["id"] utils.get_redis_for_cache().delete(key) msg_action = "handled, cache cleaned" elif (event_type == "installation_repositories" and data["action"] == "added"): for repository in data["repositories_added"]: if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover continue get_queue().enqueue(worker.installation_handler, data["installation"]["id"], [repository]) msg_action = "pushed to backend" elif (event_type == "installation_repositories" and data["action"] == "removed"): for repository in data["repositories_removed"]: if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover continue key = "queues~%s~%s~%s~*~*" % ( data["installation"]["id"], data["installation"]["account"]["login"].lower(), repository["name"].lower() ) utils.get_redis_for_cache().delete(key) msg_action = "handled, cache cleaned" elif event_type in ["installation", "installation_repositories"]: msg_action = "ignored (action %s)" % data["action"] elif event_type in ["pull_request", "pull_request_review", "status", "refresh"]: if data["repository"]["private"] and not subscription["subscribed"]: msg_action = "ignored (not public or subscribe)" elif event_type == "status" and data["state"] == "pending": msg_action = "ignored (state pending)" elif (event_type == "pull_request" and data["action"] not in [ "opened", "reopened", "closed", "synchronize", "labeled", "unlabeled"]): msg_action = "ignored (action %s)" % data["action"] else: get_queue().enqueue(worker.event_handler, event_type, subscription, data) msg_action = "pushed to backend" else: msg_action = "ignored (unexpected event_type)" if "repository" in data: repo_name = data["repository"]["full_name"] else: repo_name = data["installation"]["account"]["login"] LOG.info('[%s/%s] received "%s" event "%s", %s', data["installation"]["id"], repo_name, event_type, event_id, msg_action) return "", 202
def job_filter_and_dispatch(event_type, event_id, data): subscription = utils.get_subscription(utils.get_redis_for_cache(), data["installation"]["id"]) if not subscription["token"]: msg_action = "ignored (no token)" elif event_type == "installation" and data["action"] == "created": for repository in data["repositories"]: if repository["private"] and not subscription[ "subscribed"]: # noqa pragma: no cover continue queue.route(repository["full_name"], subscription, "installations", data["installation"]["id"], [repository]) msg_action = "pushed to backend" elif event_type == "installation" and data["action"] == "deleted": key = "queues~%s~*~*~*~*" % data["installation"]["id"] utils.get_redis_for_cache().delete(key) msg_action = "handled, cache cleaned" elif (event_type == "installation_repositories" and data["action"] == "added"): for repository in data["repositories_added"]: if repository["private"] and not subscription[ "subscribed"]: # noqa pragma: no cover continue queue.route(repository["full_name"], subscription, "installations", data["installation"]["id"], [repository]) msg_action = "pushed to backend" elif (event_type == "installation_repositories" and data["action"] == "removed"): for repository in data["repositories_removed"]: if repository["private"] and not subscription[ "subscribed"]: # noqa pragma: no cover continue key = "queues~%s~%s~%s~*~*" % ( data["installation"]["id"], data["installation"]["account"]["login"].lower(), repository["name"].lower()) utils.get_redis_for_cache().delete(key) msg_action = "handled, cache cleaned" elif event_type in ["installation", "installation_repositories"]: msg_action = "ignored (action %s)" % data["action"] elif event_type in ["pull_request", "pull_request_review", "status"]: if data["repository"]["archived"]: # pragma: no cover msg_action = "ignored (repository archived)" elif (data["repository"]["private"] and not subscription["subscribed"]): msg_action = "ignored (not public or subscribe)" elif event_type == "status" and data["state"] == "pending": msg_action = "ignored (state pending)" elif event_type == "status" and data["context"] == "mergify/pr": msg_action = "ignored (mergify status)" elif (event_type == "pull_request" and data["action"] not in [ "opened", "reopened", "closed", "synchronize", "labeled", "unlabeled" ]): msg_action = "ignored (action %s)" % data["action"] else: queue.route(data["repository"]["full_name"], subscription, "events", event_type, subscription, data) msg_action = "pushed to backend" else: msg_action = "ignored (unexpected event_type)" if "repository" in data: repo_name = data["repository"]["full_name"] else: repo_name = data["installation"]["account"]["login"] LOG.info('event %s', msg_action, event_type=event_type, event_id=event_id, install_id=data["installation"]["id"], repository=repo_name, subscribed=subscription["subscribed"])
def job_refresh(owner, repo, refresh_ref): LOG.info("%s/%s/%s: refreshing" % (owner, repo, refresh_ref)) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, owner) if not installation_id: # pragma: no cover LOG.warning("%s/%s/%s: mergify not installed" % (owner, repo, refresh_ref)) return token = integration.get_access_token(installation_id).token g = github.Github(token) r = g.get_repo("%s/%s" % (owner, repo)) try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: LOG.warning("%s/%s/%s: mergify not configured" % (owner, repo, refresh_ref)) return else: raise if refresh_ref == "full" or refresh_ref.startswith("branch/"): if refresh_ref.startswith("branch/"): branch = refresh_ref[7:] pulls = r.get_pulls(base=branch) else: branch = '*' pulls = r.get_pulls() key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(), repo.lower(), r.private, branch) utils.get_redis_for_cache().delete(key) else: try: pull_number = int(refresh_ref[5:]) except ValueError: # pragma: no cover LOG.info("%s/%s/%s: Invalid PR ref" % (owner, repo, refresh_ref)) return pulls = [r.get_pull(pull_number)] subscription = utils.get_subscription(utils.get_redis_for_cache(), installation_id) if r.archived: # pragma: no cover LOG.warning("%s/%s/%s: repository archived" % (owner, repo, refresh_ref)) return if not subscription["token"]: # pragma: no cover LOG.warning("%s/%s/%s: not public or subscribed" % (owner, repo, refresh_ref)) return if r.private and not subscription["subscribed"]: # pragma: no cover LOG.warning("%s/%s/%s: mergify not installed" % (owner, repo, refresh_ref)) return for p in pulls: # Mimic the github event format data = { 'repository': r.raw_data, 'installation': { 'id': installation_id }, 'pull_request': p.raw_data, } queue.route(r.full_name, subscription, "events", "refresh", subscription, data)
def collect_metrics(): redis = utils.get_redis_for_cache() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installations = collections.defaultdict(int) repositories_per_installation = collections.defaultdict(int) users_per_installation = collections.defaultdict(int) LOG.info("GitHub Polling started") redis.delete("badges.tmp") for installation in utils.get_installations(integration): try: _id = installation["id"] target_type = installation["target_type"] account = installation["account"]["login"] LOG.info("Get subscription", account=account) subscribed = utils.get_subscription(redis, _id)["subscribed"] installations[(subscribed, target_type)] += 1 token = integration.get_access_token(_id).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) if installation[ "target_type"] == "Organization": # pragma: no cover LOG.info("Get members", install=installation["account"]["login"]) org = g.get_organization(installation["account"]["login"]) value = len(list(org.get_members())) users_per_installation[(subscribed, target_type, account)] = value else: users_per_installation[(subscribed, target_type, account)] = 1 LOG.info("Get repos", account=account) repositories = sorted(g.get_installation(_id).get_repos(), key=operator.attrgetter("private")) for private, repos in itertools.groupby( repositories, key=operator.attrgetter("private")): configured_repos = 0 unconfigured_repos = 0 for repo in repos: try: repo.get_contents(".mergify.yml") configured_repos += 1 redis.sadd("badges.tmp", repo.full_name) except github.GithubException as e: if e.status >= 500: # pragma: no cover raise unconfigured_repos += 1 repositories_per_installation[(subscribed, target_type, account, private, True)] = configured_repos repositories_per_installation[(subscribed, target_type, account, private, False)] = unconfigured_repos except github.GithubException as e: # Ignore rate limit/abuse if e.status != 403: raise LOG.info("GitHub Polling finished") # NOTE(sileht): Prometheus can scrape data during our loop. So make it fast # to ensure we always have the good values. # Also we can't known which labels we should delete from the Gauge, # that's why we delete all of them to re-add them. # And prometheus_client doesn't provide API to that, so we just # override _metrics set_gauges(INSTALLATIONS, installations) set_gauges(USERS_PER_INSTALLATION, users_per_installation) set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation) if redis.exists("badges.tmp"): redis.rename("badges.tmp", "badges") LOG.info("Gauges and badges cache updated")
def job_filter_and_dispatch(event_type, event_id, data): subscription = utils.get_subscription( utils.get_redis_for_cache(), data["installation"]["id"]) if not subscription["token"]: msg_action = "ignored (no token)" elif event_type == "installation" and data["action"] == "deleted": # TODO(sileht): move out this engine V1 related code key = "queues~%s~*~*~*~*" % data["installation"]["id"] utils.get_redis_for_cache().delete(key) msg_action = "handled, cache cleaned" elif (event_type == "installation_repositories" and data["action"] == "removed"): for repository in data["repositories_removed"]: if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover continue # TODO(sileht): move out this engine V1 related code key = "queues~%s~%s~%s~*~*" % ( data["installation"]["id"], data["installation"]["account"]["login"].lower(), repository["name"].lower() ) utils.get_redis_for_cache().delete(key) msg_action = "handled, cache cleaned" elif event_type in ["installation", "installation_repositories"]: msg_action = "ignored (action %s)" % data["action"] elif event_type in ["pull_request", "pull_request_review", "status", "check_suite", "check_run"]: if data["repository"]["archived"]: # pragma: no cover msg_action = "ignored (repository archived)" elif (data["repository"]["private"] and not subscription["subscribed"]): msg_action = "ignored (not public or subscribe)" elif event_type == "status" and data["state"] == "pending": msg_action = "ignored (state pending)" elif event_type == "status" and data["context"] == "mergify/pr": msg_action = "ignored (mergify status)" elif (event_type in ["check_run", "check_suite"] and data[event_type]["app"]["id"] == config.INTEGRATION_ID): msg_action = "ignored (mergify %s)" % event_type elif (event_type == "pull_request" and data["action"] not in [ "opened", "reopened", "closed", "synchronize", "labeled", "unlabeled", "edited"]): msg_action = "ignored (action %s)" % data["action"] else: engine.run.s(event_type, data, subscription).apply_async() msg_action = "pushed to backend" if event_type == "pull_request": msg_action += ", action: %s" % data["action"] elif event_type == "pull_request_review": msg_action += ", action: %s, review-state: %s" % ( data["action"], data["review"]["state"]) elif event_type == "pull_request_review_comment": msg_action += ", action: %s, review-state: %s" % ( data["action"], data["comment"]["position"]) elif event_type == "status": msg_action += ", ci-status: %s, sha: %s" % ( data["state"], data["sha"]) elif event_type in ["check_run", "check_suite"]: msg_action += ( ", action: %s, status: %s, conclusion: %s, sha: %s" % ( data["action"], data[event_type]["status"], data[event_type]["conclusion"], data[event_type]["head_sha"])) else: msg_action = "ignored (unexpected event_type)" if "repository" in data: repo_name = data["repository"]["full_name"] else: repo_name = data["installation"]["account"]["login"] LOG.info('event %s', msg_action, event_type=event_type, event_id=event_id, install_id=data["installation"]["id"], sender=data["sender"]["login"], repository=repo_name, subscribed=subscription["subscribed"])