def update_with_git(ctxt, method="merge", user=None):
subscription = asyncio.run(
sub_utils.get_subscription(ctxt.client.auth.owner_id))
creds = dict((login.lower(), token)
for login, token in subscription["tokens"].items())
if user:
token = creds.get(user.lower())
if token:
creds = {user: token}
else:
raise BranchUpdateFailure(
f"Unable to rebase: user `{user}` is unknown. "
f"Please make sure `{user}` has logged in Mergify dashboard.")
for login, token in creds.items():
try:
return _do_update(ctxt, token, method)
except AuthentificationFailure as e: # pragma: no cover
ctxt.log.info(
"authentification failure, will retry another token: %s",
e,
login=login,
)
ctxt.log.warning("unable to update branch: no tokens are valid")
if ctxt.pull_from_fork and ctxt.pull["base"]["repo"]["private"]:
raise BranchUpdateFailure(
"Rebasing a branch for a forked private repository is not supported by GitHub"
)
raise AuthentificationFailure("No valid OAuth tokens")
def job_marketplace(event_type, event_id, data):
owner = data["marketplace_purchase"]["account"]["login"]
account_type = data["marketplace_purchase"]["account"]["type"]
try:
installation = github_app.get_client().get_installation(
owner, account_type=account_type
)
except exceptions.MergifyNotInstalled:
return
r = utils.get_redis_for_cache()
r.delete("subscription-cache-%s" % installation["id"])
subscription = sub_utils.get_subscription(r, installation["id"])
LOG.info(
"Marketplace event",
event_type=event_type,
event_id=event_id,
install_id=installation["id"],
sender=data["sender"]["login"],
gh_owner=owner,
subscription_active=subscription["subscription_active"],
subscription_reason=subscription["subscription_reason"],
)
def update_with_git(ctxt, method="merge"):
redis = utils.get_redis_for_cache()
subscription = sub_utils.get_subscription(redis,
ctxt.client.installation["id"])
for login, token in subscription["tokens"].items():
try:
return _do_update(ctxt, token, method)
except AuthentificationFailure as e: # pragma: no cover
ctxt.log.info(
"authentification failure, will retry another token: %s",
e,
login=login,
)
ctxt.log.warning("unable to update branch: no tokens are valid")
if (ctxt.pull["head"]["repo"]["owner"]["login"] !=
ctxt.pull["base"]["repo"]["owner"]["login"]
and ctxt.pull["base"]["repo"]["private"]):
raise BranchUpdateFailure(
"Rebasing a branch for a forked private repository is not supported by GitHub"
)
raise AuthentificationFailure("No valid OAuth tokens")
def run_engine(installation, owner, repo, pull_number, sources):
logger = logs.getLogger(__name__,
gh_repo=repo,
gh_owner=owner,
gh_pull=pull_number)
logger.debug("engine in thread start")
try:
sync_redis = utils.get_redis_for_cache()
subscription = sub_utils.get_subscription(sync_redis,
installation["id"])
logger.debug("engine get installation")
with github.get_client(owner, repo, installation) as client:
try:
pull = client.item(f"pulls/{pull_number}")
except httpx.HTTPClientSideError as e:
if e.status_code == 404:
logger.debug("pull request doesn't exists, skipping it")
return
raise
if (pull["base"]["repo"]["private"]
and not subscription["subscription_active"]):
logger.debug(
"pull request on private private repository without subscription, skipping it"
)
return
engine.run(client, pull, sources)
finally:
logger.debug("engine in thread end")
def job_marketplace(event_type, event_id, data):
owner = data["marketplace_purchase"]["account"]["login"]
account_type = data["marketplace_purchase"]["account"]["type"]
integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY)
try:
installation_id = utils.get_installation_id(
integration, owner, account_type=account_type
)
except github.GithubException as e:
LOG.warning("%s: mergify not installed", owner, error=str(e))
installation_id = None
subscription = {
"subscription_active": "Unknown",
"subscription_reason": "No",
"tokens": None,
}
else:
r = utils.get_redis_for_cache()
r.delete("subscription-cache-%s" % installation_id)
subscription = sub_utils.get_subscription(r, installation_id)
LOG.info(
"Marketplace event",
event_type=event_type,
event_id=event_id,
install_id=installation_id,
sender=data["sender"]["login"],
subscription_active=subscription["subscription_active"],
subscription_reason=subscription["subscription_reason"],
)
def job_marketplace(event_type, event_id, data):
owner = data["marketplace_purchase"]["account"]["login"]
account_type = data["marketplace_purchase"]["account"]["type"]
try:
installation_id = github_app.get_client().get_installation_id(
owner, account_type=account_type
)
except httpx.HTTPClientSideError as e:
LOG.warning("mergify not installed", gh_owner=owner, error=str(e))
installation_id = None
subscription = {
"subscription_active": "Unknown",
"subscription_reason": "No",
"tokens": None,
}
else:
r = utils.get_redis_for_cache()
r.delete("subscription-cache-%s" % installation_id)
subscription = sub_utils.get_subscription(r, installation_id)
LOG.info(
"Marketplace event",
event_type=event_type,
event_id=event_id,
install_id=installation_id,
sender=data["sender"]["login"],
gh_owner=owner,
subscription_active=subscription["subscription_active"],
subscription_reason=subscription["subscription_reason"],
)
def process(self):
pull_numbers = self.get_pulls()
self.log.info("%d pulls queued", len(pull_numbers), queue=list(pull_numbers))
if not pull_numbers:
return
pull_number = int(pull_numbers[0])
try:
installation = github.get_installation(
self.owner, self.repo, self.installation_id
)
except exceptions.MergifyNotInstalled:
self.delete_queue()
return
subscription = sub_utils.get_subscription(
utils.get_redis_for_cache(), self.installation_id,
)
with github.get_client(self.owner, self.repo, installation) as client:
data = client.item(f"pulls/{pull_number}")
try:
ctxt = context.Context(client, data, subscription)
except exceptions.RateLimited as e:
self.log.debug("rate limited", remaining_seconds=e.countdown)
return
except exceptions.MergeableStateUnknown as e: # pragma: no cover
e.ctxt.log.warning(
"pull request with mergeable_state unknown retrying later",
)
self._move_pull_at_end(pull_number)
return
try:
if ctxt.pull["base"]["ref"] != self.ref:
ctxt.log.info(
"pull request base branch have changed",
old_branch=self.ref,
new_branch=ctxt.pull["base"]["ref"],
)
self._move_pull_to_new_base_branch(ctxt, self.ref)
elif ctxt.pull["state"] == "closed" or ctxt.is_behind:
# NOTE(sileht): Pick up this pull request and rebase it again
# or update its status and remove it from the queue
ctxt.log.info(
"pull request needs to be updated again or has been closed",
)
self.handle_first_pull_in_queue(ctxt)
else:
# NOTE(sileht): Pull request has not been merged or cancelled
# yet wait next loop
ctxt.log.info("pull request checks are still in progress")
except Exception: # pragma: no cover
ctxt.log.error("Fail to process merge queue", exc_info=True)
self._move_pull_at_end(pull_number)
def process(self):
pull_numbers = self.get_pulls()
self.log.info("%d pulls queued", len(pull_numbers), queue=list(pull_numbers))
if not pull_numbers:
return
pull_number = pull_numbers[0]
with github.get_client(self.owner, self.repo) as client:
ctxt = None
try:
subscription = asyncio.run(
sub_utils.get_subscription(client.auth.owner_id)
)
data = client.item(f"pulls/{pull_number}")
ctxt = context.Context(client, data, subscription)
if ctxt.pull["base"]["ref"] != self.ref:
ctxt.log.info(
"pull request base branch have changed",
old_branch=self.ref,
new_branch=ctxt.pull["base"]["ref"],
)
self._move_pull_to_new_base_branch(ctxt.pull["number"], self.ref)
elif ctxt.pull["state"] == "closed" or ctxt.is_behind:
# NOTE(sileht): Pick up this pull request and rebase it again
# or update its status and remove it from the queue
ctxt.log.info(
"pull request needs to be updated again or has been closed",
)
self.handle_first_pull_in_queue(ctxt)
else:
# NOTE(sileht): Pull request has not been merged or cancelled
# yet wait next loop
ctxt.log.info("pull request checks are still in progress")
except Exception as exc: # pragma: no cover
log = self.log if ctxt is None else ctxt.log
if exceptions.should_be_ignored(exc):
log.info(
"Fail to process merge queue, remove the pull request from the queue",
exc_info=True,
)
self.remove_pull(ctxt.pull["number"])
elif exceptions.need_retry(exc):
log.info("Fail to process merge queue, need retry", exc_info=True)
if isinstance(exc, exceptions.MergeableStateUnknown):
# NOTE(sileht): We need GitHub to recompute the state here (by
# merging something else for example), so move it to the end
self._move_pull_at_end(pull_number)
else:
log.error("Fail to process merge queue", exc_info=True)
self._move_pull_at_end(pull_number)
def job_refresh(owner, repo, refresh_ref):
LOG.info("%s/%s/%s: refreshing", owner, repo, refresh_ref)
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
try:
installation_id = utils.get_installation_id(integration, owner, repo)
except github.GithubException as e:
LOG.warning("%s/%s/%s: mergify not installed",
owner, repo, refresh_ref, error=str(e))
return
token = integration.get_access_token(installation_id).token
g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN)
r = g.get_repo("%s/%s" % (owner, repo))
if refresh_ref == "full" or refresh_ref.startswith("branch/"):
if refresh_ref.startswith("branch/"):
branch = refresh_ref[7:]
pulls = r.get_pulls(base=branch)
else:
branch = '*'
pulls = r.get_pulls()
key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(),
repo.lower(), r.private, branch)
utils.get_redis_for_cache().delete(key)
else:
try:
pull_number = int(refresh_ref[5:])
except ValueError: # pragma: no cover
LOG.info("%s/%s/%s: Invalid PR ref", owner, repo, refresh_ref)
return
pulls = [r.get_pull(pull_number)]
subscription = sub_utils.get_subscription(utils.get_redis_for_cache(),
installation_id)
if r.archived: # pragma: no cover
LOG.warning("%s/%s/%s: repository archived",
owner, repo, refresh_ref)
return
if not subscription["tokens"]: # pragma: no cover
LOG.warning("%s/%s/%s: installation without token",
owner, repo, refresh_ref)
return
for p in pulls:
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {'id': installation_id},
'pull_request': p.raw_data,
}
engine.run.s('refresh', data).apply_async()
def job_filter_and_dispatch(event_type, event_id, data):
meter_event(event_type, data)
if "installation" in data:
installation_id = data["installation"]["id"]
installation_owner = data["installation"].get("account", {"login": "******"})[
"login"
]
subscription = sub_utils.get_subscription(
utils.get_redis_for_cache(), installation_id
)
else:
installation_id = "Unknown"
installation_owner = "Unknown"
subscription = {
"subscription_active": "Unknown",
"subscription_reason": "No",
"tokens": None,
}
reason = get_ignore_reason(subscription, event_type, data)
if reason:
msg_action = reason
elif event_type in ["push"]:
owner, _, repo = data["repository"]["full_name"].partition("/")
branch = data["ref"][11:]
msg_action = "run refresh branch %s" % branch
mergify_events.job_refresh.s(owner, repo, "branch", branch).apply_async(
countdown=10
)
else:
engine.run.s(event_type, data).apply_async()
msg_action = "pushed to backend%s" % get_extra_msg_from_event(event_type, data)
if "repository" in data:
repo_name = data["repository"]["full_name"]
private = data["repository"]["private"]
else:
repo_name = "Unknown"
private = "Unknown"
LOG.info(
"GithubApp event %s",
msg_action,
event_type=event_type,
event_id=event_id,
install_id=installation_id,
install_owner=installation_owner,
sender=data["sender"]["login"],
repository=repo_name,
private=private,
subscription_active=subscription["subscription_active"],
subscription_reason=subscription["subscription_reason"],
)
def update(pull, installation_id, method="merge"):
redis = utils.get_redis_for_cache()
subscription = sub_utils.get_subscription(redis, installation_id)
for login, token in subscription["tokens"].items():
try:
return _do_update(pull, token, method)
except AuthentificationFailure as e: # pragma: no cover
LOG.debug("authentification failure, will retry another token: %s",
e,
login=login,
pull_request=pull)
LOG.error("unable to update branch: no tokens are valid",
pull_request=pull)
def update_with_git(pull, method="merge"):
redis = utils.get_redis_for_cache()
subscription = sub_utils.get_subscription(redis, pull.installation_id)
for login, token in subscription["tokens"].items():
try:
return _do_update(pull, token, method)
except AuthentificationFailure as e: # pragma: no cover
pull.log.debug(
"authentification failure, will retry another token: %s",
e,
login=login,
)
pull.log.error("unable to update branch: no tokens are valid")
raise BranchUpdateFailure("No oauth valid tokens")
def job_refresh_all():
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
counts = [0, 0, 0]
for install in utils.get_installations(integration):
counts[0] += 1
token = integration.get_access_token(install["id"]).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
i = g.get_installation(install["id"])
subscription = sub_utils.get_subscription(utils.get_redis_for_cache(),
install["id"])
if not subscription["token"]: # pragma: no cover
continue
for r in i.get_repos():
if r.archived: # pragma: no cover
continue
if r.private and not subscription["subscribed"]:
continue
try:
r.get_contents(".mergify.yml")
except github.GithubException as e: # pragma: no cover
if e.status == 404:
continue
else:
raise
counts[1] += 1
for p in list(r.get_pulls()):
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {
'id': install["id"]
},
'pull_request': p.raw_data,
}
engine.run.s('refresh', data).apply_async()
LOG.info("Refreshing %s installations, %s repositories, "
"%s branches", *counts)
def job_marketplace(event_type, event_id, data):
owner = data["marketplace_purchase"]["account"]["login"]
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installation_id = utils.get_installation_id(integration, owner)
r = utils.get_redis_for_cache()
r.delete("subscription-cache-%s" % installation_id)
subscription = sub_utils.get_subscription(
utils.get_redis_for_cache(), installation_id)
LOG.info('Marketplace event',
event_type=event_type,
event_id=event_id,
install_id=installation_id,
sender=data["sender"]["login"],
subscription_active=subscription["subscription_active"],
subscription_reason=subscription["subscription_reason"])
def report(url):
redis = utils.get_redis_for_cache()
path = url.replace("https://github.com/", "")
try:
owner, repo, _, pull_number = path.split("/")
except ValueError:
print(f"Wrong URL: {url}")
return
slug = owner + "/" + repo
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
install_id = utils.get_installation_id(integration, owner, repo=repo)
print("* INSTALLATION ID: %s" % install_id)
cached_sub = sub_utils.get_subscription(redis, install_id)
db_sub = sub_utils._retrieve_subscription_from_db(install_id)
print("* SUBSCRIBED (cache/db): %s / %s" %
(cached_sub["subscription_active"], db_sub["subscription_active"]))
report_sub(install_id, slug, cached_sub, "ENGINE-CACHE")
report_sub(install_id, slug, db_sub, "DASHBOARD")
installation_token = integration.get_access_token(install_id).token
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
r = g.get_repo(owner + "/" + repo)
print("* REPOSITORY IS %s" % "PRIVATE" if r.private else "PUBLIC")
print("* CONFIGURATION:")
try:
mergify_config_content = rules.get_mergify_config_content(r)
except rules.NoRules: # pragma: no cover
print(".mergify.yml is missing")
else:
print(mergify_config_content.decode())
try:
mergify_config = rules.UserConfigurationSchema(mergify_config_content)
except rules.InvalidRules as e: # pragma: no cover
print("configuration is invalid %s" % str(e))
else:
pull_request_rules_raw = mergify_config["pull_request_rules"].as_dict()
pull_request_rules_raw["rules"].extend(
actions_runner.MERGIFY_RULE["rules"])
pull_request_rules = rules.PullRequestRules(**pull_request_rules_raw)
try:
p = r.get_pull(int(pull_number))
except github.UnknownObjectException:
print("Wrong pull request number")
return g, None
mp = mergify_pull.MergifyPull(g, p, install_id)
print("* PULL REQUEST:")
pprint.pprint(mp.to_dict(), width=160)
try:
print("is_behind: %s" % mp.is_behind())
except github.GithubException as e:
print("Unable to know if pull request branch is behind: %s" % e)
print("mergeable_state: %s" % mp.g_pull.mergeable_state)
print("* MERGIFY LAST CHECKS:")
checks = list(check_api.get_checks(p))
for c in checks:
if c._rawData["app"]["id"] == config.INTEGRATION_ID:
print("[%s]: %s | %s" %
(c.name, c.conclusion, c.output.get("title")))
print("> " + "\n> ".join(c.output.get("summary").split("\n")))
print("* MERGIFY LIVE MATCHES:")
match = pull_request_rules.get_pull_request_rule(mp)
summary_title, summary = actions_runner.gen_summary(
"refresh", {}, mp, match)
print("> %s" % summary_title)
print(summary)
return g, p
def update(pull, installation_id, method="merge"):
# NOTE(sileht):
# $ curl https://api.github.com/repos/sileht/repotest/pulls/2 | jq .commits
# 2
# $ git clone https://[email protected]/sileht-tester/repotest \
# --depth=$((2 + 1)) -b sileht/testpr
# $ cd repotest
# $ git remote add upstream https://[email protected]/sileht/repotest.git
# $ git log | grep Date | tail -1
# Date: Fri Mar 30 21:30:26 2018 (10 days ago)
# $ git fetch upstream master --shallow-since="Fri Mar 30 21:30:26 2018"
# $ git rebase upstream/master
# $ git push origin sileht/testpr:sileht/testpr
redis = utils.get_redis_for_cache()
subscription = sub_utils.get_subscription(redis, installation_id)
if not subscription:
LOG.error("subscription to update branch is missing")
return
token = subscription["token"]
head_repo = pull.g_pull.head.repo.full_name
base_repo = pull.g_pull.base.repo.full_name
head_branch = pull.g_pull.head.ref
base_branch = pull.g_pull.base.ref
git = utils.Gitter()
try:
git("init")
git.configure()
git.add_cred(token, "", head_repo)
git.add_cred(token, "", base_repo)
git("remote", "add", "origin",
"https://%s/%s" % (config.GITHUB_DOMAIN, head_repo))
git("remote", "add", "upstream",
"https://%s/%s" % (config.GITHUB_DOMAIN, base_repo))
depth = int(pull.g_pull.commits) + 1
git("fetch", "--quiet", "--depth=%d" % depth, "origin", head_branch)
git("checkout", "-q", "-b", head_branch, "origin/%s" % head_branch)
out = git("log", "--format=%cI")
last_commit_date = [
d for d in out.decode("utf8").split("\n") if d.strip()
][-1]
git("fetch", "--quiet", "upstream", base_branch,
"--shallow-since='%s'" % last_commit_date)
try:
_do_update(git, method, base_branch, head_branch)
except subprocess.CalledProcessError as e:
if b"unrelated histories" in e.output:
LOG.debug("Complete history cloned", pull_request=pull)
# NOTE(sileht): We currently assume we have only one parent
# commit in common. Since Git is a graph, in some case this
# graph can be more complicated.
# So, retrying with the whole git history for now
git("fetch", "--quiet", "origin", head_branch)
git("fetch", "--quiet", "upstream", base_branch)
_do_update(git, method, base_branch, head_branch)
else:
raise
return git("log", "-1", "--format=%H").decode().strip()
except Exception: # pragma: no cover
LOG.error("update branch fail", pull_request=pull, exc_info=True)
finally:
git.cleanup()
def run(event_type, data):
"""Everything starts here."""
installation_id = data["installation"]["id"]
owner = data["repository"]["owner"]["login"]
repo = data["repository"]["name"]
client = github.get_client(owner, repo, installation_id)
raw_pull = get_github_pull_from_event(client, event_type, data)
if not raw_pull: # pragma: no cover
LOG.info(
"No pull request found in the event %s, ignoring",
event_type,
gh_owner=owner,
gh_repo=repo,
)
return
pull = mergify_pull.MergifyPull(client, raw_pull)
# Override pull_request with the updated one
data["pull_request"] = pull.data
pull.log.info("Pull request found in the event %s", event_type)
if ("base" not in pull.data or "repo" not in pull.data["base"]
or len(list(pull.data["base"]["repo"].keys())) < 70):
pull.log.warning(
"the pull request payload looks suspicious",
event_type=event_type,
data=data,
)
if (event_type == "status"
and pull.data["head"]["sha"] != data["sha"]): # pragma: no cover
pull.log.info(
"No need to proceed queue (got status of an old commit)", )
return
elif (event_type in ["status", "check_suite", "check_run"]
and pull.data["merged"]): # pragma: no cover
pull.log.info(
"No need to proceed queue (got status of a merged pull request)", )
return
elif (event_type in ["check_suite", "check_run"]
and pull.data["head"]["sha"] !=
data[event_type]["head_sha"]): # pragma: no cover
pull.log.info(
"No need to proceed queue (got %s of an old "
"commit)",
event_type,
)
return
if check_configuration_changes(pull.g_pull):
pull.log.info("Configuration changed, ignoring", )
return
# BRANCH CONFIGURATION CHECKING
try:
mergify_config = rules.get_mergify_config(pull.g_pull.base.repo)
except rules.NoRules: # pragma: no cover
pull.log.info("No need to proceed queue (.mergify.yml is missing)", )
return
except rules.InvalidRules as e: # pragma: no cover
# Not configured, post status check with the error message
if event_type == "pull_request" and data["action"] in [
"opened", "synchronize"
]:
check_api.set_check_run(
pull.g_pull,
"Summary",
"completed",
"failure",
output={
"title": "The Mergify configuration is invalid",
"summary": str(e),
},
)
return
# Add global and mandatory rules
mergify_config["pull_request_rules"].rules.extend(
rules.load_pull_request_rules_schema(MERGIFY_RULE["rules"]))
subscription = sub_utils.get_subscription(utils.get_redis_for_cache(),
installation_id)
if pull.data["base"]["repo"][
"private"] and not subscription["subscription_active"]:
check_api.set_check_run(
pull.g_pull,
"Summary",
"completed",
"failure",
output={
"title": "Mergify is disabled",
"summary": subscription["subscription_reason"],
},
)
return
# CheckRun are attached to head sha, so when user add commits or force push
# we can't directly get the previous Mergify Summary. So we copy it here, then
# anything that looks at it in next celery tasks will find it.
if event_type == "pull_request" and data["action"] == "synchronize":
copy_summary_from_previous_head_sha(pull.g_pull, data["before"])
sources = [{"event_type": event_type, "data": data}]
commands_runner.spawn_pending_commands_tasks(pull, sources)
if event_type == "issue_comment":
commands_runner.run_command(pull, sources, data["comment"]["body"],
data["comment"]["user"])
else:
actions_runner.handle(mergify_config["pull_request_rules"], pull,
sources)
def report(url):
redis = utils.get_redis_for_cache()
path = url.replace("https://github.com/", "")
try:
owner, repo, _, pull_number = path.split("/")
except ValueError:
print(f"Wrong URL: {url}")
return
slug = owner + "/" + repo
try:
installation = github.get_installation(owner, repo)
except exceptions.MergifyNotInstalled:
print("* Mergify is not installed there")
return
client = github.get_client(owner, repo, installation)
print("* INSTALLATION ID: %s" % client.installation["id"])
cached_sub = sub_utils.get_subscription(redis, client.installation["id"])
db_sub = sub_utils._retrieve_subscription_from_db(
client.installation["id"])
print("* SUBSCRIBED (cache/db): %s / %s" %
(cached_sub["subscription_active"], db_sub["subscription_active"]))
report_sub(client.installation["id"], slug, cached_sub, "ENGINE-CACHE")
report_sub(client.installation["id"], slug, db_sub, "DASHBOARD")
pull_raw = client.item(f"pulls/{pull_number}")
ctxt = mergify_context.MergifyContext(client, pull_raw)
print("* REPOSITORY IS %s" %
"PRIVATE" if ctxt.pull["base"]["repo"]["private"] else "PUBLIC")
print("* CONFIGURATION:")
try:
mergify_config_content = rules.get_mergify_config_content(ctxt)
except rules.NoRules: # pragma: no cover
print(".mergify.yml is missing")
pull_request_rules = None
else:
print(mergify_config_content.decode())
try:
mergify_config = rules.UserConfigurationSchema(
mergify_config_content)
except rules.InvalidRules as e: # pragma: no cover
print("configuration is invalid %s" % str(e))
else:
pull_request_rules_raw = mergify_config[
"pull_request_rules"].as_dict()
pull_request_rules_raw["rules"].extend(
engine.MERGIFY_RULE["rules"])
pull_request_rules = rules.PullRequestRules(
**pull_request_rules_raw)
print("* PULL REQUEST:")
pprint.pprint(ctxt.to_dict(), width=160)
print("is_behind: %s" % ctxt.is_behind)
print("mergeable_state: %s" % ctxt.pull["mergeable_state"])
print("* MERGIFY LAST CHECKS:")
checks = list(check_api.get_checks(ctxt, mergify_only=True))
for c in checks:
print("[%s]: %s | %s" %
(c["name"], c["conclusion"], c["output"].get("title")))
print("> " + "\n> ".join(c["output"].get("summary").split("\n")))
if pull_request_rules is not None:
print("* MERGIFY LIVE MATCHES:")
match = pull_request_rules.get_pull_request_rule(ctxt)
summary_title, summary = actions_runner.gen_summary(
ctxt, [{
"event_type": "refresh",
"data": {}
}], match)
print("> %s" % summary_title)
print(summary)
return ctxt
def handle_installation(installation):
try:
_id = installation["id"]
target_type = installation["target_type"]
account = installation["account"]["login"]
LOG.info("Get subscription", account=account)
subs = sub_utils.get_subscription(redis, _id)
subscribed = subs["subscription_active"]
costs[(subscribed, target_type, account)] = (
subs["subscription_cost"]
)
with installations_lock:
installations[(subscribed, target_type)] += 1
token = integration.get_access_token(_id).token
g = github.Github(token, base_url="https://api.%s" %
config.GITHUB_DOMAIN)
if installation["target_type"] == "Organization":
LOG.info("Get members",
install=installation["account"]["login"])
org = g.get_organization(installation["account"]["login"])
value = len(list(org.get_members()))
users_per_installation[
(subscribed, target_type, account)] = value
else:
users_per_installation[
(subscribed, target_type, account)] = 1
LOG.info("Get repos", account=account)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
configured_repos = 0
unconfigured_repos = 0
for repo in repos:
try:
rules.get_mergify_config(repo)
configured_repos += 1
redis.sadd("badges.tmp", repo.full_name)
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
unconfigured_repos += 1
except (rules.InvalidRules, rules.NoRules):
unconfigured_repos += 1
repositories_per_installation[
(subscribed, target_type, account, private, True)
] = configured_repos
repositories_per_installation[
(subscribed, target_type, account, private, False)
] = unconfigured_repos
except github.GithubException as e: # pragma: no cover
# Ignore rate limit/abuse, authorization issues
# and GitHub malfunction
if e.status not in (403, 401) and e.status < 500:
raise
def run(client, pull, sources):
LOG.debug("engine get sub")
subscription = sub_utils.get_subscription(
utils.get_redis_for_cache(), client.installation["id"]
)
LOG.debug("engine get context")
ctxt = context.Context(client, pull, subscription)
ctxt.log.debug("engine start processing context")
issue_comment_sources = []
for source in sources:
if source["event_type"] == "issue_comment":
issue_comment_sources.append(source)
else:
ctxt.sources.append(source)
ctxt.log.debug("engine run pending commands")
commands_runner.run_pending_commands_tasks(ctxt)
if issue_comment_sources:
ctxt.log.debug("engine handle commands")
for source in issue_comment_sources:
commands_runner.handle(
ctxt,
source["data"]["comment"]["body"],
source["data"]["comment"]["user"],
)
if not ctxt.sources:
return
if ctxt.client.installation["permissions_need_to_be_updated"]:
check_api.set_check_run(
ctxt,
"Summary",
"completed",
"failure",
output={
"title": "Required GitHub permissions are missing.",
"summary": "You can accept them at https://dashboard.mergify.io/",
},
)
return
ctxt.log.debug("engine check configuration change")
if check_configuration_changes(ctxt):
ctxt.log.info("Configuration changed, ignoring")
return
ctxt.log.debug("engine get configuration")
# BRANCH CONFIGURATION CHECKING
try:
filename, mergify_config = rules.get_mergify_config(ctxt)
except rules.NoRules: # pragma: no cover
ctxt.log.info("No need to proceed queue (.mergify.yml is missing)")
return
except rules.InvalidRules as e: # pragma: no cover
# Not configured, post status check with the error message
if any(
(
s["event_type"] == "pull_request"
and s["data"]["action"] in ["opened", "synchronize"]
for s in ctxt.sources
)
):
check_api.set_check_run(
ctxt,
actions_runner.SUMMARY_NAME,
"completed",
"failure",
output={
"title": "The Mergify configuration is invalid",
"summary": str(e),
"annotations": e.get_annotations(e.filename),
},
)
return
# Add global and mandatory rules
mergify_config["pull_request_rules"].rules.extend(
rules.PullRequestRules.from_list(MERGIFY_RULE["rules"]).rules
)
if ctxt.pull["base"]["repo"]["private"] and not subscription["subscription_active"]:
check_api.set_check_run(
ctxt,
actions_runner.SUMMARY_NAME,
"completed",
"failure",
output={
"title": "Mergify is disabled",
"summary": subscription["subscription_reason"],
},
)
return
# CheckRun are attached to head sha, so when user add commits or force push
# we can't directly get the previous Mergify Summary. So we copy it here, then
# anything that looks at it in next celery tasks will find it.
synchronize_data = [
s["data"]
for s in ctxt.sources
if s["event_type"] == "pull_request"
and s["data"]["action"] == "synchronize"
and s["data"]["after"] == ctxt.pull["head"]["sha"]
]
if synchronize_data:
ctxt.log.debug("engine synchronize summary")
copy_summary_from_previous_head_sha(ctxt, synchronize_data[0]["before"])
ctxt.log.debug("engine handle actions")
actions_runner.handle(mergify_config["pull_request_rules"], ctxt)
def update(pull, installation_id, method="merge"):
# NOTE(sileht):
# $ curl https://api.github.com/repos/sileht/repotest/pulls/2 | jq .commits
# 2
# $ git clone https://[email protected]/sileht-tester/repotest \
# --depth=$((2 + 1)) -b sileht/testpr
# $ cd repotest
# $ git remote add upstream https://[email protected]/sileht/repotest.git
# $ git log | grep Date | tail -1
# Date: Fri Mar 30 21:30:26 2018 (10 days ago)
# $ git fetch upstream master --shallow-since="Fri Mar 30 21:30:26 2018"
# $ git rebase upstream/master
# $ git push origin sileht/testpr:sileht/testpr
redis = utils.get_redis_for_cache()
subscription = sub_utils.get_subscription(redis, installation_id)
if not subscription:
LOG.error("subscription to update branch is missing")
return
token = subscription["token"]
head_repo = pull.g_pull.head.repo.full_name
base_repo = pull.g_pull.base.repo.full_name
head_branch = pull.g_pull.head.ref
base_branch = pull.g_pull.base.ref
git = utils.Gitter()
try:
git("init")
git.configure()
git.add_cred(token, "", head_repo)
git.add_cred(token, "", base_repo)
git("remote", "add", "origin",
"https://%s/%s" % (config.GITHUB_DOMAIN, head_repo))
git("remote", "add", "upstream",
"https://%s/%s" % (config.GITHUB_DOMAIN, base_repo))
depth = int(pull.g_pull.commits) + 1
git("fetch", "--quiet", "--depth=%d" % depth, "origin", head_branch)
git("checkout", "-q", "-b", head_branch, "origin/%s" % head_branch)
out = git("log", "--format=%cI")
last_commit_date = [d for d in out.decode("utf8").split("\n")
if d.strip()][-1]
git("fetch", "--quiet", "upstream", pull.g_pull.base.ref,
"--shallow-since='%s'" % last_commit_date)
if method == "merge":
git("merge", "--quiet", "upstream/%s" % base_branch, "-m",
"Merge branch '%s' into '%s'" % (base_branch, head_branch))
git("push", "--quiet", "origin", head_branch)
elif method == "rebase":
git("rebase", "upstream/%s" % base_branch)
git("push", "--quiet", "origin", head_branch, "-f")
else:
raise RuntimeError("Invalid branch update method")
return git("log", "-1", "--format=%H").decode().strip()
except Exception: # pragma: no cover
LOG.error("update branch fail", pull_request=pull, exc_info=True)
finally:
git.cleanup()
def run(event_type, data):
"""Everything starts here."""
installation_id = data["installation"]["id"]
installation_token = utils.get_installation_token(installation_id)
if not installation_token:
return
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
if config.LOG_RATELIMIT: # pragma: no cover
rate = g.get_rate_limit().rate
LOG.info(
"ratelimit: %s/%s, reset at %s",
rate.remaining,
rate.limit,
rate.reset,
repository=data["repository"]["name"],
)
try:
repo = g.get_repo(data["repository"]["owner"]["login"] + "/" +
data["repository"]["name"])
except github.UnknownObjectException: # pragma: no cover
LOG.info("Repository not found in the event %s, ignoring", event_type)
return
event_pull = get_github_pull_from_event(repo, event_type, data)
if not event_pull: # pragma: no cover
LOG.info("No pull request found in the event %s, "
"ignoring", event_type)
return
# Override pull_request with the updated one
data["pull_request"] = event_pull.raw_data
LOG.info(
"Pull request found in the event %s",
event_type,
repo=repo.full_name,
pull_request=event_pull,
)
if ("base" not in event_pull.raw_data
or "repo" not in event_pull.raw_data["base"]
or len(list(event_pull.raw_data["base"]["repo"].keys())) < 70):
LOG.warning(
"the pull request payload looks suspicious",
event_type=event_type,
data=data,
pull_request=event_pull.raw_data,
repo=repo.fullname,
)
if (event_type == "status"
and event_pull.head.sha != data["sha"]): # pragma: no cover
LOG.info(
"No need to proceed queue (got status of an old commit)",
repo=repo.full_name,
pull_request=event_pull,
)
return
elif (event_type in ["status", "check_suite", "check_run"]
and event_pull.merged): # pragma: no cover
LOG.info(
"No need to proceed queue (got status of a merged "
"pull request)",
repo=repo.full_name,
pull_request=event_pull,
)
return
elif (event_type in ["check_suite", "check_run"] and event_pull.head.sha !=
data[event_type]["head_sha"]): # pragma: no cover
LOG.info(
"No need to proceed queue (got %s of an old "
"commit)",
event_type,
repo=repo.full_name,
pull_request=event_pull,
)
return
if check_configuration_changes(event_pull):
LOG.info(
"Configuration changed, ignoring",
repo=repo.full_name,
pull_request=event_pull,
)
return
# BRANCH CONFIGURATION CHECKING
try:
mergify_config = rules.get_mergify_config(repo)
except rules.NoRules: # pragma: no cover
LOG.info(
"No need to proceed queue (.mergify.yml is missing)",
repo=repo.full_name,
pull_request=event_pull,
)
return
except rules.InvalidRules as e: # pragma: no cover
# Not configured, post status check with the error message
if event_type == "pull_request" and data["action"] in [
"opened", "synchronize"
]:
check_api.set_check_run(
event_pull,
"Summary",
"completed",
"failure",
output={
"title": "The Mergify configuration is invalid",
"summary": str(e),
},
)
return
subscription = sub_utils.get_subscription(utils.get_redis_for_cache(),
installation_id)
if repo.private and not subscription["subscription_active"]:
check_api.set_check_run(
event_pull,
"Summary",
"completed",
"failure",
output={
"title": "Mergify is disabled",
"summary": subscription["subscription_reason"],
},
)
return
# CheckRun are attached to head sha, so when user add commits or force push
# we can't directly get the previous Mergify Summary. So we copy it here, then
# anything that looks at it in next celery tasks will find it.
if event_type == "pull_request" and data["action"] == "synchronize":
copy_summary_from_previous_head_sha(event_pull, data["before"])
commands_runner.spawn_pending_commands_tasks(installation_id, event_type,
data, event_pull)
if event_type == "issue_comment":
commands_runner.run_command.s(installation_id, event_type, data,
data["comment"]["body"]).apply_async()
else:
actions_runner.handle.s(
installation_id,
mergify_config["pull_request_rules"].as_dict(),
event_type,
data,
).apply_async()
def report(url):
redis = utils.get_redis_for_cache()
path = url.replace("https://github.com/", "")
owner, repo, _, pull_number = path.split("/")
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
install_id = utils.get_installation_id(integration, owner, repo=repo)
print("* INSTALLATION ID: %s" % install_id)
cached_sub = sub_utils.get_subscription(redis, install_id)
db_sub = sub_utils._retrieve_subscription_from_db(install_id)
print("* SUBSCRIBED (cache/db): %s / %s" %
(cached_sub["subscription_active"], db_sub["subscription_active"]))
print("* SUB DETAIL: %s" % db_sub["subscription_reason"])
print("* NUMBER OF CACHED TOKENS: %d" % len(cached_sub["tokens"]))
try:
for login, token in cached_sub["tokens"].items():
try:
repos = get_repositories_setuped(token, install_id)
except github.BadCredentialsException:
print("** token for %s invalid" % login)
except github.GithubException as e:
if e.status != 401:
raise
print("** token for %s invalid" % login)
else:
if any((r["full_name"] == owner + "/" + repo) for r in repos):
print("* MERGIFY INSTALLED AND ENABLED ON THIS REPOSITORY")
else:
print("* MERGIFY INSTALLED BUT DISABLED "
"ON THIS REPOSITORY")
break
else:
print("* MERGIFY DOESN'T HAVE ANY VALID OAUTH TOKENS")
except github.UnknownObjectException:
print("* MERGIFY SEEMS NOT INSTALLED")
installation_token = integration.get_access_token(install_id).token
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
r = g.get_repo(owner + "/" + repo)
print("* REPOSITORY IS %s" % "PRIVATE" if r.private else "PUBLIC")
print("* CONFIGURATION:")
try:
mergify_config_content = rules.get_mergify_config_content(r)
except rules.NoRules: # pragma: no cover
print(".mergify.yml is missing")
print(mergify_config_content.decode())
try:
mergify_config = rules.UserConfigurationSchema(mergify_config_content)
except rules.InvalidRules as e: # pragma: no cover
print("configuration is invalid %s" % str(e))
else:
pull_request_rules_raw = mergify_config["pull_request_rules"].as_dict()
pull_request_rules_raw["rules"].extend(
actions_runner.MERGIFY_RULE["rules"])
pull_request_rules = rules.PullRequestRules(**pull_request_rules_raw)
try:
p = r.get_pull(int(pull_number))
except github.UnknownObjectException:
print("Wrong pull request number")
return g, None
mp = mergify_pull.MergifyPull(g, p, install_id)
print("* PULL REQUEST:")
pprint.pprint(mp.to_dict(), width=160)
try:
print("is_behind: %s" % mp.is_behind())
except github.GithubException as e:
print("Unable to know if pull request branch is behind: %s" % e)
print("mergeable_state: %s" % mp.g_pull.mergeable_state)
print("* MERGIFY LAST CHECKS:")
checks = list(check_api.get_checks(p))
for c in checks:
if c._rawData["app"]["id"] == config.INTEGRATION_ID:
print("[%s]: %s | %s" %
(c.name, c.conclusion, c.output.get("title")))
print("> " + "\n> ".join(c.output.get("summary").split("\n")))
print("* MERGIFY LIVE MATCHES:")
match = pull_request_rules.get_pull_request_rule(mp)
summary_title, summary = actions_runner.gen_summary(
"refresh", {}, mp, match)
print("> %s" % summary_title)
print(summary)
return g, p
def job_filter_and_dispatch(event_type, event_id, data):
if "installation" not in data:
subscription = {"subscription_active": "Unknown",
"subscription_reason": "No"}
else:
subscription = sub_utils.get_subscription(
utils.get_redis_for_cache(), data["installation"]["id"])
if "installation" not in data:
msg_action = "ignored (no installation id)"
elif not subscription["tokens"]:
msg_action = "ignored (no token)"
elif event_type == "installation" and data["action"] == "deleted":
# TODO(sileht): move out this engine V1 related code
key = "queues~%s~*~*~*~*" % data["installation"]["id"]
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif (event_type == "installation_repositories" and
data["action"] == "removed"):
for repository in data["repositories_removed"]:
# TODO(sileht): move out this engine V1 related code
key = "queues~%s~%s~%s~*~*" % (
data["installation"]["id"],
data["installation"]["account"]["login"].lower(),
repository["name"].lower()
)
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif event_type in ["installation", "installation_repositories"]:
msg_action = "ignored (action %s)" % data["action"]
elif event_type in ["pull_request", "pull_request_review", "status",
"check_suite", "check_run"]:
if data["repository"]["archived"]: # pragma: no cover
msg_action = "ignored (repository archived)"
elif event_type == "status" and data["state"] == "pending":
msg_action = "ignored (state pending)"
elif (event_type in ["check_run", "check_suite"] and
data[event_type]["app"]["id"] == config.INTEGRATION_ID and
data["action"] != "rerequested"):
msg_action = "ignored (mergify %s)" % event_type
elif (event_type == "pull_request" and data["action"] not in [
"opened", "reopened", "closed", "synchronize",
"labeled", "unlabeled", "edited", "locked", "unlocked"]):
msg_action = "ignored (action %s)" % data["action"]
else:
engine.run.s(event_type, data).apply_async()
msg_action = "pushed to backend"
if event_type == "pull_request":
msg_action += ", action: %s" % data["action"]
elif event_type == "pull_request_review":
msg_action += ", action: %s, review-state: %s" % (
data["action"], data["review"]["state"])
elif event_type == "pull_request_review_comment":
msg_action += ", action: %s, review-state: %s" % (
data["action"], data["comment"]["position"])
elif event_type == "status":
msg_action += ", ci-status: %s, sha: %s" % (
data["state"], data["sha"])
elif event_type in ["check_run", "check_suite"]:
msg_action += (
", action: %s, status: %s, conclusion: %s, sha: %s" % (
data["action"],
data[event_type]["status"],
data[event_type]["conclusion"],
data[event_type]["head_sha"]))
else:
msg_action = "ignored (unexpected event_type)"
if "repository" in data:
installation_id = data["installation"]["id"]
repo_name = data["repository"]["full_name"]
private = data["repository"]["private"]
elif "installation" in data:
installation_id = data["installation"]["id"],
repo_name = data["installation"]["account"]["login"]
private = "Unknown yet"
else:
installation_id = "Unknown"
repo_name = "Unknown"
private = "Unknown"
LOG.info('GithubApp event %s', msg_action,
event_type=event_type,
event_id=event_id,
install_id=installation_id,
sender=data["sender"]["login"],
repository=repo_name,
private=private,
subscription_active=subscription["subscription_active"],
subscription_reason=subscription["subscription_reason"])
def report(url):
redis = utils.get_redis_for_cache()
path = url.replace("https://github.com/", "")
owner, repo, _, pull_number = path.split("/")
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
install_id = utils.get_installation_id(integration, owner)
print("* INSTALLATION ID: %s" % install_id)
cached_sub = sub_utils.get_subscription(redis, install_id)
db_sub = sub_utils._retrieve_subscription_from_db(install_id)
print("* SUBSCRIBED (cache/db): %s / %s" %
(cached_sub["subscription_active"], db_sub["subscription_active"]))
print("* SUB DETAIL: %s" % db_sub["subscription_reason"])
print("* NUMBER OF CACHED TOKENS: %d" % len(cached_sub["tokens"]))
try:
for login, token in cached_sub["tokens"].items():
try:
repos = get_repositories_setuped(token, install_id)
except github.BadCredentialsException:
print("** token for %s invalid" % login)
except github.GithubException as e:
if e.status != 401:
raise
print("** token for %s invalid" % login)
else:
if any((r["full_name"] == owner + "/" + repo) for r in repos):
print("* MERGIFY INSTALLED AND ENABLED ON THIS REPOSITORY")
else:
print("* MERGIFY INSTALLED BUT DISABLED "
"ON THIS REPOSITORY")
break
else:
print("* MERGIFY DOESN'T HAVE ANY VALID OAUTH TOKENS")
except github.UnknownObjectException:
print("* MERGIFY SEEMS NOT INSTALLED")
installation_token = integration.get_access_token(install_id).token
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
r = g.get_repo(owner + "/" + repo)
print("* REPOSITORY IS %s" % "PRIVATE" if r.private else "PUBLIC")
p = r.get_pull(int(pull_number))
print("* CONFIGURATION:")
print(r.get_contents(".mergify.yml").decoded_content.decode())
mp = mergify_pull.MergifyPull(g, p, install_id)
print("* PULL REQUEST:")
pprint.pprint(mp.to_dict(), width=160)
try:
print("is_behind: %s" % mp.is_behind())
except github.GithubException as e:
print("Unable to know if pull request branch is behind: %s" % e)
print("mergeable_state: %s" % mp.g_pull.mergeable_state)
print("* MERGIFY STATUSES:")
commit = p.base.repo.get_commit(p.head.sha)
for s in commit.get_combined_status().statuses:
if s.context.startswith("mergify"):
print("[%s]: %s" % (s.context, s.state))
print("* MERGIFY CHECKS:")
checks = list(check_api.get_checks(p))
for c in checks:
if c._rawData['app']['id'] == config.INTEGRATION_ID:
print("[%s]: %s | %s" %
(c.name, c.conclusion, c.output.get("title")))
print("> " + "\n> ".join(c.output.get("summary").split("\n")))
return g, p
def collect_metrics():
redis = utils.get_redis_for_cache()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installations = collections.defaultdict(int)
repositories_per_installation = collections.defaultdict(int)
users_per_installation = collections.defaultdict(int)
LOG.info("GitHub Polling started")
redis.delete("badges.tmp")
for installation in utils.get_installations(integration):
try:
_id = installation["id"]
target_type = installation["target_type"]
account = installation["account"]["login"]
LOG.info("Get subscription", account=account)
subscribed = sub_utils.get_subscription(redis, _id)["subscribed"]
installations[(subscribed, target_type)] += 1
token = integration.get_access_token(_id).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
if installation["target_type"] == "Organization":
LOG.info("Get members",
install=installation["account"]["login"])
org = g.get_organization(installation["account"]["login"])
value = len(list(org.get_members()))
users_per_installation[(subscribed, target_type,
account)] = value
else:
users_per_installation[(subscribed, target_type, account)] = 1
LOG.info("Get repos", account=account)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
configured_repos = 0
unconfigured_repos = 0
for repo in repos:
try:
repo.get_contents(".mergify.yml")
configured_repos += 1
redis.sadd("badges.tmp", repo.full_name)
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
unconfigured_repos += 1
repositories_per_installation[(subscribed, target_type,
account, private,
True)] = configured_repos
repositories_per_installation[(subscribed, target_type,
account, private,
False)] = unconfigured_repos
except github.GithubException as e:
# Ignore rate limit/abuse
if e.status != 403:
raise
LOG.info("GitHub Polling finished")
# NOTE(sileht): Prometheus can scrape data during our loop. So make it fast
# to ensure we always have the good values.
# Also we can't known which labels we should delete from the Gauge,
# that's why we delete all of them to re-add them.
# And prometheus_client doesn't provide API to that, so we just
# override _metrics
set_gauges(INSTALLATIONS, installations)
set_gauges(USERS_PER_INSTALLATION, users_per_installation)
set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation)
if redis.exists("badges.tmp"):
redis.rename("badges.tmp", "badges")
LOG.info("Gauges and badges cache updated")
def run(event_type, data):
"""Everything starts here."""
installation_id = data["installation"]["id"]
installation_token = utils.get_installation_token(installation_id)
if not installation_token:
return
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
if config.LOG_RATELIMIT: # pragma: no cover
rate = g.get_rate_limit().rate
LOG.info("ratelimit: %s/%s, reset at %s",
rate.remaining, rate.limit, rate.reset,
repository=data["repository"]["name"])
repo = g.get_repo(data["repository"]["owner"]["login"] + "/" +
data["repository"]["name"])
event_pull = get_github_pull_from_event(repo, event_type, data)
if not event_pull: # pragma: no cover
LOG.info("No pull request found in the event %s, "
"ignoring", event_type)
return
LOG.info("Pull request found in the event %s", event_type,
repo=repo.full_name,
pull_request=event_pull)
subscription = sub_utils.get_subscription(utils.get_redis_for_cache(),
installation_id)
if repo.private and not subscription["subscription_active"]:
check_api.set_check_run(
event_pull, "Summary",
"completed", "failure", output={
"title": "Mergify is disabled",
"summary": subscription["subscription_reason"],
})
return
if ("base" not in event_pull.raw_data or
"repo" not in event_pull.raw_data["base"] or
len(list(event_pull.raw_data["base"]["repo"].keys())) < 70):
LOG.warning("the pull request payload looks suspicious",
event_type=event_type,
data=data,
pull_request=event_pull.raw_data,
repo=repo.fullname)
if (event_type == "status" and
event_pull.head.sha != data["sha"]): # pragma: no cover
LOG.info("No need to proceed queue (got status of an old commit)",
repo=repo.full_name,
pull_request=event_pull)
return
elif (event_type in ["status", "check_suite", "check_run"] and
event_pull.merged): # pragma: no cover
LOG.info("No need to proceed queue (got status of a merged "
"pull request)",
repo=repo.full_name,
pull_request=event_pull)
return
elif (event_type in ["check_suite", "check_run"] and
event_pull.head.sha != data[event_type]["head_sha"]
): # pragma: no cover
LOG.info("No need to proceed queue (got %s of an old "
"commit)", event_type,
repo=repo.full_name,
pull_request=event_pull)
return
if check_configuration_changes(event_pull):
LOG.info("Configuration changed, ignoring",
repo=repo.full_name,
pull_request=event_pull)
return
# BRANCH CONFIGURATION CHECKING
try:
mergify_config = rules.get_mergify_config(repo)
except rules.NoRules: # pragma: no cover
LOG.info("No need to proceed queue (.mergify.yml is missing)",
repo=repo.full_name,
pull_request=event_pull)
return
except rules.InvalidRules as e: # pragma: no cover
# Not configured, post status check with the error message
if (event_type == "pull_request" and
data["action"] in ["opened", "synchronize"]):
check_api.set_check_run(
event_pull, "Summary", "completed",
"failure", output={
"title": "The Mergify configuration is invalid",
"summary": str(e)
})
return
create_metrics(event_type, data)
v2.handle.s(
installation_id,
mergify_config["pull_request_rules"].as_dict(),
event_type, data, event_pull.raw_data
).apply_async()
def report(url):
path = url.replace("https://github.com/", "")
try:
owner, repo, _, pull_number = path.split("/")
except ValueError:
pull_number = None
try:
owner, repo = path.split("/")
except ValueError:
print(f"Wrong URL: {url}")
return
slug = owner + "/" + repo
try:
client = github.get_client(owner, repo)
except exceptions.MergifyNotInstalled:
print("* Mergify is not installed there")
return
print("* INSTALLATION ID: %s" % client.auth.installation["id"])
cached_sub, db_sub = utils.async_run(
sub_utils.get_subscription(client.auth.owner_id),
sub_utils._retrieve_subscription_from_db(client.auth.owner_id),
)
print("* SUBSCRIBED (cache/db): %s / %s" %
(cached_sub["subscription_active"], db_sub["subscription_active"]))
report_sub(client.auth.installation["id"], slug, cached_sub,
"ENGINE-CACHE")
report_sub(client.auth.installation["id"], slug, db_sub, "DASHBOARD")
repo = client.item(f"/repos/{owner}/{repo}")
print(f"* REPOSITORY IS {'PRIVATE' if repo['private'] else 'PUBLIC'}")
utils.async_run(report_worker_status(client.auth.owner))
if pull_number:
pull_raw = client.item(f"pulls/{pull_number}")
ctxt = context.Context(
client,
pull_raw,
cached_sub,
[{
"event_type": "mergify-debugger",
"data": {}
}],
)
q = queue.Queue.from_context(ctxt)
print("* QUEUES: %s" % ", ".join([f"#{p}" for p in q.get_pulls()]))
else:
for branch in client.items("branches"):
q = queue.Queue(
utils.get_redis_for_cache(),
client.auth.installation["id"],
client.auth.owner,
client.auth.repo,
branch["name"],
)
pulls = q.get_pulls()
if not pulls:
continue
print(f"* QUEUES {branch['name']}:")
for priority, grouped_pulls in itertools.groupby(
pulls, key=lambda v: q.get_config(v)["priority"]):
try:
fancy_priority = helpers.PriorityAliases(priority).name
except ValueError:
fancy_priority = priority
formatted_pulls = ", ".join((f"#{p}" for p in grouped_pulls))
print(f"** {formatted_pulls} (priority: {fancy_priority})")
print("* CONFIGURATION:")
try:
filename, mergify_config_content = rules.get_mergify_config_content(
client)
except rules.NoRules: # pragma: no cover
print(".mergify.yml is missing")
pull_request_rules = None
else:
print(f"Config filename: {filename}")
print(mergify_config_content.decode())
try:
mergify_config = rules.UserConfigurationSchema(
mergify_config_content)
except rules.InvalidRules as e: # pragma: no cover
print("configuration is invalid %s" % str(e))
else:
pull_request_rules_raw = mergify_config[
"pull_request_rules"].as_dict()
pull_request_rules = rules.PullRequestRules.from_list(
pull_request_rules_raw["rules"] + engine.MERGIFY_RULE["rules"])
if pull_number:
print("* PULL REQUEST:")
pr_data = dict(ctxt.pull_request.items())
pprint.pprint(pr_data, width=160)
print("is_behind: %s" % ctxt.is_behind)
print("mergeable_state: %s" % ctxt.pull["mergeable_state"])
print("* MERGIFY LAST CHECKS:")
for c in ctxt.pull_engine_check_runs:
print("[%s]: %s | %s" %
(c["name"], c["conclusion"], c["output"].get("title")))
print("> " + "\n> ".join(c["output"].get("summary").split("\n")))
if pull_request_rules is not None:
print("* MERGIFY LIVE MATCHES:")
match = pull_request_rules.get_pull_request_rule(ctxt)
summary_title, summary = actions_runner.gen_summary(ctxt, match)
print("> %s" % summary_title)
print(summary)
return ctxt
else:
return client