def refresh(owner, repo, refresh_ref):
authentification()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installation_id = utils.get_installation_id(integration, owner)
if not installation_id: # pragma: no cover
flask.abort(400, "%s have not installed mergify_engine" % owner)
token = integration.get_access_token(installation_id).token
g = github.Github(token)
r = g.get_repo("%s/%s" % (owner, repo))
try:
r.get_contents(".mergify.yml")
except github.GithubException as e: # pragma: no cover
if e.status == 404:
return "No .mergify.yml", 202
else:
raise
if refresh_ref == "full" or refresh_ref.startswith("branch/"):
if refresh_ref.startswith("branch/"):
branch = refresh_ref[7:]
pulls = r.get_pulls(base=branch)
else:
branch = '*'
pulls = r.get_pulls()
key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(),
repo.lower(), r.private, branch)
utils.get_redis_for_cache().delete(key)
else:
try:
pull_number = int(refresh_ref[5:])
except ValueError: # pragma: no cover
return "Invalid PR ref", 400
pulls = [r.get_pull(pull_number)]
subscription = utils.get_subscription(utils.get_redis_for_cache(),
installation_id)
if not subscription["token"]: # pragma: no cover
return "", 202
if r.private and not subscription["subscribed"]: # pragma: no cover
return "", 202
for p in pulls:
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {
'id': installation_id
},
'pull_request': p.raw_data,
}
get_queue(r.full_name,
subscription).enqueue(worker.event_handler, "refresh",
subscription, data)
return "", 202
def refresh_all():
authentification()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
counts = [0, 0, 0]
for install in utils.get_installations(integration):
counts[0] += 1
token = integration.get_access_token(install["id"]).token
g = github.Github(token)
i = g.get_installation(install["id"])
subscription = utils.get_subscription(utils.get_redis_for_cache(),
install["id"])
if not subscription["token"]: # pragma: no cover
continue
for r in i.get_repos():
if r.private and not subscription["subscribed"]:
continue
counts[1] += 1
for p in list(r.get_pulls()):
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {'id': install["id"]},
'pull_request': p.raw_data,
}
get_queue().enqueue(worker.event_handler, "refresh",
subscription, data)
return ("Updated %s installations, %s repositories, "
"%s branches" % tuple(counts)), 202
def smart_strict_workflow_periodic_task():
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
redis = utils.get_redis_for_cache()
LOG.debug("smart strict workflow loop start")
for key in redis.keys("strict-merge-queues~*"):
_, installation_id, owner, reponame, branch = key.split("~")
try:
installation_token = integration.get_access_token(
installation_id).token
except github.UnknownObjectException: # pragma: no cover
LOG.error("token for install %d does not exists anymore (%s/%s)",
installation_id, owner, reponame)
continue
cur_key = "current-%s" % key
redis = utils.get_redis_for_cache()
pull_number = redis.get(cur_key)
if pull_number and redis.sismember(key, pull_number):
pull = mergify_pull.MergifyPull.from_number(
installation_id, installation_token, owner, reponame,
int(pull_number))
if pull.g_pull.state == "closed" or pull.is_behind():
# NOTE(sileht): Someone can have merged something manually in
# base branch in the meantime, so we have to update it again.
LOG.debug(
"pull request needs to be updated again or "
"has been closed",
installation_id=installation_id,
pull_number=pull_number,
repo=owner + "/" + reponame,
branch=branch)
else:
# NOTE(sileht): Pull request has not been merged or cancelled
# yet wait next loop
LOG.debug("pull request checks are still in progress",
installation_id=installation_id,
pull_number=pull_number,
repo=owner + "/" + reponame,
branch=branch)
continue
subscription = utils.get_subscription(redis, installation_id)
if not subscription["token"]: # pragma: no cover
LOG.error("no subscription token for updating base branch",
installation_id=installation_id,
repo=owner + "/" + reponame,
branch=branch)
continue
# NOTE(sileht): Pick up the next pull request and rebase it
update_next_pull(installation_id, installation_token, subscription,
owner, reponame, branch, key, cur_key)
LOG.debug("smart strict workflow loop end")
def subscription_cache(installation_id): # pragma: no cover
authentification()
r = utils.get_redis_for_cache()
r.delete("subscription-cache-%s" % installation_id)
subscription = utils.get_subscription(
utils.get_redis_for_cache(), installation_id)
# New subscription, create initial configuration for private repo
# public repository have already been done during the installation
# event.
if subscription["token"] and subscription["subscribed"]:
get_queue().enqueue(worker.installation_handler,
installation_id, "private")
return "Cache cleaned", 200
def job_refresh_all():
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
counts = [0, 0, 0]
for install in utils.get_installations(integration):
counts[0] += 1
token = integration.get_access_token(install["id"]).token
g = github.Github(token)
i = g.get_installation(install["id"])
subscription = utils.get_subscription(utils.get_redis_for_cache(),
install["id"])
if not subscription["token"]: # pragma: no cover
continue
for r in i.get_repos():
if r.archived: # pragma: no cover
continue
if r.private and not subscription["subscribed"]:
continue
try:
r.get_contents(".mergify.yml")
except github.GithubException as e: # pragma: no cover
if e.status == 404:
continue
else:
raise
counts[1] += 1
for p in list(r.get_pulls()):
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {
'id': install["id"]
},
'pull_request': p.raw_data,
}
queue.route(r.full_name, subscription, "events", "refresh",
subscription, data)
LOG.info("Refreshing %s installations, %s repositories, "
"%s branches" % tuple(counts))
def main(): # pragma: no cover
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installs = utils.get_installations(integration)
r = utils.get_redis_for_cache()
subscribed = 0
for i in installs:
if utils.get_subscription(r, i["id"])["subscribed"]:
subscribed += 1
print("installations: %d" % len(installs))
print("subscriptions: %d" % subscribed)
active_slugs = set()
for key in utils.get_redis_for_cache().keys("queues~*"):
_, _, owner, repo, private, branch = key.split("~")
active_slugs.add("%s/%s" % (owner, repo))
print("repositories with pending PRs: %d" % len(active_slugs))
print(" * %s" % "\n * ".join(sorted(active_slugs)))
def event_handler():
authentification()
event_type = flask.request.headers.get("X-GitHub-Event")
event_id = flask.request.headers.get("X-GitHub-Delivery")
data = flask.request.get_json()
subscription = utils.get_subscription(
utils.get_redis_for_cache(), data["installation"]["id"])
if not subscription["token"]:
msg_action = "ignored (no token)"
elif event_type == "installation" and data["action"] == "created":
for repository in data["repositories"]:
if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover
continue
get_queue().enqueue(worker.installation_handler,
data["installation"]["id"], [repository])
msg_action = "pushed to backend"
elif event_type == "installation" and data["action"] == "deleted":
key = "queues~%s~*~*~*~*" % data["installation"]["id"]
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif (event_type == "installation_repositories" and
data["action"] == "added"):
for repository in data["repositories_added"]:
if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover
continue
get_queue().enqueue(worker.installation_handler,
data["installation"]["id"], [repository])
msg_action = "pushed to backend"
elif (event_type == "installation_repositories" and
data["action"] == "removed"):
for repository in data["repositories_removed"]:
if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover
continue
key = "queues~%s~%s~%s~*~*" % (
data["installation"]["id"],
data["installation"]["account"]["login"].lower(),
repository["name"].lower()
)
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif event_type in ["installation", "installation_repositories"]:
msg_action = "ignored (action %s)" % data["action"]
elif event_type in ["pull_request", "pull_request_review", "status",
"refresh"]:
if data["repository"]["private"] and not subscription["subscribed"]:
msg_action = "ignored (not public or subscribe)"
elif event_type == "status" and data["state"] == "pending":
msg_action = "ignored (state pending)"
elif (event_type == "pull_request" and data["action"] not in [
"opened", "reopened", "closed", "synchronize",
"labeled", "unlabeled"]):
msg_action = "ignored (action %s)" % data["action"]
else:
get_queue().enqueue(worker.event_handler, event_type,
subscription, data)
msg_action = "pushed to backend"
else:
msg_action = "ignored (unexpected event_type)"
if "repository" in data:
repo_name = data["repository"]["full_name"]
else:
repo_name = data["installation"]["account"]["login"]
LOG.info('[%s/%s] received "%s" event "%s", %s',
data["installation"]["id"], repo_name,
event_type, event_id, msg_action)
return "", 202
def job_filter_and_dispatch(event_type, event_id, data):
subscription = utils.get_subscription(utils.get_redis_for_cache(),
data["installation"]["id"])
if not subscription["token"]:
msg_action = "ignored (no token)"
elif event_type == "installation" and data["action"] == "created":
for repository in data["repositories"]:
if repository["private"] and not subscription[
"subscribed"]: # noqa pragma: no cover
continue
queue.route(repository["full_name"], subscription,
"installations", data["installation"]["id"],
[repository])
msg_action = "pushed to backend"
elif event_type == "installation" and data["action"] == "deleted":
key = "queues~%s~*~*~*~*" % data["installation"]["id"]
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif (event_type == "installation_repositories"
and data["action"] == "added"):
for repository in data["repositories_added"]:
if repository["private"] and not subscription[
"subscribed"]: # noqa pragma: no cover
continue
queue.route(repository["full_name"], subscription,
"installations", data["installation"]["id"],
[repository])
msg_action = "pushed to backend"
elif (event_type == "installation_repositories"
and data["action"] == "removed"):
for repository in data["repositories_removed"]:
if repository["private"] and not subscription[
"subscribed"]: # noqa pragma: no cover
continue
key = "queues~%s~%s~%s~*~*" % (
data["installation"]["id"],
data["installation"]["account"]["login"].lower(),
repository["name"].lower())
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif event_type in ["installation", "installation_repositories"]:
msg_action = "ignored (action %s)" % data["action"]
elif event_type in ["pull_request", "pull_request_review", "status"]:
if data["repository"]["archived"]: # pragma: no cover
msg_action = "ignored (repository archived)"
elif (data["repository"]["private"]
and not subscription["subscribed"]):
msg_action = "ignored (not public or subscribe)"
elif event_type == "status" and data["state"] == "pending":
msg_action = "ignored (state pending)"
elif event_type == "status" and data["context"] == "mergify/pr":
msg_action = "ignored (mergify status)"
elif (event_type == "pull_request" and data["action"] not in [
"opened", "reopened", "closed", "synchronize", "labeled",
"unlabeled"
]):
msg_action = "ignored (action %s)" % data["action"]
else:
queue.route(data["repository"]["full_name"], subscription,
"events", event_type, subscription, data)
msg_action = "pushed to backend"
else:
msg_action = "ignored (unexpected event_type)"
if "repository" in data:
repo_name = data["repository"]["full_name"]
else:
repo_name = data["installation"]["account"]["login"]
LOG.info('event %s',
msg_action,
event_type=event_type,
event_id=event_id,
install_id=data["installation"]["id"],
repository=repo_name,
subscribed=subscription["subscribed"])
def job_refresh(owner, repo, refresh_ref):
LOG.info("%s/%s/%s: refreshing" % (owner, repo, refresh_ref))
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installation_id = utils.get_installation_id(integration, owner)
if not installation_id: # pragma: no cover
LOG.warning("%s/%s/%s: mergify not installed" %
(owner, repo, refresh_ref))
return
token = integration.get_access_token(installation_id).token
g = github.Github(token)
r = g.get_repo("%s/%s" % (owner, repo))
try:
r.get_contents(".mergify.yml")
except github.GithubException as e: # pragma: no cover
if e.status == 404:
LOG.warning("%s/%s/%s: mergify not configured" %
(owner, repo, refresh_ref))
return
else:
raise
if refresh_ref == "full" or refresh_ref.startswith("branch/"):
if refresh_ref.startswith("branch/"):
branch = refresh_ref[7:]
pulls = r.get_pulls(base=branch)
else:
branch = '*'
pulls = r.get_pulls()
key = "queues~%s~%s~%s~%s~%s" % (installation_id, owner.lower(),
repo.lower(), r.private, branch)
utils.get_redis_for_cache().delete(key)
else:
try:
pull_number = int(refresh_ref[5:])
except ValueError: # pragma: no cover
LOG.info("%s/%s/%s: Invalid PR ref" %
(owner, repo, refresh_ref))
return
pulls = [r.get_pull(pull_number)]
subscription = utils.get_subscription(utils.get_redis_for_cache(),
installation_id)
if r.archived: # pragma: no cover
LOG.warning("%s/%s/%s: repository archived" %
(owner, repo, refresh_ref))
return
if not subscription["token"]: # pragma: no cover
LOG.warning("%s/%s/%s: not public or subscribed" %
(owner, repo, refresh_ref))
return
if r.private and not subscription["subscribed"]: # pragma: no cover
LOG.warning("%s/%s/%s: mergify not installed" %
(owner, repo, refresh_ref))
return
for p in pulls:
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {
'id': installation_id
},
'pull_request': p.raw_data,
}
queue.route(r.full_name, subscription, "events", "refresh",
subscription, data)
def collect_metrics():
redis = utils.get_redis_for_cache()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installations = collections.defaultdict(int)
repositories_per_installation = collections.defaultdict(int)
users_per_installation = collections.defaultdict(int)
LOG.info("GitHub Polling started")
redis.delete("badges.tmp")
for installation in utils.get_installations(integration):
try:
_id = installation["id"]
target_type = installation["target_type"]
account = installation["account"]["login"]
LOG.info("Get subscription", account=account)
subscribed = utils.get_subscription(redis, _id)["subscribed"]
installations[(subscribed, target_type)] += 1
token = integration.get_access_token(_id).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
if installation[
"target_type"] == "Organization": # pragma: no cover
LOG.info("Get members",
install=installation["account"]["login"])
org = g.get_organization(installation["account"]["login"])
value = len(list(org.get_members()))
users_per_installation[(subscribed, target_type,
account)] = value
else:
users_per_installation[(subscribed, target_type, account)] = 1
LOG.info("Get repos", account=account)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
configured_repos = 0
unconfigured_repos = 0
for repo in repos:
try:
repo.get_contents(".mergify.yml")
configured_repos += 1
redis.sadd("badges.tmp", repo.full_name)
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
unconfigured_repos += 1
repositories_per_installation[(subscribed, target_type,
account, private,
True)] = configured_repos
repositories_per_installation[(subscribed, target_type,
account, private,
False)] = unconfigured_repos
except github.GithubException as e:
# Ignore rate limit/abuse
if e.status != 403:
raise
LOG.info("GitHub Polling finished")
# NOTE(sileht): Prometheus can scrape data during our loop. So make it fast
# to ensure we always have the good values.
# Also we can't known which labels we should delete from the Gauge,
# that's why we delete all of them to re-add them.
# And prometheus_client doesn't provide API to that, so we just
# override _metrics
set_gauges(INSTALLATIONS, installations)
set_gauges(USERS_PER_INSTALLATION, users_per_installation)
set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation)
if redis.exists("badges.tmp"):
redis.rename("badges.tmp", "badges")
LOG.info("Gauges and badges cache updated")
def job_filter_and_dispatch(event_type, event_id, data):
subscription = utils.get_subscription(
utils.get_redis_for_cache(), data["installation"]["id"])
if not subscription["token"]:
msg_action = "ignored (no token)"
elif event_type == "installation" and data["action"] == "deleted":
# TODO(sileht): move out this engine V1 related code
key = "queues~%s~*~*~*~*" % data["installation"]["id"]
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif (event_type == "installation_repositories" and
data["action"] == "removed"):
for repository in data["repositories_removed"]:
if repository["private"] and not subscription["subscribed"]: # noqa pragma: no cover
continue
# TODO(sileht): move out this engine V1 related code
key = "queues~%s~%s~%s~*~*" % (
data["installation"]["id"],
data["installation"]["account"]["login"].lower(),
repository["name"].lower()
)
utils.get_redis_for_cache().delete(key)
msg_action = "handled, cache cleaned"
elif event_type in ["installation", "installation_repositories"]:
msg_action = "ignored (action %s)" % data["action"]
elif event_type in ["pull_request", "pull_request_review", "status",
"check_suite", "check_run"]:
if data["repository"]["archived"]: # pragma: no cover
msg_action = "ignored (repository archived)"
elif (data["repository"]["private"] and not
subscription["subscribed"]):
msg_action = "ignored (not public or subscribe)"
elif event_type == "status" and data["state"] == "pending":
msg_action = "ignored (state pending)"
elif event_type == "status" and data["context"] == "mergify/pr":
msg_action = "ignored (mergify status)"
elif (event_type in ["check_run", "check_suite"] and
data[event_type]["app"]["id"] == config.INTEGRATION_ID):
msg_action = "ignored (mergify %s)" % event_type
elif (event_type == "pull_request" and data["action"] not in [
"opened", "reopened", "closed", "synchronize",
"labeled", "unlabeled", "edited"]):
msg_action = "ignored (action %s)" % data["action"]
else:
engine.run.s(event_type, data, subscription).apply_async()
msg_action = "pushed to backend"
if event_type == "pull_request":
msg_action += ", action: %s" % data["action"]
elif event_type == "pull_request_review":
msg_action += ", action: %s, review-state: %s" % (
data["action"], data["review"]["state"])
elif event_type == "pull_request_review_comment":
msg_action += ", action: %s, review-state: %s" % (
data["action"], data["comment"]["position"])
elif event_type == "status":
msg_action += ", ci-status: %s, sha: %s" % (
data["state"], data["sha"])
elif event_type in ["check_run", "check_suite"]:
msg_action += (
", action: %s, status: %s, conclusion: %s, sha: %s" % (
data["action"],
data[event_type]["status"],
data[event_type]["conclusion"],
data[event_type]["head_sha"]))
else:
msg_action = "ignored (unexpected event_type)"
if "repository" in data:
repo_name = data["repository"]["full_name"]
else:
repo_name = data["installation"]["account"]["login"]
LOG.info('event %s', msg_action,
event_type=event_type,
event_id=event_id,
install_id=data["installation"]["id"],
sender=data["sender"]["login"],
repository=repo_name,
subscribed=subscription["subscribed"])
