def job_refresh_all():
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
counts = [0, 0, 0]
for install in utils.get_installations(integration):
counts[0] += 1
token = integration.get_access_token(install["id"]).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
i = g.get_installation(install["id"])
for r in i.get_repos():
if r.archived: # pragma: no cover
continue
try:
r.get_contents(".mergify.yml")
except github.GithubException as e: # pragma: no cover
if e.status == 404:
continue
else:
raise
counts[1] += 1
for p in list(r.get_pulls()):
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {'id': install["id"]},
'pull_request': p.raw_data,
}
engine.run.s('refresh', data).apply_async()
LOG.info("Refreshing %s installations, %s repositories, "
"%s branches", *counts)
def refresh_all():
authentification()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
counts = [0, 0, 0]
for install in utils.get_installations(integration):
counts[0] += 1
token = integration.get_access_token(install["id"]).token
g = github.Github(token)
i = g.get_installation(install["id"])
subscription = utils.get_subscription(utils.get_redis_for_cache(),
install["id"])
if not subscription["token"]: # pragma: no cover
continue
for r in i.get_repos():
if r.private and not subscription["subscribed"]:
continue
counts[1] += 1
for p in list(r.get_pulls()):
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {'id': install["id"]},
'pull_request': p.raw_data,
}
get_queue().enqueue(worker.event_handler, "refresh",
subscription, data)
return ("Updated %s installations, %s repositories, "
"%s branches" % tuple(counts)), 202
def job_refresh_all():
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
counts = [0, 0, 0]
for install in utils.get_installations(integration):
counts[0] += 1
token = integration.get_access_token(install["id"]).token
g = github.Github(token)
i = g.get_installation(install["id"])
subscription = utils.get_subscription(utils.get_redis_for_cache(),
install["id"])
if not subscription["token"]: # pragma: no cover
continue
for r in i.get_repos():
if r.archived: # pragma: no cover
continue
if r.private and not subscription["subscribed"]:
continue
try:
r.get_contents(".mergify.yml")
except github.GithubException as e: # pragma: no cover
if e.status == 404:
continue
else:
raise
counts[1] += 1
for p in list(r.get_pulls()):
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {
'id': install["id"]
},
'pull_request': p.raw_data,
}
queue.route(r.full_name, subscription, "events", "refresh",
subscription, data)
LOG.info("Refreshing %s installations, %s repositories, "
"%s branches" % tuple(counts))
def main(): # pragma: no cover
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installs = utils.get_installations(integration)
r = utils.get_redis_for_cache()
subscribed = 0
for i in installs:
if utils.get_subscription(r, i["id"])["subscribed"]:
subscribed += 1
print("installations: %d" % len(installs))
print("subscriptions: %d" % subscribed)
active_slugs = set()
for key in utils.get_redis_for_cache().keys("queues~*"):
_, _, owner, repo, private, branch = key.split("~")
active_slugs.add("%s/%s" % (owner, repo))
print("repositories with pending PRs: %d" % len(active_slugs))
print(" * %s" % "\n * ".join(sorted(active_slugs)))
def stargazer():
parser = argparse.ArgumentParser(
description='Stargazers counter for mergify')
parser.add_argument("-n",
"--number",
type=int,
default=20,
help="Number of repo to show")
args = parser.parse_args()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
stars = []
for installation in utils.get_installations(integration):
try:
_id = installation["id"]
token = integration.get_access_token(_id).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
for repo in repos:
try:
repo.get_contents(".mergify.yml")
stars.append((repo.stargazers_count, repo.full_name))
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
except github.GithubException as e:
# Ignore rate limit/abuse
if e.status != 403:
raise
for stars_count, repo in sorted(stars, reverse=True)[:args.number]:
print("%s: %s", repo, stars_count)
def refresh_all():
authentification()
counts = [0, 0, 0]
for install in utils.get_installations(INTEGRATION):
counts[0] += 1
token = INTEGRATION.get_access_token(install["id"]).token
g = github.Github(token)
i = g.get_installation(install["id"])
for r in i.get_repos():
counts[1] += 1
pulls = r.get_pulls()
for p in pulls:
# Mimic the github event format
data = {
'repository': r.raw_data,
'installation': {'id': install["id"]},
'pull_request': p.raw_data,
}
get_queue().enqueue(worker.event_handler, "refresh", data)
return ("Updated %s installations, %s repositories, "
"%s branches" % tuple(counts)), 202
def collect_metrics():
redis = utils.get_redis_for_cache()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installations = collections.defaultdict(int)
repositories_per_installation = collections.defaultdict(int)
users_per_installation = collections.defaultdict(int)
LOG.info("GitHub Polling started")
redis.delete("badges.tmp")
for installation in utils.get_installations(integration):
try:
_id = installation["id"]
target_type = installation["target_type"]
account = installation["account"]["login"]
LOG.info("Get subscription", account=account)
subscribed = sub_utils.get_subscription(redis, _id)["subscribed"]
installations[(subscribed, target_type)] += 1
token = integration.get_access_token(_id).token
g = github.Github(token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
if installation["target_type"] == "Organization":
LOG.info("Get members",
install=installation["account"]["login"])
org = g.get_organization(installation["account"]["login"])
value = len(list(org.get_members()))
users_per_installation[(subscribed, target_type,
account)] = value
else:
users_per_installation[(subscribed, target_type, account)] = 1
LOG.info("Get repos", account=account)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
configured_repos = 0
unconfigured_repos = 0
for repo in repos:
try:
repo.get_contents(".mergify.yml")
configured_repos += 1
redis.sadd("badges.tmp", repo.full_name)
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
unconfigured_repos += 1
repositories_per_installation[(subscribed, target_type,
account, private,
True)] = configured_repos
repositories_per_installation[(subscribed, target_type,
account, private,
False)] = unconfigured_repos
except github.GithubException as e:
# Ignore rate limit/abuse
if e.status != 403:
raise
LOG.info("GitHub Polling finished")
# NOTE(sileht): Prometheus can scrape data during our loop. So make it fast
# to ensure we always have the good values.
# Also we can't known which labels we should delete from the Gauge,
# that's why we delete all of them to re-add them.
# And prometheus_client doesn't provide API to that, so we just
# override _metrics
set_gauges(INSTALLATIONS, installations)
set_gauges(USERS_PER_INSTALLATION, users_per_installation)
set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation)
if redis.exists("badges.tmp"):
redis.rename("badges.tmp", "badges")
LOG.info("Gauges and badges cache updated")
def collect_metrics():
redis = utils.get_redis_for_cache()
integration = github.GithubIntegration(config.INTEGRATION_ID,
config.PRIVATE_KEY)
installations_lock = threading.Lock()
installations = collections.defaultdict(int)
repositories_per_installation = collections.defaultdict(int)
users_per_installation = collections.defaultdict(int)
costs = collections.defaultdict(int)
LOG.info("GitHub Polling started")
redis.delete("badges.tmp")
@tenacity.retry(
retry=_exception_need_retry,
wait=_wait_time_for_exception,
stop=tenacity.stop_after_attempt(3),
)
def handle_installation(installation):
try:
_id = installation["id"]
target_type = installation["target_type"]
account = installation["account"]["login"]
LOG.info("Get subscription", account=account)
subs = sub_utils.get_subscription(redis, _id)
subscribed = subs["subscription_active"]
costs[(subscribed, target_type, account)] = (
subs["subscription_cost"]
)
with installations_lock:
installations[(subscribed, target_type)] += 1
token = integration.get_access_token(_id).token
g = github.Github(token, base_url="https://api.%s" %
config.GITHUB_DOMAIN)
if installation["target_type"] == "Organization":
LOG.info("Get members",
install=installation["account"]["login"])
org = g.get_organization(installation["account"]["login"])
value = len(list(org.get_members()))
users_per_installation[
(subscribed, target_type, account)] = value
else:
users_per_installation[
(subscribed, target_type, account)] = 1
LOG.info("Get repos", account=account)
repositories = sorted(g.get_installation(_id).get_repos(),
key=operator.attrgetter("private"))
for private, repos in itertools.groupby(
repositories, key=operator.attrgetter("private")):
configured_repos = 0
unconfigured_repos = 0
for repo in repos:
try:
rules.get_mergify_config(repo)
configured_repos += 1
redis.sadd("badges.tmp", repo.full_name)
except github.GithubException as e:
if e.status >= 500: # pragma: no cover
raise
unconfigured_repos += 1
except (rules.InvalidRules, rules.NoRules):
unconfigured_repos += 1
repositories_per_installation[
(subscribed, target_type, account, private, True)
] = configured_repos
repositories_per_installation[
(subscribed, target_type, account, private, False)
] = unconfigured_repos
except github.GithubException as e: # pragma: no cover
# Ignore rate limit/abuse, authorization issues
# and GitHub malfunction
if e.status not in (403, 401) and e.status < 500:
raise
with futures.ThreadPoolExecutor() as executor:
list(executor.map(
handle_installation, utils.get_installations(integration)
))
LOG.info("GitHub Polling finished")
# NOTE(sileht): Prometheus can scrape data during our loop. So make it fast
# to ensure we always have the good values.
# Also we can't known which labels we should delete from the Gauge,
# that's why we delete all of them to re-add them.
# And prometheus_client doesn't provide API to that, so we just
# override _metrics
set_gauges(INSTALLATIONS, installations)
set_gauges(USERS_PER_INSTALLATION, users_per_installation)
set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation)
set_gauges(COSTS, costs)
if redis.exists("badges.tmp"):
redis.rename("badges.tmp", "badges")
LOG.info("Gauges and badges cache updated")