def job_refresh_all(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) i = g.get_installation(install["id"]) for r in i.get_repos(): if r.archived: # pragma: no cover continue try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: continue else: raise counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': install["id"]}, 'pull_request': p.raw_data, } engine.run.s('refresh', data).apply_async() LOG.info("Refreshing %s installations, %s repositories, " "%s branches", *counts)
def refresh_all(): authentification() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) subscription = utils.get_subscription(utils.get_redis_for_cache(), install["id"]) if not subscription["token"]: # pragma: no cover continue for r in i.get_repos(): if r.private and not subscription["subscribed"]: continue counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': install["id"]}, 'pull_request': p.raw_data, } get_queue().enqueue(worker.event_handler, "refresh", subscription, data) return ("Updated %s installations, %s repositories, " "%s branches" % tuple(counts)), 202
def job_refresh_all(): integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) counts = [0, 0, 0] for install in utils.get_installations(integration): counts[0] += 1 token = integration.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) subscription = utils.get_subscription(utils.get_redis_for_cache(), install["id"]) if not subscription["token"]: # pragma: no cover continue for r in i.get_repos(): if r.archived: # pragma: no cover continue if r.private and not subscription["subscribed"]: continue try: r.get_contents(".mergify.yml") except github.GithubException as e: # pragma: no cover if e.status == 404: continue else: raise counts[1] += 1 for p in list(r.get_pulls()): # Mimic the github event format data = { 'repository': r.raw_data, 'installation': { 'id': install["id"] }, 'pull_request': p.raw_data, } queue.route(r.full_name, subscription, "events", "refresh", subscription, data) LOG.info("Refreshing %s installations, %s repositories, " "%s branches" % tuple(counts))
def main(): # pragma: no cover integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installs = utils.get_installations(integration) r = utils.get_redis_for_cache() subscribed = 0 for i in installs: if utils.get_subscription(r, i["id"])["subscribed"]: subscribed += 1 print("installations: %d" % len(installs)) print("subscriptions: %d" % subscribed) active_slugs = set() for key in utils.get_redis_for_cache().keys("queues~*"): _, _, owner, repo, private, branch = key.split("~") active_slugs.add("%s/%s" % (owner, repo)) print("repositories with pending PRs: %d" % len(active_slugs)) print(" * %s" % "\n * ".join(sorted(active_slugs)))
def stargazer(): parser = argparse.ArgumentParser( description='Stargazers counter for mergify') parser.add_argument("-n", "--number", type=int, default=20, help="Number of repo to show") args = parser.parse_args() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) stars = [] for installation in utils.get_installations(integration): try: _id = installation["id"] token = integration.get_access_token(_id).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) repositories = sorted(g.get_installation(_id).get_repos(), key=operator.attrgetter("private")) for private, repos in itertools.groupby( repositories, key=operator.attrgetter("private")): for repo in repos: try: repo.get_contents(".mergify.yml") stars.append((repo.stargazers_count, repo.full_name)) except github.GithubException as e: if e.status >= 500: # pragma: no cover raise except github.GithubException as e: # Ignore rate limit/abuse if e.status != 403: raise for stars_count, repo in sorted(stars, reverse=True)[:args.number]: print("%s: %s", repo, stars_count)
def refresh_all(): authentification() counts = [0, 0, 0] for install in utils.get_installations(INTEGRATION): counts[0] += 1 token = INTEGRATION.get_access_token(install["id"]).token g = github.Github(token) i = g.get_installation(install["id"]) for r in i.get_repos(): counts[1] += 1 pulls = r.get_pulls() for p in pulls: # Mimic the github event format data = { 'repository': r.raw_data, 'installation': {'id': install["id"]}, 'pull_request': p.raw_data, } get_queue().enqueue(worker.event_handler, "refresh", data) return ("Updated %s installations, %s repositories, " "%s branches" % tuple(counts)), 202
def collect_metrics(): redis = utils.get_redis_for_cache() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installations = collections.defaultdict(int) repositories_per_installation = collections.defaultdict(int) users_per_installation = collections.defaultdict(int) LOG.info("GitHub Polling started") redis.delete("badges.tmp") for installation in utils.get_installations(integration): try: _id = installation["id"] target_type = installation["target_type"] account = installation["account"]["login"] LOG.info("Get subscription", account=account) subscribed = sub_utils.get_subscription(redis, _id)["subscribed"] installations[(subscribed, target_type)] += 1 token = integration.get_access_token(_id).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) if installation["target_type"] == "Organization": LOG.info("Get members", install=installation["account"]["login"]) org = g.get_organization(installation["account"]["login"]) value = len(list(org.get_members())) users_per_installation[(subscribed, target_type, account)] = value else: users_per_installation[(subscribed, target_type, account)] = 1 LOG.info("Get repos", account=account) repositories = sorted(g.get_installation(_id).get_repos(), key=operator.attrgetter("private")) for private, repos in itertools.groupby( repositories, key=operator.attrgetter("private")): configured_repos = 0 unconfigured_repos = 0 for repo in repos: try: repo.get_contents(".mergify.yml") configured_repos += 1 redis.sadd("badges.tmp", repo.full_name) except github.GithubException as e: if e.status >= 500: # pragma: no cover raise unconfigured_repos += 1 repositories_per_installation[(subscribed, target_type, account, private, True)] = configured_repos repositories_per_installation[(subscribed, target_type, account, private, False)] = unconfigured_repos except github.GithubException as e: # Ignore rate limit/abuse if e.status != 403: raise LOG.info("GitHub Polling finished") # NOTE(sileht): Prometheus can scrape data during our loop. So make it fast # to ensure we always have the good values. # Also we can't known which labels we should delete from the Gauge, # that's why we delete all of them to re-add them. # And prometheus_client doesn't provide API to that, so we just # override _metrics set_gauges(INSTALLATIONS, installations) set_gauges(USERS_PER_INSTALLATION, users_per_installation) set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation) if redis.exists("badges.tmp"): redis.rename("badges.tmp", "badges") LOG.info("Gauges and badges cache updated")
def collect_metrics(): redis = utils.get_redis_for_cache() integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installations_lock = threading.Lock() installations = collections.defaultdict(int) repositories_per_installation = collections.defaultdict(int) users_per_installation = collections.defaultdict(int) costs = collections.defaultdict(int) LOG.info("GitHub Polling started") redis.delete("badges.tmp") @tenacity.retry( retry=_exception_need_retry, wait=_wait_time_for_exception, stop=tenacity.stop_after_attempt(3), ) def handle_installation(installation): try: _id = installation["id"] target_type = installation["target_type"] account = installation["account"]["login"] LOG.info("Get subscription", account=account) subs = sub_utils.get_subscription(redis, _id) subscribed = subs["subscription_active"] costs[(subscribed, target_type, account)] = ( subs["subscription_cost"] ) with installations_lock: installations[(subscribed, target_type)] += 1 token = integration.get_access_token(_id).token g = github.Github(token, base_url="https://api.%s" % config.GITHUB_DOMAIN) if installation["target_type"] == "Organization": LOG.info("Get members", install=installation["account"]["login"]) org = g.get_organization(installation["account"]["login"]) value = len(list(org.get_members())) users_per_installation[ (subscribed, target_type, account)] = value else: users_per_installation[ (subscribed, target_type, account)] = 1 LOG.info("Get repos", account=account) repositories = sorted(g.get_installation(_id).get_repos(), key=operator.attrgetter("private")) for private, repos in itertools.groupby( repositories, key=operator.attrgetter("private")): configured_repos = 0 unconfigured_repos = 0 for repo in repos: try: rules.get_mergify_config(repo) configured_repos += 1 redis.sadd("badges.tmp", repo.full_name) except github.GithubException as e: if e.status >= 500: # pragma: no cover raise unconfigured_repos += 1 except (rules.InvalidRules, rules.NoRules): unconfigured_repos += 1 repositories_per_installation[ (subscribed, target_type, account, private, True) ] = configured_repos repositories_per_installation[ (subscribed, target_type, account, private, False) ] = unconfigured_repos except github.GithubException as e: # pragma: no cover # Ignore rate limit/abuse, authorization issues # and GitHub malfunction if e.status not in (403, 401) and e.status < 500: raise with futures.ThreadPoolExecutor() as executor: list(executor.map( handle_installation, utils.get_installations(integration) )) LOG.info("GitHub Polling finished") # NOTE(sileht): Prometheus can scrape data during our loop. So make it fast # to ensure we always have the good values. # Also we can't known which labels we should delete from the Gauge, # that's why we delete all of them to re-add them. # And prometheus_client doesn't provide API to that, so we just # override _metrics set_gauges(INSTALLATIONS, installations) set_gauges(USERS_PER_INSTALLATION, users_per_installation) set_gauges(REPOSITORIES_PER_INSTALLATION, repositories_per_installation) set_gauges(COSTS, costs) if redis.exists("badges.tmp"): redis.rename("badges.tmp", "badges") LOG.info("Gauges and badges cache updated")