def print_changeset_banner(changeset: ChangeSet) -> None:
ttywrite("%s %s\n (%s %sadditions%s, %s %sdeletions%s, %s %sreplacements%s, %s %sinfos%s)" %
(highlight("Changes from module"), changeset.source, changeset.additions, Fore.GREEN,
Fore.RESET, changeset.deletions, Fore.RED, Fore.RESET, changeset.replacements, Fore.YELLOW, Fore.RESET,
changeset.infos, Fore.LIGHTBLUE_EX, Fore.RESET))
cols, lines = shutil.get_terminal_size()
if changeset.description:
wrapper = ANSITextWrapper(width=cols - 1, initial_indent=" ", subsequent_indent=" ", expand_tabs=True,
tabsize=4)
ttywrite()
ttywrite(wrapper.fill(changeset.description))
ttywrite()
def execute_remove_all_status_checks(
change: Change[str], branch: Branch,
existing_checks: Set[str]) -> Change[str]:
print_debug("Removing all status checks from branch %s" %
highlight(branch.name))
try:
if existing_checks:
branch.remove_required_status_checks()
except GithubException as e:
print_error(str(e))
return change.failure()
else:
return change.success()
def execute_test_protection(change: Change[str], branch: Branch,
existing_checks: Set[str],
known_checks: Set[str]) -> Change[str]:
print_debug("[%s] Changing status checks on branch '%s' to [%s]" %
(highlight(repo.name), highlight(branch.name),
highlight(", ".join(list(known_checks)))))
try:
if existing_checks:
branch.edit_required_status_checks(strict=True,
contexts=list(known_checks))
else:
safe_branch_edit_protection(
branch,
strict=True,
contexts=list(known_checks),
)
except GithubException as e:
print_error(
"Can't edit required status checks on repo %s branch %s: %s" %
(repo.name, branch.name, str(e)))
return change.failure()
return change.success()
def apply_team_access(self, change: Change[str], repo: Repository,
role: str) -> Change[str]:
if change.action not in [ChangeActions.ADD, ChangeActions.REMOVE]:
return change.skipped()
if change.action == ChangeActions.REMOVE and change.before is not None:
if change.before in self.org_teams:
try:
self.org_teams[change.before].remove_from_repos(repo)
except GithubException as e:
print_error("Can't remove team %s from repo %s (%s)" %
(highlight(change.before), highlight(
repo.name), str(e)))
return change.failure()
return change.success()
else:
# the team was probably removed by another change
print_debug("Unknown team %s to remove from repo %s" %
(highlight(change.before), highlight(repo.name)))
return change.success()
elif change.action == ChangeActions.ADD and change.after is not None:
if change.after in self.org_teams:
try:
self.org_teams[change.after].set_repo_permission(
repo, role)
except GithubException as e:
print_error(
"Can't set permission %s for team %s on repo %s (%s)" %
(highlight(role), highlight(change.after),
highlight(repo.name), highlight(str(e))))
return change.failure()
return change.success()
else:
print_error("Unknown team %s to add to repo %s" %
(highlight(change.after), highlight(repo.name)))
return change.failure()
return change.success()
def apply_team_change(change: Change[Team],
org: Organization) -> Change[Team]:
if change.action not in [ChangeActions.ADD, ChangeActions.REMOVE]:
print_warning("Unsupported change action for teams: %s" %
change.action)
return change.skipped()
if change.action == ChangeActions.ADD and change.after is not None:
to_create = change.after # type: Team
if not isinstance(to_create, Team):
raise ErrorMessage("Create action without team to create")
created = org.create_team(to_create.name,
permission=to_create.default_permission,
privacy=to_create.privacy)
created.edit(created.name, description=to_create.description)
to_create.id = created.id
return change.success()
elif change.action == ChangeActions.REMOVE and change.before is not None:
try:
print_debug("Retrieving team id %s for deletion" %
highlight(str(change.before.id)))
to_delete = org.get_team(change.before.id) # type: GithubTeam
except GithubException:
# if the team is already gone... ok
return change.success()
try:
print_debug("Deleting team id %s" %
highlight(str(change.before.id)))
to_delete.delete()
except GithubException as e:
print_error("Can't delete team id %s: %s" %
(highlight(str(change.before.id)), str(e)))
return change.failure()
change.before.id = None
return change.success()
def execute_set_repo_features(
change: Change[str],
repo: Repository,
enable_wiki: Optional[bool] = None,
enable_issues: Optional[bool] = None,
enable_projects: Optional[bool] = None) -> Change[str]:
if change.action == ChangeActions.REPLACE:
print_debug("[%s] Setting features" % highlight(repo.name))
kw = {
'has_wiki':
NotSet if enable_wiki is None else enable_wiki,
'has_issues':
NotSet if enable_issues is None else enable_issues,
'has_projects':
NotSet if enable_projects is None else enable_projects
}
try:
repo.edit(**kw)
except GithubException:
return change.failure()
return change.success()
def assemble_changedict(args: Namespace,
org: Organization) -> Dict[str, ChangeSet]:
changedict = {}
if args.skip_org_changes:
print_warning("Skipping org changes (as per --no-org-changes)")
else:
pbar = None
if utils.enable_progressbar:
pbar = progressbar(len(modules))
for modulename, moduledef in modules.items(
): # type: str, GHConfModuleDef
if utils.enable_progressbar and pbar:
pbar.update()
try:
print_info("Building org changeset for %s" % modulename)
cslist = moduledef.build_organization_changesets(org)
for cs in cslist:
changedict.update(cs.todict())
except NotImplementedError:
print_debug(
"%s does not support creating an organization changeset. It might not modify the "
"org at all or it might just not report it." %
utils.highlight(modulename))
if pbar:
pbar.close()
capcache = {} # type: Dict[str, bool]
repolist = assemble_repolist(args, org)
pbar = None
repocount = len(repolist)
repofmt = "{{ix:>{len}}}/{count} Processing repo {{repo}}".format(
len=len(str(repocount)), count=str(repocount))
if utils.enable_progressbar:
pbar = progressbar(repocount)
for ix, repo in enumerate(repolist):
if utils.enable_progressbar:
pbar.update()
if utils.enable_verbose_output:
print_info(repofmt.format(ix=ix, repo=repo.full_name))
branches = list(repo.get_branches())
for modulename, moduledef in modules.items():
if not capcache.get(modulename, True):
print_debug(
"Capability cache for module %s indicates no support for repos"
% modulename)
continue
try:
print_debug("Building repo changeset for %s => %s" %
(modulename, repo.name))
cslist = moduledef.build_repository_changesets(
org, repo, branches)
for cs in cslist:
changedict.update(cs.todict())
except NotImplementedError:
print_debug(
"%s does not support creating a repo changeset for repo %s. It might just not "
"make any modifications at all or it might not report them."
%
(utils.highlight(modulename), utils.highlight(repo.name)))
capcache[modulename] = False
continue
pbar.close()
return changedict
def main() -> None:
module_parser = ArgumentParser(add_help=False)
module_parser.add_argument("-m",
"--module",
action="append",
dest="modules",
default=[],
help=SUPPRESS)
module_parser.add_argument("--debug",
action="store_true",
dest="debug",
default=False)
module_parser.add_argument("--no-color",
action="store_true",
dest="no_color",
default=False)
preargs, _ = module_parser.parse_known_args()
utils.enable_debug_output = preargs.debug
utils.init_color(preargs.no_color)
if preargs.modules:
import importlib
for module in preargs.modules: # type: str
if ":" in module:
module, entrypoint_name = module.split(":", 1)
else:
entrypoint_name = "entry_point"
try:
print_debug("Loading module %s:%s" % (module, entrypoint_name))
mod = importlib.import_module(module)
if hasattr(mod, entrypoint_name):
entrypoint = getattr(mod, entrypoint_name)
try:
i = iter(entrypoint)
mods = entrypoint
except TypeError:
mods = [entrypoint]
for ep in mods:
if isinstance(ep, type):
try:
modules["%s::%s" %
(module,
ep.__class__.__name__)] = ep()
except Exception as e:
raise utils.ErrorMessage(
"Unable to instantiate `entry_point` for module %s"
% module) from e
elif isinstance(ep, GHConfModuleDef):
modules["%s::%s" %
(module, ep.__class__.__name__)] = ep
else:
raise utils.ErrorMessage(
"Module entry point %s is neither an instance of GHConfModuleDef, "
"a list of GHConfModuleDef or a subclass of GHConfModuleDef."
% module)
else:
raise utils.ErrorMessage(
"Module %s has no `entry_point` top-level variable" %
module)
except ImportError as e:
raise utils.ErrorMessage(
"Can't import module %s (use --debug for more information)"
% module) from e
parser = ArgumentParser(
description=
"ghconf is a tool that parses declarative configuration files in a Python DSL "
"and then runs Python modules against a preconfigured PyGithub instance. This "
"allows us to apply common GitHub configuration through GitHub's v3 REST API "
"to all repositories that are part of our organization.")
parser.add_argument(
"-o",
"--organization",
dest="org",
default="optile",
help=
"The GitHub organization to run against. The GitHub API token must have write access to "
"this organization.")
parser.add_argument(
"-r",
"--repo",
dest="repos",
action="append",
default=[],
help=
"Specify one or more repositories to run the configuration against. (Optional. If not "
"specified, changes will be made to all repos in the org as modules see fit.)"
)
parser.add_argument(
"-re",
"--repo-regex",
dest="reporegexes",
action="append",
default=[],
help=
"Specify one or more regular expressions to match repositories to run the configuration "
"against. (Optional. If not specified, changes will be made to all repos in the org as "
"modules see fit.)")
parser.add_argument(
"--no-repo-changes",
dest="skip_repo_changes",
action="store_true",
default=False,
help="When set, ghconf will only execute org level changes.")
parser.add_argument(
"--no-org-changes",
dest="skip_org_changes",
action="store_true",
default=False,
help="When set, ghconf will not execute org level changes.")
parser.add_argument(
"--github-token",
dest="github_token",
default=None,
help=
"A GitHub API token for the user specified through '--github-user' to use for accessing "
"the GitHub API. (Envvar: GITHUB_TOKEN)")
parser.add_argument(
"--module",
dest="modules",
action="append",
default=[],
required=True,
help=
"Specify Python modules as configuration that will be imported by ghconf."
)
parser.add_argument("--debug",
dest="debug",
action="store_true",
default=False,
help="Enable debug output about API interactions.")
parser.add_argument(
"--list-unconfigured-repos",
dest="list_unconfigured",
action="store_true",
default=False,
help=
"List the names of all repositories that remain untouched with the current configuration"
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help=
"Verbose output. Include informational output, like objects that don't change."
)
parser.add_argument("--no-color",
dest="no_color",
action="store_true",
default=False,
help="Don't output ANSI colors")
parser.add_argument("--no-progressbar",
dest="no_progressbar",
action="store_true",
default=False,
help="Skip the progress bar")
parser.add_argument(
"--plan",
dest="plan",
action="store_true",
default=False,
help=
"Evaluate all changes and show what the tool would change with the current configuration."
)
parser.add_argument(
"--execute",
dest="execute",
action="store_true",
default=False,
help=
"Execute any detected changes without asking first. If this is not set, ghconf will ask "
"for permission before executing any changes.")
for modulename, moduledef in modules.items(): # type: str, GHConfModuleDef
try:
print_debug("Add args for %s" % modulename)
moduledef.add_args(parser)
except NotImplementedError:
pass
args = parser.parse_args(sys.argv[1:])
if args.verbose:
utils.enable_verbose_output = True
utils.enable_progressbar = not args.no_progressbar
if args.github_token:
ghcgithub.init_github(args.github_token, dry_run=args.plan)
elif os.getenv("GITHUB_TOKEN"):
ghcgithub.init_github(os.getenv("GITHUB_TOKEN"), dry_run=args.plan)
else:
raise utils.ErrorMessage(
"'--github-token' or environment variable GITHUB_TOKEN must be set"
)
for modulename, moduledef in modules.items():
try:
print_debug("Validate args for %s" % modulename)
moduledef.validate_args(args)
except NotImplementedError:
pass
try:
print_debug("Initialize GitHub API, load organization")
org = ghcgithub.gh.get_organization(args.org) # type: Organization
except GithubException:
raise utils.ErrorMessage(
"No such Github organization %s for the given API token" %
args.org)
if args.plan:
# banner
print_info("=" * (shutil.get_terminal_size()[0] - 15))
print_info("{{:^{width}}}".format(width=shutil.get_terminal_size()[0] -
15).format("Plan mode"))
print_info("=" * (shutil.get_terminal_size()[0] - 15))
###
print_changedict(assemble_changedict(args, org))
elif args.list_unconfigured:
print_info("=" * (shutil.get_terminal_size()[0] - 15))
print_info(
"{{:^{width}}}".format(width=shutil.get_terminal_size()[0] -
15).format("Unconfigured repositories"))
print_info("=" * (shutil.get_terminal_size()[0] - 15))
repolist = assemble_repolist(args, org)
pbar = progressbar(len(repolist) * len(modules))
for repo in repolist:
branches = list(repo.get_branches())
for modulename, moduledef in modules.items():
pbar.update()
try:
if moduledef.applies_to_repository(org, repo, branches):
repo.ghconf_touched = True
break
except NotImplementedError:
continue
if not hasattr(repo, "ghconf_touched"):
repo.ghconf_touched = False
for repo in repolist:
if not repo.ghconf_touched:
print_wrapped(repo.full_name)
pbar.close()
else:
# banner
print_info("=" * (shutil.get_terminal_size()[0] - 15))
print_info("{{:^{width}}}".format(width=shutil.get_terminal_size()[0] -
15).format(
utils.highlight("Execute mode")))
print_info("=" * (shutil.get_terminal_size()[0] - 15))
###
changedict = assemble_changedict(args, org)
if args.execute:
apply_changedict(changedict)
else:
print_changedict(changedict)
choice = prompt("Proceed and execute? [y/N] ",
choices=["y", "n"],
default="n")
if choice == "y":
apply_changedict(changedict)
else:
print_info("Execution cancelled")
def protect_pr_branch_with_tests_if_any_exist(
org: Organization, repo: Repository,
branches: Dict[str, Branch]) -> List[Change[str]]:
def execute_test_protection(change: Change[str], branch: Branch,
existing_checks: Set[str],
known_checks: Set[str]) -> Change[str]:
print_debug("[%s] Changing status checks on branch '%s' to [%s]" %
(highlight(repo.name), highlight(branch.name),
highlight(", ".join(list(known_checks)))))
try:
if existing_checks:
branch.edit_required_status_checks(strict=True,
contexts=list(known_checks))
else:
safe_branch_edit_protection(
branch,
strict=True,
contexts=list(known_checks),
)
except GithubException as e:
print_error(
"Can't edit required status checks on repo %s branch %s: %s" %
(repo.name, branch.name, str(e)))
return change.failure()
return change.success()
prb = get_pr_branch(repo, branches)
if not prb:
return []
existing_checks = set() # type: Set[str]
try:
rqs = prb.get_required_status_checks()
except GithubException:
# the repository has currently no status checks
pass
else:
if len(rqs.contexts) > 0:
# The repository already has some status checks, in that case we do nothing
existing_checks = set(rqs.contexts)
print_debug("Branch %s on repo %s already has status checks [%s]" %
(highlight(prb.name), highlight(
repo.name), highlight(", ".join(existing_checks))))
# the repository currently has no status checks, let's see if any came in within the last 7 days
sevendaysago = datetime.now() - timedelta(days=7)
known_checks = set() # type: Set[str]
for commit in repo.get_commits(prb.name, since=sevendaysago):
for status in commit.get_statuses(): # type: CommitStatus
known_checks.add(status.context)
if known_checks and known_checks != existing_checks:
# add all known checks as required checks
print_debug('Adding checks [%s] to branch %s on repo %s' %
(highlight(", ".join(known_checks)), highlight(
prb.name), highlight(repo.name)))
return [
Change(
meta=ChangeMetadata(
executor=execute_test_protection,
params=[prb, existing_checks, known_checks]),
action=ChangeActions.REPLACE
if existing_checks else ChangeActions.ADD,
before="%s checks" %
len(existing_checks) if existing_checks else "No checks",
after="%s checks" % len(known_checks),
)
]
return []
def _protect_branch(branch: Branch,
required_review_count: int) -> List[Change[str]]:
def execute_review_protection(
change: Change[str], branch: Branch,
existing_protection: Optional[BranchProtection],
review_count: int) -> Change[str]:
try:
if branch.protected and existing_protection and existing_protection.required_pull_request_reviews:
if review_count > 0:
print_debug(
"Replacing review protection on branch %s (%s reviews)"
% (highlight(branch.name), str(review_count)))
branch.edit_required_pull_request_reviews(
required_approving_review_count=review_count)
else:
print_debug("Removing review protection on branch: %s" %
highlight(branch.name))
branch.remove_required_pull_request_reviews()
elif review_count > 0:
print_debug(
"Adding review protection on branch: %s (%s reviews)" %
(highlight(branch.name), str(review_count)))
safe_branch_edit_protection(
branch, required_approving_review_count=review_count)
except GithubException as e:
print_error("Can't set review protection on branch %s to %s: %s" %
(highlight(branch.name), str(review_count), str(e)))
return change.failure()
return change.success()
change_needed = False
prot = None
current_reqcount = 0
# The Github API will gladly return a required review count > 0 for a branch that had a required review
# count previously, but it has now been turned off. So we need to correlate a bunch of information to find
# out whether the branch actually requires reviews or not.
if branch.protected:
prot = branch.get_protection()
if prot and prot.required_pull_request_reviews:
rpr = prot.required_pull_request_reviews # type: RequiredPullRequestReviews
if rpr.required_approving_review_count == required_review_count:
print_debug(
"Branch %s already requires %s reviews" % (highlight(
branch.name), highlight(str(required_review_count))))
change_needed = False
else:
current_reqcount = rpr.required_approving_review_count
change_needed = True
else:
if required_review_count == 0 and (
prot is None
or prot.required_pull_request_reviews is None):
print_debug(
"Branch %s required no review and requested count is %s" %
(highlight(branch.name), highlight("zero")))
change_needed = False
else:
change_needed = True
else:
change_needed = True
if change_needed:
change = Change(
meta=ChangeMetadata(executor=execute_review_protection,
params=[branch, prot, required_review_count]),
action=ChangeActions.REPLACE
if branch.protected else ChangeActions.ADD,
before="Require %s reviews" %
current_reqcount if branch.protected else "No protection",
after="Require %s reviews" % required_review_count,
cosmetic_prefix="Protect branch<%s>:" % branch.name)
return [change]
return []