def remove_ref_for_repos(repos, ref, use_tag=True, dry=True):
"""
Given an iterable of repository full names (like "edx/edx-platform") and
a tag name, this function attempts to delete the named ref from each
GitHub repository listed in the iterable. If the ref does not exist on
the repo, it is skipped.
This function returns True if any repos had the reference removed,
or False if no repos were modified. If an error occurs while trying
to remove a ref from a repo, the function will continue trying to
remove refs from all the other repos in the iterable -- but after all repos
have been attempted, this function will raise a RuntimeError with
a list of all the repos that did not have the ref removed.
Trying to remove a ref from a repo that does not have that ref
to begin with is *not* treated as an error.
"""
if ref.startswith('refs/'):
ref = ref[len('refs/'):]
if not (ref.startswith("heads/") or ref.startswith('tags/')):
ref = "{type}/{name}".format(
type="tags" if use_tag else "heads",
name=ref,
)
failures = {}
modified = False
for repo in repos:
try:
ref_obj = repo.ref(ref)
if ref_obj is None:
# tag didn't exist to begin with; not an error
continue
dry_echo(
dry,
u'Deleting ref {} from repo {}'.format(
ref_obj.ref, repo.full_name
),
fg='red'
)
if not dry:
ref_obj.delete()
modified = True
except GitHubError as err:
# Oops, we got a failure. Record it and move on.
failures[repo.full_name] = err
if failures:
msg = (
u"Failed to remove the ref from the following repos: {repos}"
).format(
repos=", ".join(failures.keys())
)
err = RuntimeError(msg)
err.failures = failures
raise err
return modified
def main(hub, ref, use_tag, override_ref, overrides, interactive, quiet,
reverse, skip_invalid, input_repos, output_repos, included_repos,
skip_repos, dry, orgs, branches):
"""Create/remove tags & branches on GitHub repos for Open edX releases."""
if input_repos:
with open(input_repos) as f:
repos = {
ShortRepository.from_dict(r["repo"], hub): r["data"]
for r in json.load(f)
}
else:
repos = openedx_release_repos(hub, orgs, branches)
if output_repos:
with open(output_repos, "w") as f:
dumped = [{
"repo": repo.as_dict(),
"data": data
} for repo, data in repos.items()]
json.dump(dumped, f, indent=2, sort_keys=True)
if not repos:
raise ValueError(
"No repos marked for openedx-release in their openedx.yaml files!")
if included_repos:
repos = include_only_repos(repos, included_repos)
repos = trim_skipped_repos(repos, skip_repos)
repos = trim_dependent_repos(repos)
repos = trim_indecisive_repos(repos)
repos = override_repo_refs(
repos,
override_ref=override_ref,
overrides=dict(overrides or ()),
)
archived = archived_repos(repos.keys())
if archived:
if dry:
dry_echo(
dry, "Will need to unarchive these repos: {}".format(", ".join(
repo.full_name for repo in archived)))
else:
ensure_writable(archived)
try:
ret = do_the_work(repos, ref, use_tag, reverse, skip_invalid,
interactive, quiet, dry)
finally:
for repo in archived:
dry_echo(dry, f"Re-archiving {repo.full_name}")
if not dry:
repo.edit(repo.name, archived=True)
return ret
def set_or_delete_labels(dry, repo, new_labels):
new_labels = {label: data['color'] for label, data in new_labels.items()}
existing_labels = {label.name: label for label in repo.iter_labels()}
existing_names = set(existing_labels.keys())
desired_names = set(new_labels.keys())
for label in desired_names - existing_names:
dry_echo(dry,
"Creating label '{}' ({})".format(label, new_labels[label]),
fg="green")
if not dry:
repo.create_label(label, new_labels[label])
for label in desired_names & existing_names:
if existing_labels[label].color.lower() != new_labels[label].lower():
dry_echo(dry,
"Updating label '{}' to {}".format(
label, new_labels[label]),
fg="yellow")
if not dry:
existing_labels[label].update(label, new_labels[label])
for label in existing_names - desired_names:
dry_echo(dry, "Deleting label '{}'".format(label), fg="red")
if not dry:
existing_labels[label].delete()
def main(hub, ref, use_tag, override_ref, overrides, interactive, quiet,
reverse, skip_invalid, skip_repos, dry, orgs, branches):
"""Create/remove tags & branches on GitHub repos for Open edX releases."""
repos = openedx_release_repos(hub, orgs, branches)
if not repos:
raise ValueError(
u"No repos marked for openedx-release in their openedx.yaml files!"
)
repos = trim_skipped_repos(repos, skip_repos)
repos = trim_dependent_repos(repos)
repos = override_repo_refs(
repos,
override_ref=override_ref,
overrides=dict(overrides or ()),
)
archived = archived_repos(repos.keys())
if archived:
if dry:
dry_echo(
dry, u"Will need to unarchive these repos: {}".format(
", ".join(repo.full_name for repo in archived)))
else:
ensure_writable(archived)
try:
ret = do_the_work(repos, ref, use_tag, reverse, skip_invalid,
interactive, quiet, dry)
finally:
for repo in archived:
dry_echo(dry, u"Re-archiving {}".format(repo.full_name))
if not dry:
repo.edit(repo.name, archived=True)
return ret
def set_or_delete_labels(dry, repo, new_labels):
desired_colors = {
label: data['color']
for label, data in new_labels.items() if 'color' in data
}
undesired_names = {
label
for label, data in new_labels.items() if data.get('delete', False)
}
existing_labels = {label.name: label for label in repo.iter_labels()}
existing_names = set(existing_labels.keys())
desired_names = set(desired_colors.keys())
for label in desired_names - existing_names:
dry_echo(dry,
"Creating label '{}' ({})".format(label,
desired_colors[label]),
fg="green")
if not dry:
new_label = repo.create_label(label, desired_colors[label])
if new_label is None:
click.secho("Couldn't create label!", fg='red', bold=True)
for label in desired_names & existing_names:
if existing_labels[label].color.lower() != desired_colors[label].lower(
):
dry_echo(dry,
"Updating label '{}' to {}".format(
label, desired_colors[label]),
fg="yellow")
if not dry:
ret = existing_labels[label].update(label,
desired_colors[label])
if not ret:
click.secho("Couldn't update label!", fg='red', bold=True)
for label in undesired_names & existing_names:
dry_echo(dry, "Deleting label '{}'".format(label), fg="red")
if not dry:
ret = existing_labels[label].delete()
if not ret:
click.secho("Couldn't delete label!", fg='red', bold=True)
def set_or_delete_labels(dry, repo, new_labels):
desired_colors = {label: data['color'] for label, data in new_labels.items() if 'color' in data}
undesired_names = {label for label, data in new_labels.items() if data.get('delete', False)}
existing_labels = {label.name: label for label in repo.iter_labels()}
existing_names = set(existing_labels.keys())
desired_names = set(desired_colors.keys())
for label in desired_names - existing_names:
dry_echo(
dry,
"Creating label '{}' ({})".format(label, desired_colors[label]),
fg="green"
)
if not dry:
new_label = repo.create_label(label, desired_colors[label])
if new_label is None:
click.secho("Couldn't create label!", fg='red', bold=True)
for label in desired_names & existing_names:
if existing_labels[label].color.lower() != desired_colors[label].lower():
dry_echo(
dry,
"Updating label '{}' to {}".format(label, desired_colors[label]),
fg="yellow"
)
if not dry:
ret = existing_labels[label].update(label, desired_colors[label])
if not ret:
click.secho("Couldn't update label!", fg='red', bold=True)
for label in undesired_names & existing_names:
dry_echo(
dry,
"Deleting label '{}'".format(label),
fg="red"
)
if not dry:
ret = existing_labels[label].delete()
if not ret:
click.secho("Couldn't delete label!", fg='red', bold=True)
def explode(hub, dry):
"""
Explode the repos.yaml file out into pull requests for all of the
repositories specified in that file.
"""
repo_tools_data = hub.repository('edx', 'repo-tools-data')
repos_yaml = repo_tools_data.file_contents('repos.yaml').decoded
repos = yaml.safe_load(repos_yaml)
for repo, repo_data in repos.items():
user, _, repo_name = repo.partition('/')
if repo_data is None:
repo_data = {}
if 'owner' not in repo_data:
repo_data['owner'] = 'MUST FILL IN OWNER'
if 'area' in repo_data:
repo_data.setdefault('tags', []).append(repo_data['area'])
del repo_data['area']
repo_data.setdefault('oeps', {})
file_contents = yaml.safe_dump(repo_data, indent=4)
file_contents = textwrap.dedent("""
# This file describes this Open edX repo, as described in OEP-2:
# http://open-edx-proposals.readthedocs.io/en/latest/oeps/oep-0002.html#specification
{}
""").format(file_contents).strip() + "\n"
gh_repo = hub.repository(user, repo_name)
if gh_repo.fork:
LOGGER.info("Skipping %s because it is a fork", gh_repo.full_name)
continue
try:
parent_commit = gh_repo.branch(gh_repo.default_branch).commit.sha
except:
LOGGER.warning(
"No commit on default branch %s in repo %s",
gh_repo.default_branch,
gh_repo.full_name
)
continue
if not dry:
if gh_repo.branch(BRANCH_NAME) is None:
gh_repo.create_ref(
'refs/heads/{}'.format(BRANCH_NAME),
parent_commit
)
try:
contents = gh_repo.file_contents(OPEN_EDX_YAML, ref=BRANCH_NAME)
except NotFoundError:
contents = None
if contents is None:
dry_echo(
dry,
"Creating openedx.yaml file on branch {repo}:{branch}".format(
repo=gh_repo.full_name,
branch=BRANCH_NAME,
),
fg='green',
)
click.secho(file_contents, fg='blue')
if not dry:
try:
gh_repo.create_file(
path=OPEN_EDX_YAML,
message='Add an OEP-2 compliant openedx.yaml file',
content=file_contents,
branch=BRANCH_NAME,
)
except TypeError:
# Sadly, TypeError means there was a permissions issue...
LOGGER.exception("Unable to create openedx.yaml")
continue
else:
if contents.decoded != file_contents:
dry_echo(
dry,
"Updated openedx.yaml file on branch {repo}:{branch}".format(
repo=gh_repo.full_name,
branch=BRANCH_NAME,
),
fg='green',
)
click.secho(file_contents, fg='blue')
if not dry:
gh_repo.update_file(
path=OPEN_EDX_YAML,
message='Update the OEP-2 openedx.yaml file',
content=file_contents,
branch=BRANCH_NAME,
sha=contents.sha if contents is not None else None,
)
pr_body = textwrap.dedent("""
This adds an `openedx.yaml` file, as described by OEP-2:
http://open-edx-proposals.readthedocs.io/en/latest/oeps/oep-0002.html
The data in this file was transformed from the contents of
edx/repo-tools-data:repos.yaml
""")
pr_title = 'Add an OEP-2 compliant openedx.yaml file'
existing_pr = [
pr
for pr
in gh_repo.pull_requests(
head='edx:{}'.format(BRANCH_NAME),
state='open'
)
]
if existing_pr:
pull = existing_pr[0]
if pull.title != pr_title or pull.body != pr_body:
dry_echo(
dry,
textwrap.dedent("""\
Updated pull request {repo}#{number}: {title}
URL: {url}
""").format(
url=pull.html_url,
repo=gh_repo.full_name,
number=pull.number,
title=pull.title,
),
fg='green'
)
if not dry:
pull.update(
title=pr_title,
body=pr_body,
)
else:
dry_echo(
dry,
textwrap.dedent("""\
Created pull request {repo}#{number}: {title}
URL: {url}
""").format(
url=pull.html_url if not dry else "N/A",
repo=gh_repo.full_name,
number=pull.number if not dry else "XXXX",
title=pull.title if not dry else pr_title,
),
fg='green'
)
if not dry:
pull = gh_repo.create_pull(
title=pr_title,
base=gh_repo.default_branch,
head=BRANCH_NAME,
body=pr_body
)
def remove_ref_for_repos(repos, ref, use_tag=True, dry=True):
"""
Delete the ref `ref` from each repository in `repos`.
If the ref does not exist on the repo, it is skipped.
This function returns True if any repos had the reference removed,
or False if no repos were modified. If an error occurs while trying
to remove a ref from a repo, the function will continue trying to
remove refs from all the other repos in the iterable -- but after all repos
have been attempted, this function will raise a TagReleaseError with
a list of all the repos that did not have the ref removed.
Trying to remove a ref from a repo that does not have that ref
to begin with is *not* treated as an error.
Arguments:
repos: An iterable of Repository objects.
ref (str): the ref to remove.
use_tag (bool): ref is a tag (True) or a branch (False).
dry (bool): if True, don't do anything, but print what would be done.
Returns:
True if any repos had the ref removed, False if no repos were modified.
"""
if ref.startswith('refs/'):
ref = ref[len('refs/'):]
if not (ref.startswith("heads/") or ref.startswith('tags/')):
ref = "{type}/{ref}".format(
type="tags" if use_tag else "heads",
ref=ref,
)
failures = {}
modified = False
for repo in repos:
try:
try:
ref_obj = repo.ref(ref)
except NotFoundError:
# tag didn't exist to begin with; not an error
continue
dry_echo(dry,
u'Deleting ref {} from repo {}'.format(
ref_obj.ref, repo.full_name),
fg='red')
if not dry:
ref_obj.delete()
modified = True
except GitHubError as err:
# Oops, we got a failure. Record it and move on.
failures[repo.full_name] = err
if failures:
msg = (u"Failed to remove the ref from the following repos: {repos}"
).format(repos=", ".join(failures.keys()))
err = TagReleaseError(msg)
err.failures = failures
raise err
return modified
def create_ref_for_repos(ref_info,
ref,
use_tag=True,
rollback_on_fail=True,
dry=True):
"""
Create refs on the given repos.
If `rollback_on_fail` is True, then on any failure, try to delete the refs
that we're just created, so that we don't fail in a partially-completed
state. (Note that this is *not* a reliable rollback -- other people could
have already fetched the refs from GitHub, or the deletion attempt might
itself fail!)
If this function succeeds, it will return True. If this function fails,
but the world is in a consistent state, this function will return False.
The world is consistent if *no* refs were successfully created on repos in the first
place, or all created refs were were successfully rolled
back (because `rollback_on_fail` is set to True). If this function fails,
and the world is in an inconsistent state, this function will raise a
TagReleaseError. This could happen if some (but not all) of the refs are created,
and either rollback is not attempted (because `rollback_on_fail` is set to
False), or rollback fails.
Arguments:
ref_info (dict mapping Repositories to commit info dicts)
ref (str): the ref to create.
use_tag (bool): make a tag (True) or a branch (False).
rollback_on_fail (bool)
dry (bool): if True, don't do anything, but print what would be done.
Returns
True on success, False otherwise.
"""
if not ref.startswith("refs/"):
ref = u"refs/{type}/{ref}".format(
type="tags" if use_tag else "heads",
ref=ref,
)
succeeded = []
failed_resp = None
failed_repo = None
for repo, commit_info in ref_info.items():
try:
dry_echo(dry,
u'Creating ref {} with sha {} in repo {}'.format(
ref, commit_info['sha'], repo.full_name),
fg='green')
if not dry:
created_ref = repo.create_ref(ref=ref, sha=commit_info['sha'])
if created_ref is None:
failed_resp = FakeResponse(
text="Something went terribly wrong, not sure what")
failed_repo = repo
break
succeeded.append((repo, created_ref))
except GitHubError as exc:
failed_resp = exc.response
failed_repo = repo
# don't try to tag any others, just stop
break
if failed_resp is None:
return True
# if we got to this point, then there was a failure.
try:
original_err_msg = failed_resp.json()["message"]
except Exception:
original_err_msg = failed_resp.text
if not succeeded:
msg = (u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. No refs have been created on any repos."
).format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
)
log.error(msg)
return False
if rollback_on_fail:
rollback_failures = []
for repo, created_ref in succeeded:
try:
dry_echo(dry,
u'Deleting ref {} from repo {}'.format(
created_ref.ref, repo.full_name),
fg='red')
if not dry:
created_ref.delete()
except GitHubError as exc:
rollback_failures.append(repo.full_name)
if rollback_failures:
msg = (u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. "
u"Attempted to roll back, but failed to delete ref on "
u"the following repos: {rollback_failures}").format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
rollback_failures=", ".join(rollback_failures))
err = TagReleaseError(msg)
err.response = failed_resp
err.repos = rollback_failures
raise err
else:
msg = (
u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. However, all refs were successfully "
u"rolled back.").format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
)
log.error(msg)
return False
else:
# don't try to rollback, just raise an error
msg = (u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. No rollback attempted. Refs exist on "
u"the following repos: {tagged_repos}").format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
tagged_repos=", ".join(repo.full_name
for repo, _ in succeeded))
err = TagReleaseError(msg)
err.response = failed_resp
err.repos = succeeded
raise err
def create_ref_for_repos(ref_info, ref, use_tag=True, rollback_on_fail=True, dry=True):
"""
Actually create refs on the given repos.
If `rollback_on_fail` is True, then on any failure, try to delete the refs
that we're just created, so that we don't fail in a partially-completed
state. (Note that this is *not* a reliable rollback -- other people could
have already fetched the refs from GitHub, or the deletion attempt might
itself fail!)
If this function succeeds, it will return True. If this function fails,
but the world is in a consistent state, this function will return False.
The world is consistent if *no* refs were successfully created on repos in the first
place, or all created refs were were successfully rolled
back (because `rollback_on_fail` is set to True). If this function fails,
and the world is in an inconsistent state, this function will raise a
RuntimeError. This could happen if some (but not all) of the refs are created,
and either rollback is not attempted (because `rollback_on_fail` is set to
False), or rollback fails.
"""
if not ref.startswith("refs/"):
ref = u"refs/{type}/{name}".format(
type="tags" if use_tag else "heads",
name=ref,
)
succeeded = []
failed_resp = None
failed_repo = None
for repo, commit_info in ref_info.items():
try:
dry_echo(
dry,
u'Creating ref {} with sha {} in repo {}'.format(
ref, commit_info['sha'], repo.full_name
),
fg='green'
)
if not dry:
created_ref = repo.create_ref(ref=ref, sha=commit_info['sha'])
if created_ref is None:
failed_resp = FakeResponse(text="Something went terribly wrong, not sure what")
failed_repo = repo
break
succeeded.append((repo, created_ref))
except GitHubError as exc:
failed_resp = exc.response
failed_repo = repo
# don't try to tag any others, just stop
break
if failed_resp is None:
return True
# if we got to this point, then there was a failure.
try:
original_err_msg = failed_resp.json()["message"]
except Exception:
original_err_msg = failed_resp.text
if not succeeded:
msg = (
u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. No refs have been created on any repos."
).format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
)
log.error(msg)
return False
if rollback_on_fail:
rollback_failures = []
for repo, created_ref in succeeded:
try:
dry_echo(
dry,
u'Deleting ref {} from repo {}'.format(
created_ref.ref, repo.full_name
),
fg='red'
)
if not dry:
created_ref.delete()
except GitHubError as exc:
rollback_failures.append(repo.full_name)
if rollback_failures:
msg = (
u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. "
u"Attempted to roll back, but failed to delete ref on "
u"the following repos: {rollback_failures}"
).format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
rollback_failures=", ".join(rollback_failures)
)
err = RuntimeError(msg)
err.response = failed_resp
err.repos = rollback_failures
raise err
else:
msg = (
u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. However, all refs were successfully "
u"rolled back."
).format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
)
log.error(msg)
return False
else:
# don't try to rollback, just raise an error
msg = (
u"Failed to create {ref} on {failed_repo}. "
u"Error was {orig_err}. No rollback attempted. Refs exist on "
u"the following repos: {tagged_repos}"
).format(
ref=ref,
failed_repo=failed_repo.full_name,
orig_err=original_err_msg,
tagged_repos=", ".join(repo.full_name for repo, _ in succeeded)
)
err = RuntimeError(msg)
err.response = failed_resp
err.repos = succeeded
raise err