def identify_products_missing_tags(products_to_tag):
problems = []
for name, data in products_to_tag.items():
repo = data['repo']
t_tag = data['target_tag']
yikes = GitTagMissingError(
textwrap.dedent("""\
tag: {gt} missing from repo: {repo} @
sha: {sha}
(eups version: {et})
external repo: {v}
replace existing tag: {update}\
""").format(
repo=repo.full_name,
sha=t_tag.sha,
gt=t_tag.name,
et=data['eups_version'],
v=data['v'],
update=data['update_tag'],
))
problems.append(yikes)
error(yikes)
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return problems
def identify_products_missing_tags(products_to_tag):
problems = []
for name, data in products_to_tag.items():
repo = data['repo']
t_tag = data['target_tag']
yikes = GitTagMissingError(textwrap.dedent("""\
tag: {gt} missing from repo: {repo} @
sha: {sha}
(eups version: {et})
external repo: {v}
replace existing tag: {update}\
""").format(
repo=repo.full_name,
sha=t_tag.sha,
gt=t_tag.name,
et=data['eups_version'],
v=data['v'],
update=data['update_tag'],
))
problems.append(yikes)
error(yikes)
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return problems
def delete_repos(repos, fail_fast=False, dry_run=False, delay=0):
assert isinstance(repos, list), type(repos)
problems = []
for r in repos:
assert isinstance(r, github.Repository.Repository), type(r)
if delay:
sleep(delay)
try:
info("deleting: {r}".format(r=r.full_name))
if dry_run:
info(' (noop)')
continue
r.delete()
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'FAILED - does your token have delete_repo scope?'
yikes = pygithub.CaughtRepositoryError(r, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
return problems
def delete_repos(repos, fail_fast=False, dry_run=False, delay=0):
assert isinstance(repos, list), type(repos)
problems = []
for r in repos:
assert isinstance(r, github.Repository.Repository), type(r)
if delay:
sleep(delay)
try:
info("deleting: {r}".format(r=r.full_name))
if dry_run:
info(' (noop)')
continue
r.delete()
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'FAILED - does your token have delete_repo scope?'
yikes = pygithub.CaughtRepositoryError(r, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
return problems
def create_forks(dst_org, src_repos, fail_fast=False, dry_run=False):
assert isinstance(dst_org, github.Organization.Organization),\
type(dst_org)
assert isinstance(src_repos, list), type(src_repos)
repo_count = len(src_repos)
dst_repos = []
skipped_repos = []
problems = []
with pbar.eta_bar(msg='forking', max_value=repo_count) as progress:
repo_idx = 0
for r in src_repos:
progress.update(repo_idx)
repo_idx += 1
# XXX per
# https://developer.github.com/v3/repos/forks/#create-a-fork
# fork creation is async and pygithub doesn't appear to wait.
# https://github.com/PyGithub/PyGithub/blob/c44469965e4ea368b78c4055a8afcfcf08314585/github/Organization.py#L321-L336
# so its possible that this may fail in some strange way such as
# not returning all repo data, but it hasn't yet been observed.
# get current time before API call in case fork creation is slow.
now = datetime.datetime.now()
debug("forking {r}".format(r=r.full_name))
if dry_run:
debug(' (noop)')
continue
try:
fork = dst_org.create_fork(r)
dst_repos.append(fork)
debug(" -> {r}".format(r=fork.full_name))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
if 'Empty repositories cannot be forked.' in e.data['message']:
warn("{r} is empty and can not be forked".format(
r=r.full_name))
skipped_repos.append(r)
continue
msg = "error forking repo {r}".format(r=r.full_name)
yikes = pygithub.CaughtOrganizationError(dst_org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if fork.created_at < now:
warn("fork of {r} already exists\n created_at {ctime}".format(
r=fork.full_name, ctime=fork.created_at))
return dst_repos, skipped_repos, problems
def check_tags(repos, tags, ignore_existing=False, fail_fast=False):
""" check if tags already exist in repos"""
debug("looking for {n} tag(s):".format(n=len(tags)))
[debug(" {t}".format(t=t)) for t in tags]
debug("in {n} repo(s):".format(n=len(repos)))
[debug(" {r}".format(r=r.full_name)) for r in repos]
# present/missing tags by repo name
present_tags = {}
absent_tags = {}
problems = []
for r in repos:
has_tags = find_tags_in_repo(r, tags)
if has_tags:
if not ignore_existing:
yikes = GitTagExistsError(
"tag(s) {tag} already exists in repos {r}".format(
tag=list(has_tags.keys()),
r=r.full_name
))
if fail_fast:
raise yikes
problems.append(yikes)
error(yikes)
present_tags[r.full_name] = {
'repo': r,
'tags': list(has_tags.values()),
}
missing_tags = [x for x in tags if x not in has_tags]
if missing_tags:
absent_tags[r.full_name] = {
'repo': r,
'need_tags': missing_tags,
}
debug(textwrap.dedent("""\
found:
{n_with:>4} repos with tag(s)
{n_none:>4} repos with no tag(s)
{errors:>4} repos with error(s)\
""").format(
n_with=len(present_tags),
n_none=len(absent_tags),
errors=len(problems),
))
return present_tags, absent_tags, problems
def main():
try:
try:
run()
except codetools.DogpileError as e:
error(e)
n = len(e.errors)
sys.exit(n if n < 256 else 255)
else:
sys.exit(0)
finally:
if 'g' in globals():
pygithub.debug_ratelimit(g)
except SystemExit as e:
debug("exit {status}".format(status=str(e)))
raise e
def main():
try:
try:
run()
except codetools.DogpileError as e:
error(e)
n = len(e.errors)
sys.exit(n if n < 256 else 255)
else:
sys.exit(0)
finally:
if 'g' in globals():
pygithub.debug_ratelimit(g)
except SystemExit as e:
debug("exit {status}".format(status=str(e)))
raise e
def check_repos(repos, allow_teams, deny_teams, fail_fast=False):
problems = []
for r in repos:
try:
pygithub.check_repo_teams(
r,
allow_teams=allow_teams,
deny_teams=deny_teams,
team_names=[t.name for t in find_repo_teams(r)],
)
except pygithub.RepositoryTeamMembershipError as e:
if fail_fast:
raise
problems.append(e)
error(e)
continue
return problems
def delete_teams(teams, fail_fast=False, dry_run=False, delay=0):
assert isinstance(teams, list), type(teams)
problems = []
for t in teams:
if delay:
sleep(delay)
try:
info("deleting team: '{t}'".format(t=t.name))
if dry_run:
info(' (noop)')
continue
t.delete()
except github.RateLimitExceededException:
raise
except github.GithubException as e:
yikes = pygithub.CaughtTeamError(t, e)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
return problems
def delete_teams(teams, fail_fast=False, dry_run=False, delay=0):
assert isinstance(teams, list), type(teams)
problems = []
for t in teams:
if delay:
sleep(delay)
try:
info("deleting team: '{t}'".format(t=t.name))
if dry_run:
info(' (noop)')
continue
t.delete()
except github.RateLimitExceededException:
raise
except github.GithubException as e:
yikes = pygithub.CaughtTeamError(t, e)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
return problems
def cross_reference_products(
eups_products,
manifest_products,
ignore_manifest_versions=False,
fail_fast=False,
):
"""
Cross reference EupsTag and Manifest data and return a merged result
Parameters
----------
eups_products:
manifest:
fail_fast: bool
ignore_manifest_versions: bool
Returns
-------
products: dict
Raises
------
RuntimeError
Upon error if `fail_fast` is `True`.
"""
products = {}
problems = []
for name, eups_data in eups_products.items():
try:
manifest_data = manifest_products[name]
except KeyError:
yikes = RuntimeError(
textwrap.dedent("""\
failed to find record in manifest for:
{product} {eups_version}\
""").format(
product=name,
eups_version=eups_data['eups_version'],
))
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if ignore_manifest_versions:
# ignore the manifest eups_version string by simply setting it to
# the eups tag value. This ensures that the eups tag value will be
# passed though.
manifest_data = manifest_data.copy()
manifest_data['eups_version'] = eups_data['eups_version']
if eups_data['eups_version'] != manifest_data['eups_version']:
yikes = RuntimeError(
textwrap.dedent("""\
eups version string mismatch:
eups tag: {product} {eups_eups_version}
manifest: {product} {manifest_eups_version}\
""").format(
product=name,
eups_eups_version=eups_data['eups_version'],
manifest_eups_version=manifest_data['eups_version'],
))
if fail_fast:
raise yikes
problems.append(yikes)
error(yikes)
products[name] = eups_data.copy()
products[name].update(manifest_data)
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return products, problems
def tag_products(
products,
fail_fast=False,
dry_run=False,
):
problems = []
for name, data in products.items():
repo = data['repo']
t_tag = data['target_tag']
info(textwrap.dedent("""\
tagging repo: {repo} @
sha: {sha} as {gt}
(eups version: {et})
external repo: {v}
replace existing tag: {update}\
""").format(
repo=repo.full_name,
sha=t_tag.sha,
gt=t_tag.name,
et=data['eups_version'],
v=data['v'],
update=data['update_tag'],
))
if dry_run:
info(' (noop)')
continue
try:
tag_obj = repo.create_git_tag(
t_tag.name,
t_tag.message,
t_tag.sha,
'commit',
tagger=t_tag.tagger,
)
debug(" created tag object {tag_obj}".format(tag_obj=tag_obj))
if data['update_tag']:
ref = pygithub.find_tag_by_name(
repo,
t_tag.name,
safe=False,
)
ref.edit(tag_obj.sha, force=True)
debug(" updated existing ref: {ref}".format(ref=ref))
else:
ref = repo.create_git_ref(
"refs/tags/{t}".format(t=t_tag.name),
tag_obj.sha
)
debug(" created ref: {ref}".format(ref=ref))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = "error creating tag: {t}".format(t=t_tag.name)
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if problems:
msg = "{n} tag failures".format(n=len(problems))
raise codetools.DogpileError(problems, msg)
def get_repo_for_products(org,
products,
allow_teams,
ext_teams,
deny_teams,
fail_fast=False):
debug("allowed teams: {allow}".format(allow=allow_teams))
debug("external teams: {ext}".format(ext=ext_teams))
debug("denied teams: {deny}".format(deny=deny_teams))
resolved_products = {}
problems = []
for name, data in products.items():
debug("looking for git repo for: {name} [{ver}]".format(
name=name, ver=data['eups_version']))
try:
repo = org.get_repo(name)
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = "error getting repo by name: {r}".format(r=name)
yikes = pygithub.CaughtOrganizationError(org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
continue
debug(" found: {slug}".format(slug=repo.full_name))
try:
repo_team_names = [t.name for t in repo.get_teams()]
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
continue
debug(" teams: {teams}".format(teams=repo_team_names))
try:
pygithub.check_repo_teams(repo,
allow_teams=allow_teams,
deny_teams=deny_teams,
team_names=repo_team_names)
except pygithub.RepositoryTeamMembershipError as e:
if fail_fast:
raise
problems.append(e)
error(e)
continue
has_ext_team = any(x in repo_team_names for x in ext_teams)
debug(" external repo: {v}".format(v=has_ext_team))
resolved_products[name] = data.copy()
resolved_products[name]['repo'] = repo
resolved_products[name]['v'] = has_ext_team
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return resolved_products, problems
def check_product_tags(
products,
git_tag,
tag_message_template,
tagger,
force_tag=False,
fail_fast=False,
ignore_git_message=False,
ignore_git_tagger=False,
):
assert isinstance(tagger, github.InputGitAuthor), type(tagger)
checked_products = {}
problems = []
for name, data in products.items():
repo = data['repo']
tag_name = git_tag
# prefix tag name with `v`?
if data['v'] and re.match('\d', tag_name):
tag_name = "v{git_tag}".format(git_tag=tag_name)
# message can not be formatted until we've determined if the tag must
# be prefixed. The 'v' prefix appearing in the tag message is required
# to match historical behavior and allow verification of past releases.
message = tag_message_template.format(git_tag=tag_name)
# "target tag"
t_tag = codekit.pygithub.TargetTag(
name=tag_name,
sha=data['sha'],
message=message,
tagger=tagger,
)
# control whether to create a new tag or update an existing one
update_tag = False
try:
# if the existing tag is in sync, do nothing
if check_existing_git_tag(
repo,
t_tag,
ignore_git_message=ignore_git_message,
ignore_git_tagger=ignore_git_tagger,
):
warn(textwrap.dedent("""\
No action for {repo}
existing tag: {tag} is already in sync\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
continue
except github.RateLimitExceededException:
raise
except GitTagExistsError as e:
# if force_tag is set, and the tag already exists, set
# update_tag and fall through. Otherwise, treat it as any other
# exception.
if force_tag:
update_tag = True
warn(textwrap.dedent("""\
existing tag: {tag} WILL BE MOVED\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
elif fail_fast:
raise
else:
problems.append(e)
error(e)
continue
except github.GithubException as e:
msg = "error checking for existance of tag: {t}".format(
t=t_tag.name,
)
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
else:
problems.append(yikes)
error(yikes)
continue
checked_products[name] = data.copy()
checked_products[name]['target_tag'] = t_tag
checked_products[name]['update_tag'] = update_tag
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return checked_products, problems
def get_repo_for_products(
org,
products,
allow_teams,
ext_teams,
deny_teams,
fail_fast=False
):
debug("allowed teams: {allow}".format(allow=allow_teams))
debug("external teams: {ext}".format(ext=ext_teams))
debug("denied teams: {deny}".format(deny=deny_teams))
resolved_products = {}
problems = []
for name, data in products.items():
debug("looking for git repo for: {name} [{ver}]".format(
name=name,
ver=data['eups_version']
))
try:
repo = org.get_repo(name)
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = "error getting repo by name: {r}".format(r=name)
yikes = pygithub.CaughtOrganizationError(org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
continue
debug(" found: {slug}".format(slug=repo.full_name))
try:
repo_team_names = [t.name for t in repo.get_teams()]
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
continue
debug(" teams: {teams}".format(teams=repo_team_names))
try:
pygithub.check_repo_teams(
repo,
allow_teams=allow_teams,
deny_teams=deny_teams,
team_names=repo_team_names
)
except pygithub.RepositoryTeamMembershipError as e:
if fail_fast:
raise
problems.append(e)
error(e)
continue
has_ext_team = any(x in repo_team_names for x in ext_teams)
debug(" external repo: {v}".format(v=has_ext_team))
resolved_products[name] = data.copy()
resolved_products[name]['repo'] = repo
resolved_products[name]['v'] = has_ext_team
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return resolved_products, problems
def run():
args = parse_args()
codetools.setup_logging(args.debug)
global g
g = pygithub.login_github(token_path=args.token_path, token=args.token)
# protect destination org
codetools.validate_org(args.dst_org)
src_org = g.get_organization(args.src_org)
dst_org = g.get_organization(args.dst_org)
info("forking repos from: {org}".format(org=src_org.login))
info(" to: {org}".format(org=dst_org.login))
debug('looking for repos -- this can take a while for large orgs...')
if args.team:
debug('checking that selection team(s) exist')
try:
org_teams = list(src_org.get_teams())
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
raise pygithub.CaughtOrganizationError(src_org, e, msg) from None
missing_teams = [n for n in args.team if n not in
[t.name for t in org_teams]]
if missing_teams:
error("{n} team(s) do not exist:".format(n=len(missing_teams)))
[error(" '{t}'".format(t=n)) for n in missing_teams]
return
fork_teams = [t for t in org_teams if t.name in args.team]
repos = pygithub.get_repos_by_team(fork_teams)
debug('selecting repos by membership in team(s):')
[debug(" '{t}'".format(t=t.name)) for t in fork_teams]
else:
repos = pygithub.get_repos_by_team(fork_teams)
src_repos = list(itertools.islice(repos, args.limit))
repo_count = len(src_repos)
if not repo_count:
debug('nothing to do -- exiting')
return
debug("found {n} repos to be forked from org {src_org}:".format(
n=repo_count,
src_org=src_org.login
))
[debug(" {r}".format(r=r.full_name)) for r in src_repos]
if args.copy_teams:
debug('checking source repo team membership...')
# dict of repo and team objects, keyed by repo name
src_rt = find_teams_by_repo(src_repos)
# extract a non-duplicated list of team names from all repos being
# forked as a dict, keyed by team name
src_teams = find_used_teams(src_rt)
debug('found {n} teams in use within org {o}:'.format(
n=len(src_teams),
o=src_org.login
))
[debug(" '{t}'".format(t=t)) for t in src_teams.keys()]
# check for conflicting teams in dst org before attempting to create
# any forks so its possible to bail out before any resources have been
# created.
debug('checking teams in destination org')
conflicting_teams = pygithub.get_teams_by_name(
dst_org,
list(src_teams.keys())
)
if conflicting_teams:
raise TeamError(
"found {n} conflicting teams in {o}: {teams}".format(
n=len(conflicting_teams),
o=dst_org.login,
teams=[t.name for t in conflicting_teams]
))
debug('there is no spoon...')
problems = []
pygithub.debug_ratelimit(g)
dst_repos, skipped_repos, err = create_forks(
dst_org,
src_repos,
fail_fast=args.fail_fast,
dry_run=args.dry_run
)
if err:
problems += err
if args.copy_teams:
# filter out repos which were skipped
# dict of str(fork_repo.name): fork_repo
dst_forks = dict((r.name, r) for r in dst_repos)
bad_repos = dict((r.name, r) for r in skipped_repos)
# dict of str(team.name): [repos] to be created
dst_teams = {}
for name, repos in src_teams.items():
dst_teams[name] = [dst_forks[r.name] for r in repos
if r.name not in bad_repos]
_, err = create_teams(
dst_org,
dst_teams,
with_repos=True,
fail_fast=args.fail_fast,
dry_run=args.dry_run
)
if err:
problems += err
if problems:
msg = "{n} errors forking repo(s)/teams(s)".format(
n=len(problems))
raise codetools.DogpileError(problems, msg)
def create_forks(
dst_org,
src_repos,
fail_fast=False,
dry_run=False
):
assert isinstance(dst_org, github.Organization.Organization),\
type(dst_org)
assert isinstance(src_repos, list), type(src_repos)
repo_count = len(src_repos)
dst_repos = []
skipped_repos = []
problems = []
with pbar.eta_bar(msg='forking', max_value=repo_count) as progress:
repo_idx = 0
for r in src_repos:
progress.update(repo_idx)
repo_idx += 1
# XXX per
# https://developer.github.com/v3/repos/forks/#create-a-fork
# fork creation is async and pygithub doesn't appear to wait.
# https://github.com/PyGithub/PyGithub/blob/c44469965e4ea368b78c4055a8afcfcf08314585/github/Organization.py#L321-L336
# so its possible that this may fail in some strange way such as
# not returning all repo data, but it hasn't yet been observed.
# get current time before API call in case fork creation is slow.
now = datetime.datetime.now()
debug("forking {r}".format(r=r.full_name))
if dry_run:
debug(' (noop)')
continue
try:
fork = dst_org.create_fork(r)
dst_repos.append(fork)
debug(" -> {r}".format(r=fork.full_name))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
if 'Empty repositories cannot be forked.' in e.data['message']:
warn("{r} is empty and can not be forked".format(
r=r.full_name
))
skipped_repos.append(r)
continue
msg = "error forking repo {r}".format(r=r.full_name)
yikes = pygithub.CaughtOrganizationError(dst_org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if fork.created_at < now:
warn("fork of {r} already exists\n created_at {ctime}".format(
r=fork.full_name,
ctime=fork.created_at
))
return dst_repos, skipped_repos, problems
def run():
args = parse_args()
codetools.setup_logging(args.debug)
global g
g = pygithub.login_github(token_path=args.token_path, token=args.token)
# protect destination org
codetools.validate_org(args.dst_org)
src_org = g.get_organization(args.src_org)
dst_org = g.get_organization(args.dst_org)
info("forking repos from: {org}".format(org=src_org.login))
info(" to: {org}".format(org=dst_org.login))
debug('looking for repos -- this can take a while for large orgs...')
if args.team:
debug('checking that selection team(s) exist')
try:
org_teams = list(src_org.get_teams())
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
raise pygithub.CaughtOrganizationError(src_org, e, msg) from None
missing_teams = [
n for n in args.team if n not in [t.name for t in org_teams]
]
if missing_teams:
error("{n} team(s) do not exist:".format(n=len(missing_teams)))
[error(" '{t}'".format(t=n)) for n in missing_teams]
return
fork_teams = [t for t in org_teams if t.name in args.team]
repos = pygithub.get_repos_by_team(fork_teams)
debug('selecting repos by membership in team(s):')
[debug(" '{t}'".format(t=t.name)) for t in fork_teams]
else:
repos = pygithub.get_repos_by_team(fork_teams)
src_repos = list(itertools.islice(repos, args.limit))
repo_count = len(src_repos)
if not repo_count:
debug('nothing to do -- exiting')
return
debug("found {n} repos to be forked from org {src_org}:".format(
n=repo_count, src_org=src_org.login))
[debug(" {r}".format(r=r.full_name)) for r in src_repos]
if args.copy_teams:
debug('checking source repo team membership...')
# dict of repo and team objects, keyed by repo name
src_rt = find_teams_by_repo(src_repos)
# extract a non-duplicated list of team names from all repos being
# forked as a dict, keyed by team name
src_teams = find_used_teams(src_rt)
debug('found {n} teams in use within org {o}:'.format(n=len(src_teams),
o=src_org.login))
[debug(" '{t}'".format(t=t)) for t in src_teams.keys()]
# check for conflicting teams in dst org before attempting to create
# any forks so its possible to bail out before any resources have been
# created.
debug('checking teams in destination org')
conflicting_teams = pygithub.get_teams_by_name(dst_org,
list(src_teams.keys()))
if conflicting_teams:
raise TeamError(
"found {n} conflicting teams in {o}: {teams}".format(
n=len(conflicting_teams),
o=dst_org.login,
teams=[t.name for t in conflicting_teams]))
debug('there is no spoon...')
problems = []
pygithub.debug_ratelimit(g)
dst_repos, skipped_repos, err = create_forks(dst_org,
src_repos,
fail_fast=args.fail_fast,
dry_run=args.dry_run)
if err:
problems += err
if args.copy_teams:
# filter out repos which were skipped
# dict of str(fork_repo.name): fork_repo
dst_forks = dict((r.name, r) for r in dst_repos)
bad_repos = dict((r.name, r) for r in skipped_repos)
# dict of str(team.name): [repos] to be created
dst_teams = {}
for name, repos in src_teams.items():
dst_teams[name] = [
dst_forks[r.name] for r in repos if r.name not in bad_repos
]
_, err = create_teams(dst_org,
dst_teams,
with_repos=True,
fail_fast=args.fail_fast,
dry_run=args.dry_run)
if err:
problems += err
if problems:
msg = "{n} errors forking repo(s)/teams(s)".format(n=len(problems))
raise codetools.DogpileError(problems, msg)
def check_product_tags(
products,
git_tag,
tag_message_template,
tagger,
force_tag=False,
fail_fast=False,
ignore_git_message=False,
ignore_git_tagger=False,
):
assert isinstance(tagger, github.InputGitAuthor), type(tagger)
checked_products = {}
problems = []
for name, data in products.items():
repo = data['repo']
tag_name = git_tag
# prefix tag name with `v`?
if data['v'] and re.match('\d', tag_name):
tag_name = "v{git_tag}".format(git_tag=tag_name)
# message can not be formatted until we've determined if the tag must
# be prefixed. The 'v' prefix appearing in the tag message is required
# to match historical behavior and allow verification of past releases.
message = tag_message_template.format(git_tag=tag_name)
# "target tag"
t_tag = codekit.pygithub.TargetTag(
name=tag_name,
sha=data['sha'],
message=message,
tagger=tagger,
)
# control whether to create a new tag or update an existing one
update_tag = False
try:
# if the existing tag is in sync, do nothing
if check_existing_git_tag(
repo,
t_tag,
ignore_git_message=ignore_git_message,
ignore_git_tagger=ignore_git_tagger,
):
warn(
textwrap.dedent("""\
No action for {repo}
existing tag: {tag} is already in sync\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
continue
except github.RateLimitExceededException:
raise
except GitTagExistsError as e:
# if force_tag is set, and the tag already exists, set
# update_tag and fall through. Otherwise, treat it as any other
# exception.
if force_tag:
update_tag = True
warn(
textwrap.dedent("""\
existing tag: {tag} WILL BE MOVED\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
elif fail_fast:
raise
else:
problems.append(e)
error(e)
continue
except github.GithubException as e:
msg = "error checking for existance of tag: {t}".format(
t=t_tag.name, )
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
else:
problems.append(yikes)
error(yikes)
continue
checked_products[name] = data.copy()
checked_products[name]['target_tag'] = t_tag
checked_products[name]['update_tag'] = update_tag
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return checked_products, problems
def create_teams(org,
teams,
with_repos=False,
ignore_existing=False,
fail_fast=False,
dry_run=False):
assert isinstance(org, github.Organization.Organization), type(org)
assert isinstance(teams, dict), type(teams)
# it takes fewer api calls to create team(s) with an explicit list of
# members after all repos have been forked but this blows up if the team
# already exists.
debug("creating teams in {org}".format(org=org.login))
# dict of dst org teams keyed by name (str) with team object as value
dst_teams = {}
problems = []
batch_repos = 50
for name, repos in teams.items():
pygithub.debug_ratelimit(g)
debug("creating team {o}/'{t}'".format(o=org.login, t=name))
if dry_run:
debug(' (noop)')
continue
dst_t = None
try:
if with_repos:
debug(" with {n} member repos:".format(n=len(repos)))
[debug(" {r}".format(r=r.full_name)) for r in repos]
leftover_repos = repos[batch_repos:]
if leftover_repos:
debug(" creating team with first {b} of {n} repos".format(
b=batch_repos, n=len(repos)))
dst_t = org.create_team(name, repo_names=repos[:batch_repos])
if leftover_repos:
# add any repos over the batch limit individually to team
for r in leftover_repos:
debug(" adding repo {r}".format(r=r.full_name))
dst_t.add_to_repos(r)
else:
dst_t = org.create_team(name)
except github.RateLimitExceededException:
raise
except github.GithubException as e:
# if the error is for any cause other than the team already
# existing, puke.
team_exists = False
if ignore_existing and 'errors' in e.data:
for oops in e.data['errors']:
msg = oops['message']
if 'Name has already been taken' in msg:
# find existing team
dst_t = pygithub.get_teams_by_name(org, name)[0]
team_exists = True
if not (ignore_existing and team_exists):
msg = "error creating team: {t}".format(t=name)
yikes = pygithub.CaughtOrganizationError(org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
break
else:
dst_teams[dst_t.name] = dst_t
return dst_teams, problems
def cross_reference_products(
eups_products,
manifest_products,
ignore_manifest_versions=False,
fail_fast=False,
):
"""
Cross reference EupsTag and Manifest data and return a merged result
Parameters
----------
eups_products:
manifest:
fail_fast: bool
ignore_manifest_versions: bool
Returns
-------
products: dict
Raises
------
RuntimeError
Upon error if `fail_fast` is `True`.
"""
products = {}
problems = []
for name, eups_data in eups_products.items():
try:
manifest_data = manifest_products[name]
except KeyError:
yikes = RuntimeError(textwrap.dedent("""\
failed to find record in manifest for:
{product} {eups_version}\
""").format(
product=name,
eups_version=eups_data['eups_version'],
))
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if ignore_manifest_versions:
# ignore the manifest eups_version string by simply setting it to
# the eups tag value. This ensures that the eups tag value will be
# passed though.
manifest_data = manifest_data.copy()
manifest_data['eups_version'] = eups_data['eups_version']
if eups_data['eups_version'] != manifest_data['eups_version']:
yikes = RuntimeError(textwrap.dedent("""\
eups version string mismatch:
eups tag: {product} {eups_eups_version}
manifest: {product} {manifest_eups_version}\
""").format(
product=name,
eups_eups_version=eups_data['eups_version'],
manifest_eups_version=manifest_data['eups_version'],
))
if fail_fast:
raise yikes
problems.append(yikes)
error(yikes)
products[name] = eups_data.copy()
products[name].update(manifest_data)
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return products, problems
def create_teams(
org,
teams,
with_repos=False,
ignore_existing=False,
fail_fast=False,
dry_run=False
):
assert isinstance(org, github.Organization.Organization), type(org)
assert isinstance(teams, dict), type(teams)
# it takes fewer api calls to create team(s) with an explicit list of
# members after all repos have been forked but this blows up if the team
# already exists.
debug("creating teams in {org}".format(org=org.login))
# dict of dst org teams keyed by name (str) with team object as value
dst_teams = {}
problems = []
batch_repos = 50
for name, repos in teams.items():
pygithub.debug_ratelimit(g)
debug("creating team {o}/'{t}'".format(
o=org.login,
t=name
))
if dry_run:
debug(' (noop)')
continue
dst_t = None
try:
if with_repos:
debug(" with {n} member repos:".format(n=len(repos)))
[debug(" {r}".format(r=r.full_name)) for r in repos]
leftover_repos = repos[batch_repos:]
if leftover_repos:
debug(" creating team with first {b} of {n} repos"
.format(
b=batch_repos,
n=len(repos)
))
dst_t = org.create_team(name, repo_names=repos[:batch_repos])
if leftover_repos:
# add any repos over the batch limit individually to team
for r in leftover_repos:
debug(" adding repo {r}".format(r=r.full_name))
dst_t.add_to_repos(r)
else:
dst_t = org.create_team(name)
except github.RateLimitExceededException:
raise
except github.GithubException as e:
# if the error is for any cause other than the team already
# existing, puke.
team_exists = False
if ignore_existing and 'errors' in e.data:
for oops in e.data['errors']:
msg = oops['message']
if 'Name has already been taken' in msg:
# find existing team
dst_t = pygithub.get_teams_by_name(org, name)[0]
team_exists = True
if not (ignore_existing and team_exists):
msg = "error creating team: {t}".format(t=name)
yikes = pygithub.CaughtOrganizationError(org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
break
else:
dst_teams[dst_t.name] = dst_t
return dst_teams, problems
def tag_products(
products,
fail_fast=False,
dry_run=False,
):
problems = []
for name, data in products.items():
repo = data['repo']
t_tag = data['target_tag']
info(
textwrap.dedent("""\
tagging repo: {repo} @
sha: {sha} as {gt}
(eups version: {et})
external repo: {v}
replace existing tag: {update}\
""").format(
repo=repo.full_name,
sha=t_tag.sha,
gt=t_tag.name,
et=data['eups_version'],
v=data['v'],
update=data['update_tag'],
))
if dry_run:
info(' (noop)')
continue
try:
tag_obj = repo.create_git_tag(
t_tag.name,
t_tag.message,
t_tag.sha,
'commit',
tagger=t_tag.tagger,
)
debug(" created tag object {tag_obj}".format(tag_obj=tag_obj))
if data['update_tag']:
ref = pygithub.find_tag_by_name(
repo,
t_tag.name,
safe=False,
)
ref.edit(tag_obj.sha, force=True)
debug(" updated existing ref: {ref}".format(ref=ref))
else:
ref = repo.create_git_ref("refs/tags/{t}".format(t=t_tag.name),
tag_obj.sha)
debug(" created ref: {ref}".format(ref=ref))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = "error creating tag: {t}".format(t=t_tag.name)
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if problems:
msg = "{n} tag failures".format(n=len(problems))
raise codetools.DogpileError(problems, msg)