def create_forks(dst_org, src_repos, fail_fast=False, dry_run=False):
assert isinstance(dst_org, github.Organization.Organization),\
type(dst_org)
assert isinstance(src_repos, list), type(src_repos)
repo_count = len(src_repos)
dst_repos = []
skipped_repos = []
problems = []
with pbar.eta_bar(msg='forking', max_value=repo_count) as progress:
repo_idx = 0
for r in src_repos:
progress.update(repo_idx)
repo_idx += 1
# XXX per
# https://developer.github.com/v3/repos/forks/#create-a-fork
# fork creation is async and pygithub doesn't appear to wait.
# https://github.com/PyGithub/PyGithub/blob/c44469965e4ea368b78c4055a8afcfcf08314585/github/Organization.py#L321-L336
# so its possible that this may fail in some strange way such as
# not returning all repo data, but it hasn't yet been observed.
# get current time before API call in case fork creation is slow.
now = datetime.datetime.now()
debug("forking {r}".format(r=r.full_name))
if dry_run:
debug(' (noop)')
continue
try:
fork = dst_org.create_fork(r)
dst_repos.append(fork)
debug(" -> {r}".format(r=fork.full_name))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
if 'Empty repositories cannot be forked.' in e.data['message']:
warn("{r} is empty and can not be forked".format(
r=r.full_name))
skipped_repos.append(r)
continue
msg = "error forking repo {r}".format(r=r.full_name)
yikes = pygithub.CaughtOrganizationError(dst_org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if fork.created_at < now:
warn("fork of {r} already exists\n created_at {ctime}".format(
r=fork.full_name, ctime=fork.created_at))
return dst_repos, skipped_repos, problems
def wait_for_user_panic(**kwargs):
"""Display a scary message and count down progresss bar so an interative
user a chance to panic and kill the program.
Parameters
----------
kwargs
Passed verbatim to countdown_timer()
"""
warn('Now is the time to panic and Ctrl-C')
countdown_timer(**kwargs)
def wait_for_user_panic(**kwargs):
"""Display a scary message and count down progresss bar so an interative
user a chance to panic and kill the program.
Parameters
----------
kwargs
Passed verbatim to countdown_timer()
"""
warn('Now is the time to panic and Ctrl-C')
countdown_timer(**kwargs)
def delete_all_repos(org, **kwargs):
assert isinstance(org, github.Organization.Organization), type(org)
limit = kwargs.pop('limit', None)
try:
repos = list(itertools.islice(org.get_repos(), limit))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting repos'
raise pygithub.CaughtOrganizationError(org, e, msg) from None
info("found {n} repos in {org}".format(n=len(repos), org=org.login))
[debug(" {r}".format(r=r.full_name)) for r in repos]
if repos:
warn("Deleting all repos in {org}".format(org=org.login))
pbar.wait_for_user_panic_once()
return delete_repos(repos, **kwargs)
def delete_all_teams(org, **kwargs):
assert isinstance(org, github.Organization.Organization), type(org)
limit = kwargs.pop('limit', None)
try:
teams = list(itertools.islice(org.get_teams(), limit))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
raise pygithub.CaughtOrganizationError(org, e, msg) from None
info("found {n} teams in {org}".format(n=len(teams), org=org.login))
[debug(" '{t}'".format(t=t.name)) for t in teams]
if teams:
warn("Deleting all teams in {org}".format(org=org.login))
pbar.wait_for_user_panic_once()
return delete_teams(teams, **kwargs)
def delete_all_repos(org, **kwargs):
assert isinstance(org, github.Organization.Organization), type(org)
limit = kwargs.pop('limit', None)
try:
repos = list(itertools.islice(org.get_repos(), limit))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting repos'
raise pygithub.CaughtOrganizationError(org, e, msg) from None
info("found {n} repos in {org}".format(n=len(repos), org=org.login))
[debug(" {r}".format(r=r.full_name)) for r in repos]
if repos:
warn("Deleting all repos in {org}".format(org=org.login))
pbar.wait_for_user_panic_once()
return delete_repos(repos, **kwargs)
def delete_all_teams(org, **kwargs):
assert isinstance(org, github.Organization.Organization), type(org)
limit = kwargs.pop('limit', None)
try:
teams = list(itertools.islice(org.get_teams(), limit))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
raise pygithub.CaughtOrganizationError(org, e, msg) from None
info("found {n} teams in {org}".format(n=len(teams), org=org.login))
[debug(" '{t}'".format(t=t.name)) for t in teams]
if teams:
warn("Deleting all teams in {org}".format(org=org.login))
pbar.wait_for_user_panic_once()
return delete_teams(teams, **kwargs)
def untag_repos(present_tags, **kwargs):
if not present_tags:
info('nothing to do')
return
warn('Deleting tag(s)')
pbar.wait_for_user_panic_once()
info("untagging {n} repo(s) [tags]:".format(n=len(present_tags)))
max_name_len = len(max(present_tags, key=len))
for k in present_tags:
info(" {repo: >{w}} {tags}".format(
w=max_name_len,
repo=k,
tags=[tag_name_from_ref(ref) for ref in present_tags[k]['tags']]
))
for k in present_tags:
r = present_tags[k]['repo']
tags = present_tags[k]['tags']
delete_refs(r, tags, **kwargs)
def run():
"""Move the repos"""
args = parse_args()
codetools.setup_logging(args.debug)
global g
g = pygithub.login_github(token_path=args.token_path, token=args.token)
org = g.get_organization(args.org)
# only iterate over all teams once
try:
teams = list(org.get_teams())
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
raise pygithub.CaughtOrganizationError(org, e, msg) from None
old_team = find_team(teams, args.oldteam)
new_team = find_team(teams, args.newteam)
move_me = args.repos
debug(len(move_me), 'repos to be moved')
added = []
removed = []
for name in move_me:
try:
r = org.get_repo(name)
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = "error getting repo by name: {r}".format(r=name)
raise pygithub.CaughtOrganizationError(org, e, msg) from None
# Add team to the repo
debug("Adding {repo} to '{team}' ...".format(
repo=r.full_name,
team=args.newteam
))
if not args.dry_run:
try:
new_team.add_to_repos(r)
added += r.full_name
debug(' ok')
except github.RateLimitExceededException:
raise
except github.GithubException as e:
debug(' FAILED')
if old_team.name in 'Owners':
warn("Removing repo {repo} from team 'Owners' is not allowed"
.format(repo=r.full_name))
debug("Removing {repo} from '{team}' ...".format(
repo=r.full_name,
team=args.oldteam
))
if not args.dry_run:
try:
old_team.remove_from_repos(r)
removed += r.full_name
debug(' ok')
except github.RateLimitExceededException:
raise
except github.GithubException as e:
debug(' FAILED')
info('Added:', added)
info('Removed:', removed)
def check_product_tags(
products,
git_tag,
tag_message_template,
tagger,
force_tag=False,
fail_fast=False,
ignore_git_message=False,
ignore_git_tagger=False,
):
assert isinstance(tagger, github.InputGitAuthor), type(tagger)
checked_products = {}
problems = []
for name, data in products.items():
repo = data['repo']
tag_name = git_tag
# prefix tag name with `v`?
if data['v'] and re.match('\d', tag_name):
tag_name = "v{git_tag}".format(git_tag=tag_name)
# message can not be formatted until we've determined if the tag must
# be prefixed. The 'v' prefix appearing in the tag message is required
# to match historical behavior and allow verification of past releases.
message = tag_message_template.format(git_tag=tag_name)
# "target tag"
t_tag = codekit.pygithub.TargetTag(
name=tag_name,
sha=data['sha'],
message=message,
tagger=tagger,
)
# control whether to create a new tag or update an existing one
update_tag = False
try:
# if the existing tag is in sync, do nothing
if check_existing_git_tag(
repo,
t_tag,
ignore_git_message=ignore_git_message,
ignore_git_tagger=ignore_git_tagger,
):
warn(textwrap.dedent("""\
No action for {repo}
existing tag: {tag} is already in sync\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
continue
except github.RateLimitExceededException:
raise
except GitTagExistsError as e:
# if force_tag is set, and the tag already exists, set
# update_tag and fall through. Otherwise, treat it as any other
# exception.
if force_tag:
update_tag = True
warn(textwrap.dedent("""\
existing tag: {tag} WILL BE MOVED\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
elif fail_fast:
raise
else:
problems.append(e)
error(e)
continue
except github.GithubException as e:
msg = "error checking for existance of tag: {t}".format(
t=t_tag.name,
)
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
else:
problems.append(yikes)
error(yikes)
continue
checked_products[name] = data.copy()
checked_products[name]['target_tag'] = t_tag
checked_products[name]['update_tag'] = update_tag
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return checked_products, problems
def create_forks(
dst_org,
src_repos,
fail_fast=False,
dry_run=False
):
assert isinstance(dst_org, github.Organization.Organization),\
type(dst_org)
assert isinstance(src_repos, list), type(src_repos)
repo_count = len(src_repos)
dst_repos = []
skipped_repos = []
problems = []
with pbar.eta_bar(msg='forking', max_value=repo_count) as progress:
repo_idx = 0
for r in src_repos:
progress.update(repo_idx)
repo_idx += 1
# XXX per
# https://developer.github.com/v3/repos/forks/#create-a-fork
# fork creation is async and pygithub doesn't appear to wait.
# https://github.com/PyGithub/PyGithub/blob/c44469965e4ea368b78c4055a8afcfcf08314585/github/Organization.py#L321-L336
# so its possible that this may fail in some strange way such as
# not returning all repo data, but it hasn't yet been observed.
# get current time before API call in case fork creation is slow.
now = datetime.datetime.now()
debug("forking {r}".format(r=r.full_name))
if dry_run:
debug(' (noop)')
continue
try:
fork = dst_org.create_fork(r)
dst_repos.append(fork)
debug(" -> {r}".format(r=fork.full_name))
except github.RateLimitExceededException:
raise
except github.GithubException as e:
if 'Empty repositories cannot be forked.' in e.data['message']:
warn("{r} is empty and can not be forked".format(
r=r.full_name
))
skipped_repos.append(r)
continue
msg = "error forking repo {r}".format(r=r.full_name)
yikes = pygithub.CaughtOrganizationError(dst_org, e, msg)
if fail_fast:
raise yikes from None
problems.append(yikes)
error(yikes)
if fork.created_at < now:
warn("fork of {r} already exists\n created_at {ctime}".format(
r=fork.full_name,
ctime=fork.created_at
))
return dst_repos, skipped_repos, problems
def check_product_tags(
products,
git_tag,
tag_message_template,
tagger,
force_tag=False,
fail_fast=False,
ignore_git_message=False,
ignore_git_tagger=False,
):
assert isinstance(tagger, github.InputGitAuthor), type(tagger)
checked_products = {}
problems = []
for name, data in products.items():
repo = data['repo']
tag_name = git_tag
# prefix tag name with `v`?
if data['v'] and re.match('\d', tag_name):
tag_name = "v{git_tag}".format(git_tag=tag_name)
# message can not be formatted until we've determined if the tag must
# be prefixed. The 'v' prefix appearing in the tag message is required
# to match historical behavior and allow verification of past releases.
message = tag_message_template.format(git_tag=tag_name)
# "target tag"
t_tag = codekit.pygithub.TargetTag(
name=tag_name,
sha=data['sha'],
message=message,
tagger=tagger,
)
# control whether to create a new tag or update an existing one
update_tag = False
try:
# if the existing tag is in sync, do nothing
if check_existing_git_tag(
repo,
t_tag,
ignore_git_message=ignore_git_message,
ignore_git_tagger=ignore_git_tagger,
):
warn(
textwrap.dedent("""\
No action for {repo}
existing tag: {tag} is already in sync\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
continue
except github.RateLimitExceededException:
raise
except GitTagExistsError as e:
# if force_tag is set, and the tag already exists, set
# update_tag and fall through. Otherwise, treat it as any other
# exception.
if force_tag:
update_tag = True
warn(
textwrap.dedent("""\
existing tag: {tag} WILL BE MOVED\
""").format(
repo=repo.full_name,
tag=t_tag.name,
))
elif fail_fast:
raise
else:
problems.append(e)
error(e)
continue
except github.GithubException as e:
msg = "error checking for existance of tag: {t}".format(
t=t_tag.name, )
yikes = pygithub.CaughtRepositoryError(repo, e, msg)
if fail_fast:
raise yikes from None
else:
problems.append(yikes)
error(yikes)
continue
checked_products[name] = data.copy()
checked_products[name]['target_tag'] = t_tag
checked_products[name]['update_tag'] = update_tag
if problems:
error("{n} product(s) have error(s)".format(n=len(problems)))
return checked_products, problems
def run():
"""Move the repos"""
args = parse_args()
codetools.setup_logging(args.debug)
global g
g = pygithub.login_github(token_path=args.token_path, token=args.token)
org = g.get_organization(args.org)
# only iterate over all teams once
try:
teams = list(org.get_teams())
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = 'error getting teams'
raise pygithub.CaughtOrganizationError(org, e, msg) from None
old_team = find_team(teams, args.oldteam)
new_team = find_team(teams, args.newteam)
move_me = args.repos
debug(len(move_me), 'repos to be moved')
added = []
removed = []
for name in move_me:
try:
r = org.get_repo(name)
except github.RateLimitExceededException:
raise
except github.GithubException as e:
msg = "error getting repo by name: {r}".format(r=name)
raise pygithub.CaughtOrganizationError(org, e, msg) from None
# Add team to the repo
debug("Adding {repo} to '{team}' ...".format(repo=r.full_name,
team=args.newteam))
if not args.dry_run:
try:
new_team.add_to_repos(r)
added += r.full_name
debug(' ok')
except github.RateLimitExceededException:
raise
except github.GithubException as e:
debug(' FAILED')
if old_team.name in 'Owners':
warn("Removing repo {repo} from team 'Owners' is not allowed".
format(repo=r.full_name))
debug("Removing {repo} from '{team}' ...".format(repo=r.full_name,
team=args.oldteam))
if not args.dry_run:
try:
old_team.remove_from_repos(r)
removed += r.full_name
debug(' ok')
except github.RateLimitExceededException:
raise
except github.GithubException as e:
debug(' FAILED')
info('Added:', added)
info('Removed:', removed)