def create_forks(dst_org, src_repos, fail_fast=False, dry_run=False): assert isinstance(dst_org, github.Organization.Organization),\ type(dst_org) assert isinstance(src_repos, list), type(src_repos) repo_count = len(src_repos) dst_repos = [] skipped_repos = [] problems = [] with pbar.eta_bar(msg='forking', max_value=repo_count) as progress: repo_idx = 0 for r in src_repos: progress.update(repo_idx) repo_idx += 1 # XXX per # https://developer.github.com/v3/repos/forks/#create-a-fork # fork creation is async and pygithub doesn't appear to wait. # https://github.com/PyGithub/PyGithub/blob/c44469965e4ea368b78c4055a8afcfcf08314585/github/Organization.py#L321-L336 # so its possible that this may fail in some strange way such as # not returning all repo data, but it hasn't yet been observed. # get current time before API call in case fork creation is slow. now = datetime.datetime.now() debug("forking {r}".format(r=r.full_name)) if dry_run: debug(' (noop)') continue try: fork = dst_org.create_fork(r) dst_repos.append(fork) debug(" -> {r}".format(r=fork.full_name)) except github.RateLimitExceededException: raise except github.GithubException as e: if 'Empty repositories cannot be forked.' in e.data['message']: warn("{r} is empty and can not be forked".format( r=r.full_name)) skipped_repos.append(r) continue msg = "error forking repo {r}".format(r=r.full_name) yikes = pygithub.CaughtOrganizationError(dst_org, e, msg) if fail_fast: raise yikes from None problems.append(yikes) error(yikes) if fork.created_at < now: warn("fork of {r} already exists\n created_at {ctime}".format( r=fork.full_name, ctime=fork.created_at)) return dst_repos, skipped_repos, problems
def get_candidate_teams(org, target_teams): assert isinstance(org, github.Organization.Organization), type(org) try: teams = list(org.get_teams()) except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting teams' raise pygithub.CaughtOrganizationError(org, e, msg) from None debug("looking for teams: {teams}".format(teams=target_teams)) tag_teams = [t for t in teams if t.name in target_teams] debug("found teams: {teams}".format(teams=tag_teams)) if not tag_teams: raise RuntimeError('No teams found') return tag_teams
def delete_all_repos(org, **kwargs): assert isinstance(org, github.Organization.Organization), type(org) limit = kwargs.pop('limit', None) try: repos = list(itertools.islice(org.get_repos(), limit)) except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting repos' raise pygithub.CaughtOrganizationError(org, e, msg) from None info("found {n} repos in {org}".format(n=len(repos), org=org.login)) [debug(" {r}".format(r=r.full_name)) for r in repos] if repos: warn("Deleting all repos in {org}".format(org=org.login)) pbar.wait_for_user_panic_once() return delete_repos(repos, **kwargs)
def delete_all_teams(org, **kwargs): assert isinstance(org, github.Organization.Organization), type(org) limit = kwargs.pop('limit', None) try: teams = list(itertools.islice(org.get_teams(), limit)) except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting teams' raise pygithub.CaughtOrganizationError(org, e, msg) from None info("found {n} teams in {org}".format(n=len(teams), org=org.login)) [debug(" '{t}'".format(t=t.name)) for t in teams] if teams: warn("Deleting all teams in {org}".format(org=org.login)) pbar.wait_for_user_panic_once() return delete_teams(teams, **kwargs)
def run(): """List repos and teams""" args = parse_args() codetools.setup_logging(args.debug) global g g = pygithub.login_github(token_path=args.token_path, token=args.token) if not args.hide: args.hide = [] org = g.get_organization(args.organization) try: repos = list(org.get_repos()) except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting repos' raise pygithub.CaughtOrganizationError(org, e, msg) from None for r in repos: try: teamnames = [ t.name for t in r.get_teams() if t.name not in args.hide ] except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting teams' raise pygithub.CaughtRepositoryError(r, e, msg) from None maxt = args.maxt if (args.maxt is not None and args.maxt >= 0) else len(teamnames) if args.debug: print("MAXT=", maxt) if args.mint <= len(teamnames) <= maxt: print(r.name.ljust(40) + args.delimiter.join(teamnames))
def run(): args = parse_args() codetools.setup_logging(args.debug) global g g = pygithub.login_github(token_path=args.token_path, token=args.token) # protect destination org codetools.validate_org(args.dst_org) src_org = g.get_organization(args.src_org) dst_org = g.get_organization(args.dst_org) info("forking repos from: {org}".format(org=src_org.login)) info(" to: {org}".format(org=dst_org.login)) debug('looking for repos -- this can take a while for large orgs...') if args.team: debug('checking that selection team(s) exist') try: org_teams = list(src_org.get_teams()) except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting teams' raise pygithub.CaughtOrganizationError(src_org, e, msg) from None missing_teams = [ n for n in args.team if n not in [t.name for t in org_teams] ] if missing_teams: error("{n} team(s) do not exist:".format(n=len(missing_teams))) [error(" '{t}'".format(t=n)) for n in missing_teams] return fork_teams = [t for t in org_teams if t.name in args.team] repos = pygithub.get_repos_by_team(fork_teams) debug('selecting repos by membership in team(s):') [debug(" '{t}'".format(t=t.name)) for t in fork_teams] else: repos = pygithub.get_repos_by_team(fork_teams) src_repos = list(itertools.islice(repos, args.limit)) repo_count = len(src_repos) if not repo_count: debug('nothing to do -- exiting') return debug("found {n} repos to be forked from org {src_org}:".format( n=repo_count, src_org=src_org.login)) [debug(" {r}".format(r=r.full_name)) for r in src_repos] if args.copy_teams: debug('checking source repo team membership...') # dict of repo and team objects, keyed by repo name src_rt = find_teams_by_repo(src_repos) # extract a non-duplicated list of team names from all repos being # forked as a dict, keyed by team name src_teams = find_used_teams(src_rt) debug('found {n} teams in use within org {o}:'.format(n=len(src_teams), o=src_org.login)) [debug(" '{t}'".format(t=t)) for t in src_teams.keys()] # check for conflicting teams in dst org before attempting to create # any forks so its possible to bail out before any resources have been # created. debug('checking teams in destination org') conflicting_teams = pygithub.get_teams_by_name(dst_org, list(src_teams.keys())) if conflicting_teams: raise TeamError( "found {n} conflicting teams in {o}: {teams}".format( n=len(conflicting_teams), o=dst_org.login, teams=[t.name for t in conflicting_teams])) debug('there is no spoon...') problems = [] pygithub.debug_ratelimit(g) dst_repos, skipped_repos, err = create_forks(dst_org, src_repos, fail_fast=args.fail_fast, dry_run=args.dry_run) if err: problems += err if args.copy_teams: # filter out repos which were skipped # dict of str(fork_repo.name): fork_repo dst_forks = dict((r.name, r) for r in dst_repos) bad_repos = dict((r.name, r) for r in skipped_repos) # dict of str(team.name): [repos] to be created dst_teams = {} for name, repos in src_teams.items(): dst_teams[name] = [ dst_forks[r.name] for r in repos if r.name not in bad_repos ] _, err = create_teams(dst_org, dst_teams, with_repos=True, fail_fast=args.fail_fast, dry_run=args.dry_run) if err: problems += err if problems: msg = "{n} errors forking repo(s)/teams(s)".format(n=len(problems)) raise codetools.DogpileError(problems, msg)
def create_teams(org, teams, with_repos=False, ignore_existing=False, fail_fast=False, dry_run=False): assert isinstance(org, github.Organization.Organization), type(org) assert isinstance(teams, dict), type(teams) # it takes fewer api calls to create team(s) with an explicit list of # members after all repos have been forked but this blows up if the team # already exists. debug("creating teams in {org}".format(org=org.login)) # dict of dst org teams keyed by name (str) with team object as value dst_teams = {} problems = [] batch_repos = 50 for name, repos in teams.items(): pygithub.debug_ratelimit(g) debug("creating team {o}/'{t}'".format(o=org.login, t=name)) if dry_run: debug(' (noop)') continue dst_t = None try: if with_repos: debug(" with {n} member repos:".format(n=len(repos))) [debug(" {r}".format(r=r.full_name)) for r in repos] leftover_repos = repos[batch_repos:] if leftover_repos: debug(" creating team with first {b} of {n} repos".format( b=batch_repos, n=len(repos))) dst_t = org.create_team(name, repo_names=repos[:batch_repos]) if leftover_repos: # add any repos over the batch limit individually to team for r in leftover_repos: debug(" adding repo {r}".format(r=r.full_name)) dst_t.add_to_repos(r) else: dst_t = org.create_team(name) except github.RateLimitExceededException: raise except github.GithubException as e: # if the error is for any cause other than the team already # existing, puke. team_exists = False if ignore_existing and 'errors' in e.data: for oops in e.data['errors']: msg = oops['message'] if 'Name has already been taken' in msg: # find existing team dst_t = pygithub.get_teams_by_name(org, name)[0] team_exists = True if not (ignore_existing and team_exists): msg = "error creating team: {t}".format(t=name) yikes = pygithub.CaughtOrganizationError(org, e, msg) if fail_fast: raise yikes from None problems.append(yikes) error(yikes) break else: dst_teams[dst_t.name] = dst_t return dst_teams, problems
def get_repo_for_products(org, products, allow_teams, ext_teams, deny_teams, fail_fast=False): debug("allowed teams: {allow}".format(allow=allow_teams)) debug("external teams: {ext}".format(ext=ext_teams)) debug("denied teams: {deny}".format(deny=deny_teams)) resolved_products = {} problems = [] for name, data in products.items(): debug("looking for git repo for: {name} [{ver}]".format( name=name, ver=data['eups_version'])) try: repo = org.get_repo(name) except github.RateLimitExceededException: raise except github.GithubException as e: msg = "error getting repo by name: {r}".format(r=name) yikes = pygithub.CaughtOrganizationError(org, e, msg) if fail_fast: raise yikes from None problems.append(yikes) error(yikes) continue debug(" found: {slug}".format(slug=repo.full_name)) try: repo_team_names = [t.name for t in repo.get_teams()] except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting teams' yikes = pygithub.CaughtRepositoryError(repo, e, msg) if fail_fast: raise yikes from None problems.append(yikes) error(yikes) continue debug(" teams: {teams}".format(teams=repo_team_names)) try: pygithub.check_repo_teams(repo, allow_teams=allow_teams, deny_teams=deny_teams, team_names=repo_team_names) except pygithub.RepositoryTeamMembershipError as e: if fail_fast: raise problems.append(e) error(e) continue has_ext_team = any(x in repo_team_names for x in ext_teams) debug(" external repo: {v}".format(v=has_ext_team)) resolved_products[name] = data.copy() resolved_products[name]['repo'] = repo resolved_products[name]['v'] = has_ext_team if problems: error("{n} product(s) have error(s)".format(n=len(problems))) return resolved_products, problems
def run(): """Move the repos""" args = parse_args() codetools.setup_logging(args.debug) global g g = pygithub.login_github(token_path=args.token_path, token=args.token) org = g.get_organization(args.org) # only iterate over all teams once try: teams = list(org.get_teams()) except github.RateLimitExceededException: raise except github.GithubException as e: msg = 'error getting teams' raise pygithub.CaughtOrganizationError(org, e, msg) from None old_team = find_team(teams, args.oldteam) new_team = find_team(teams, args.newteam) move_me = args.repos debug(len(move_me), 'repos to be moved') added = [] removed = [] for name in move_me: try: r = org.get_repo(name) except github.RateLimitExceededException: raise except github.GithubException as e: msg = "error getting repo by name: {r}".format(r=name) raise pygithub.CaughtOrganizationError(org, e, msg) from None # Add team to the repo debug("Adding {repo} to '{team}' ...".format(repo=r.full_name, team=args.newteam)) if not args.dry_run: try: new_team.add_to_repos(r) added += r.full_name debug(' ok') except github.RateLimitExceededException: raise except github.GithubException as e: debug(' FAILED') if old_team.name in 'Owners': warn("Removing repo {repo} from team 'Owners' is not allowed". format(repo=r.full_name)) debug("Removing {repo} from '{team}' ...".format(repo=r.full_name, team=args.oldteam)) if not args.dry_run: try: old_team.remove_from_repos(r) removed += r.full_name debug(' ok') except github.RateLimitExceededException: raise except github.GithubException as e: debug(' FAILED') info('Added:', added) info('Removed:', removed)