def close_issue(
title_regex: str, repos: Iterable[plug.StudentRepo], api: plug.PlatformAPI
) -> None:
"""Close issues whose titles match the title_regex in student repos.
Args:
title_regex: A regex to match against issue titles.
assignment_names: Names of assignments.
teams: Team objects specifying student groups.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
repo_urls = (repo.url for repo in repos)
platform_repos = progresswrappers.get_repos(repo_urls, api)
for repo in platform_repos:
to_close = [
issue
for issue in api.get_repo_issues(repo)
if re.match(title_regex, issue.title)
and issue.state == plug.IssueState.OPEN
]
for issue in to_close:
api.close_issue(issue)
msg = f"Closed {repo.name}/#{issue.number}='{issue.title}'"
platform_repos.write(msg) # type: ignore
plug.log.info(msg)
def end_reviews(
assignment_names: Iterable[str],
students: Iterable[plug.StudentTeam],
double_blind_key: Optional[str],
api: plug.PlatformAPI,
) -> None:
"""Clean up review allocations.
If normal no-blind review has been performed (i.e. ``double_blind_key`` is
``None``), then only review teams are deleted. If ``double_blind_key`` is
provided, both review teams and anonymous repo copies are deleted.
Args:
assignment_names: Names of assignments.
students: An iterble of student teams.
double_blind_key: If not None, double-blind review is assumed and the
key is used to compute hashed review team names.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
review_team_names = [
_review_team_name(student, assignment_name, double_blind_key)
for student in students for assignment_name in assignment_names
]
teams = progresswrappers.get_teams(review_team_names,
api,
desc="Deleting review teams")
for team in teams:
api.delete_team(team)
plug.log.info(f"Deleted team {team.name}")
progresswrappers.end_progress(teams)
if double_blind_key:
_delete_anonymous_repos(assignment_names, students, double_blind_key,
api)
def _create_update_push_tuples(
teams: Iterable[plug.StudentTeam],
template_repos: Iterable[plug.TemplateRepo],
api: plug.PlatformAPI,
) -> Iterable[PushSpec]:
"""Create push tuples for existing repos. Repos that don't exist are
ignored.
Args:
teams: An iterable of teams.
template_repos: Template repositories.
api: A platform API instance.
Returns:
A list of PushSpec namedtuples for all student repo urls that relate to
any of the master repo urls.
"""
urls_to_templates = {}
for team, template_repo in itertools.product(teams, template_repos):
repo_url, *_ = api.get_repo_urls(
[template_repo.name], team_names=[team.name]
)
urls_to_templates[repo_url] = template_repo
for repo in api.get_repos(list(urls_to_templates.keys())):
template = urls_to_templates[repo.url]
branch = git.active_branch(template.path)
yield PushSpec(template.path, api.insert_auth(repo.url), branch)
def check_peer_review_progress(
assignment_names: Iterable[str],
teams: Iterable[plug.Team],
title_regex: str,
num_reviews: int,
api: plug.PlatformAPI,
) -> None:
"""Check which teams have opened peer review issues in their allotted
review repos
Args:
assignment_names: Names of assignments.
teams: An iterable of student teams.
title_regex: A regex to match against issue titles.
num_reviews: Amount of reviews each student is expected to have made.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
teams = list(teams)
reviews = collections.defaultdict(list)
review_team_names = [
plug.generate_review_team_name(team, assignment_name) for team in teams
for assignment_name in assignment_names
]
review_teams = progresswrappers.get_teams(review_team_names,
api,
desc="Processing review teams")
for review_team in review_teams:
repos = list(api.get_team_repos(review_team))
if len(repos) != 1:
plug.log.warning(
f"Expected {review_team.name} to have 1 associated "
f"repo, found {len(repos)}. "
f"Skipping...")
continue
reviewed_repo = repos[0]
expected_reviewers = set(review_team.members)
reviewing_teams = _extract_reviewing_teams(teams, expected_reviewers)
review_issue_authors = {
issue.author
for issue in api.get_repo_issues(reviewed_repo)
if re.match(title_regex, issue.title)
}
for team in reviewing_teams:
reviews[str(team)].append(
plug.Review(
repo=reviewed_repo.name,
done=any(
map(review_issue_authors.__contains__, team.members)),
))
plug.echo(
formatters.format_peer_review_progress_output(
reviews, [team.name for team in teams], num_reviews))
def create_teams(
teams: Iterable[plug.StudentTeam],
permission: plug.TeamPermission,
api: plug.PlatformAPI,
) -> Iterable[plug.Team]:
"""Create teams.
Args:
teams: An iterable of teams to create.
permission: The permission to assign to the team with respect to its
repositories.
api: A platform API instance.
Returns:
An iterable of wrappers around created teams.
"""
teams = list(teams)
existing_teams_dict = {
existing.name: existing
for existing in api.get_teams({t.name for t in teams})
}
for required_team in teams:
team = existing_teams_dict.get(required_team.name) or api.create_team(
required_team.name,
members=required_team.members,
permission=permission,
)
existing_members = set(team.members)
new_members = set(required_team.members) - existing_members
api.assign_members(team, new_members, permission)
# FIXME the returned team won't have the correct members if any new
# ones are added. This should be fixed by disconnecting members
# from teams, and having an api call "get_team_members"
yield team
def _delete_anonymous_repos(
assignment_names: Iterable[str],
student_teams: Iterable[plug.StudentTeam],
double_blind_key: str,
api: plug.PlatformAPI,
):
"""Delete any anonymous repos created for these students and
assignments.
"""
anon_repo_names = [
_hash_if_key(
plug.generate_repo_name(student_team, assignment_name),
key=double_blind_key,
) for student_team, assignment_name in itertools.product(
student_teams, assignment_names)
]
anon_repo_urls = api.get_repo_urls(anon_repo_names)
anon_repos = api.get_repos(anon_repo_urls)
anon_repos_progress = plug.cli.io.progress_bar(
anon_repos,
desc="Deleting anonymous repo copies",
total=len(anon_repo_names),
)
for repo in anon_repos_progress:
api.delete_repo(repo)
progresswrappers.end_progress(anon_repos_progress)
def callback(args: argparse.Namespace, api: plug.PlatformAPI) -> None:
repo_name_to_team: Mapping[str, plug.StudentTeam] = {
plug.generate_repo_name(student_team.name, assignment_name):
student_team
for student_team in args.students
for assignment_name in args.assignments
}
repo_names = list(repo_name_to_team.keys())
if "multi_issues_file" in args and args.multi_issues_file is not None:
issues_file = pathlib.Path(args.multi_issues_file).resolve()
all_issues = _parse_multi_issues_file(issues_file)
else:
issues_dir = pathlib.Path(args.issues_dir).resolve()
all_issues = _collect_issues(repo_names, issues_dir)
issues = _extract_expected_issues(all_issues, repo_names,
args.allow_missing)
for repo_name, issue in issues:
open_issue = args.batch_mode or _ask_for_open(issue, repo_name,
args.truncation_length)
if open_issue:
repo = api.get_repo(repo_name, repo_name_to_team[repo_name].name)
api.create_issue(issue.title, issue.body, repo)
else:
plug.echo("Skipping {}".format(repo_name))
def open_issues_from_hook_results(
hook_results: Mapping[str, List[plug.Result]],
repos: Iterable[plug.StudentRepo],
api: plug.PlatformAPI,
) -> None:
"""Open all issues from the hook results in the given repos. Issues given
in the hook results that do not belong to the repos are ignored, and repos
provided without corresponding issues in the hook results have no effect.
Args:
hook_results: A hook results dictionary.
repos: Student repos to open issues in.
api: plug.PlatformAPI,
"""
url_to_repo = {repo.url: repo for repo in repos}
for repo_url, repo_data in hook_results["repos"][0].data.items():
if repo_url in url_to_repo and repo_data["issues"]:
repo = url_to_repo[repo_url]
platform_repo = api.get_repo(repo.name, repo.team.name)
for issue_data in repo_data["issues"].values():
issue = api.create_issue(issue_data["title"],
issue_data["body"], platform_repo)
msg = (
f"Opened issue {repo.name}/#{issue.number}-'{issue.title}'"
)
plug.echo(msg)
def _get_or_create_team(team_name: str, api: plug.PlatformAPI) -> plug.Team:
matches = api.get_teams(team_names=[team_name])
try:
return next(iter(matches))
except StopIteration:
return api.create_team(TEACHERS_TEAM_NAME,
permission=plug.TeamPermission.PULL)
def command(self, api: plug.PlatformAPI):
team = api.get_teams(team_names=[self.team_name])
api.create_repo(
self.repo_name,
description=description,
private=private,
team=team,
)
def end_reviews_repobee_4(allocations_file: pathlib.Path,
api: plug.PlatformAPI) -> None:
"""Preview version of RepoBee 4's version of :py:fync:`end_reviews`."""
review_allocations = json.loads(
allocations_file.read_text(sys.getdefaultencoding()))["allocations"]
review_team_names = {
allocation["review_team"]["name"]
for allocation in review_allocations
}
for team in progresswrappers.get_teams(review_team_names, api):
api.delete_team(team)
def post_setup(self, repo: plug.StudentRepo, api: plug.PlatformAPI):
"""Add a created student repo to the teachers team."""
platform_repo = next(iter(api.get_repos([repo.url])))
teachers_team = _get_or_create_team(TEACHERS_TEAM_NAME, api)
api.assign_repo(
team=teachers_team,
repo=platform_repo,
permission=plug.TeamPermission.PULL,
)
return plug.Result(
name="tamanager",
status=plug.Status.SUCCESS,
msg=f"Added to the {TEACHERS_TEAM_NAME} team",
)
def _push_to_platform(student_repos: List[plug.StudentRepo],
api: plug.PlatformAPI) -> None:
push_tuples = [
_repobee.git.Push(repo.path, api.insert_auth(repo.url),
_DEFAULT_BRANCH) for repo in student_repos
]
_repobee.git.push(push_tuples)
def _repo_names_to_urls(repo_names: Iterable[str], org_name: str,
api: plug.PlatformAPI) -> Tuple[List[str], List[str]]:
"""Use the repo_names to extract urls to the repos. Look for git
repos with the correct names in the local directory and create local uris
for them. For the rest, create urls to the repos assuming they are in the
target organization. Do note that there is _no_ guarantee that the remote
repos exist as checking this takes too much time with the REST API.
A possible improvement would be to use the GraphQL API for this function.
Args:
repo_names: names of repositories.
org_name: Name of the organization these repos are expected in.
api: An API instance.
Returns:
A tuple of lists with (non_local_urls, local_uris).
Raises:
ParseError: If local templates are found, but allow_local is False.
"""
local = [
name for name in repo_names if util.is_git_repo(os.path.abspath(name))
]
non_local = [name for name in repo_names if name not in local]
non_local_urls = api.get_repo_urls(non_local, org_name)
local_uris = [
pathlib.Path(os.path.abspath(repo_name)).as_uri()
for repo_name in local
]
return non_local_urls, local_uris
def _create_push_tuples(
teams: List[plug.Team],
template_repos: Iterable[plug.TemplateRepo],
api: plug.PlatformAPI,
) -> Iterable[Tuple[bool, PushSpec]]:
"""Create push tuples for newly created repos. Repos that already exist are
ignored.
Args:
teams: An iterable of teams.
template_repos: Template repositories.
api: A platform API instance.
Returns:
A list of tuples (created, push_tuple) for all student repo urls
that relate to any of the master repo urls. ``created`` indicates
whether or not the student repo was created in this invocation.
"""
for team, template_repo in itertools.product(teams, template_repos):
repo_name = plug.generate_repo_name(team, template_repo.name)
created, repo = _create_or_fetch_repo(
name=repo_name,
description=f"{repo_name} created for {team.name}",
private=True,
team=team,
api=api,
)
yield created, PushSpec(
local_path=template_repo.path,
repo_url=api.insert_auth(repo.url),
branch=git.active_branch(template_repo.path),
metadata=dict(repo=repo, team=team),
)
def check_reviews_repobee_4(allocations_file: pathlib.Path, title_regex: str,
api: plug.PlatformAPI) -> None:
"""Preview version of the `reviews check` command for RepoBee 4."""
data = json.loads(allocations_file.read_text(sys.getdefaultencoding()))
review_allocations = data["allocations"]
num_reviews = int(data["num_reviews"])
expected_reviewers = {
allocation["reviewed_repo"]["url"]:
allocation["review_team"]["members"]
for allocation in review_allocations
}
reviewed_repos = progresswrappers.get_repos(expected_reviewers.keys(), api)
reviews = collections.defaultdict(list)
for reviewed_repo in reviewed_repos:
review_issue_authors = {
issue.author
for issue in api.get_repo_issues(reviewed_repo)
if re.match(title_regex, issue.title)
}
for expected_reviewer in expected_reviewers[reviewed_repo.url]:
reviews[expected_reviewer].append(
plug.Review(
repo=reviewed_repo.name,
done=expected_reviewer in review_issue_authors,
))
plug.echo(
formatters.format_peer_review_progress_output(
reviews,
list(itertools.chain.from_iterable(expected_reviewers.values())),
num_reviews,
))
def _try_insert_auth(repo: Union[plug.TemplateRepo, plug.StudentRepo],
api: plug.PlatformAPI) -> str:
"""Try to insert authentication into the URL."""
try:
return api.insert_auth(repo.url)
except plug.InvalidURL:
return repo.url
def update_student_repos(
template_repo_urls: plug.types.SizedIterable[str],
teams: plug.types.SizedIterable[plug.StudentTeam],
api: plug.PlatformAPI,
issue: Optional[plug.Issue] = None,
) -> Mapping[str, List[plug.Result]]:
"""Attempt to update all student repos related to one of the master repos.
Args:
template_repo_urls: URLs to master repos. Must be in the organization
that the api is set up for.
teams: An iterable of student teams.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
issue: An optional issue to open in repos to which pushing fails.
"""
if len(set(template_repo_urls)) != len(template_repo_urls):
raise ValueError("template_repo_urls contains duplicates")
with tempfile.TemporaryDirectory() as tmpdir:
workdir = pathlib.Path(tmpdir)
template_repos = [
plug.TemplateRepo(
name=urlutil.extract_repo_name(url),
url=url,
_path=workdir / api.extract_repo_name(url),
)
for url in template_repo_urls
]
plug.log.info("Cloning into master repos ...")
_clone_all(template_repos, cwd=workdir, api=api)
hook_results = plugin.execute_setup_tasks(
template_repos, api, cwd=pathlib.Path(tmpdir)
)
push_tuple_iter = _create_update_push_tuples(
teams, template_repos, api
)
push_tuple_iter_progress = plug.cli.io.progress_bar(
push_tuple_iter,
desc="Setting up student repos",
total=len(teams) * len(template_repos),
)
successful_pts, failed_pts = git.push(
push_tuples=push_tuple_iter_progress
)
if failed_pts and issue:
plug.echo("Opening issue in repos to which push failed")
urls_without_auth = [
re.sub("https://.*?@", "https://", pt.repo_url)
for pt in failed_pts
]
_open_issue_by_urls(urls_without_auth, issue, api)
plug.log.info("Done!")
return hook_results
def setup_student_repos(
template_repo_urls: Iterable[str],
teams: Iterable[plug.StudentTeam],
api: plug.PlatformAPI,
) -> Mapping[str, List[plug.Result]]:
"""Setup student repositories based on master repo templates. Performs three
primary tasks:
1. Create the specified teams on the target platform and add the
specified members to their teams. If a team already exists, it is left
as-is. If a student is already in a team they are assigned to, nothing
happens. If no account exists for some specified username, that
particular student is ignored, but any associated teams are still
created (even if a missing user is the only member of that team).
2. For each master repository, create one student repo per team and add
it to the corresponding student team. If a repository already exists,
it is skipped.
3. Push files from the master repos to the corresponding student repos.
Args:
template_repo_urls: URLs to master repos.
teams: An iterable of student teams specifying the teams to be setup.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
teams = list(teams)
with tempfile.TemporaryDirectory() as tmpdir:
workdir = pathlib.Path(tmpdir)
template_repos = [
plug.TemplateRepo(
name=urlutil.extract_repo_name(url),
url=url,
_path=workdir / api.extract_repo_name(url),
)
for url in template_repo_urls
]
plug.log.info("Cloning into master repos ...")
_clone_all(template_repos, cwd=workdir, api=api)
pre_setup_results = plugin.execute_setup_tasks(
template_repos, api, cwd=pathlib.Path(tmpdir)
)
platform_teams = _create_platform_teams(teams, api)
to_push, preexisting = _create_state_separated_push_tuples(
platform_teams, template_repos, api
)
successful_pts, _ = git.push(push_tuples=to_push)
post_setup_results = _execute_post_setup_hook(
successful_pts, preexisting, api
)
return _combine_dicts(pre_setup_results, post_setup_results)
def _get_anonymous_review_team(
student_team: plug.StudentTeam,
assignment: str,
key: str,
api: plug.PlatformAPI,
) -> plug.Team:
review_team, *_ = list(
api.get_teams([_repobee.hash.keyed_hash(student_team.name, key, 20)]))
return review_team
def _repo_tuple_generator(
assignment_names: List[str],
teams: List[plug.StudentTeam],
api: plug.PlatformAPI,
) -> Iterable[plug.StudentRepo]:
for assignment_name in assignment_names:
for team in teams:
url, *_ = api.get_repo_urls([assignment_name],
team_names=[team.name])
name = plug.generate_repo_name(team, assignment_name)
yield plug.StudentRepo(name=name, url=url, team=team)
def _update_local_repos(local: List[plug.StudentRepo],
api: plug.PlatformAPI) -> None:
expected_basedir = local[0].path.parent.parent
assert all(
map(lambda repo: repo.path.parent.parent == expected_basedir, local))
specs = [
CloneSpec(repo_url=api.insert_auth(repo.url), dest=repo.path)
for repo in local
]
# TODO figure out what to do when a local update fails
clone(specs)
def command(self, api: plug.PlatformAPI) -> Optional[plug.Result]:
teachers_team = _get_or_create_team(TEACHERS_TEAM_NAME, api)
existing_members = teachers_team.members
new_members = list(set(self.teachers) - set(existing_members))
api.assign_members(teachers_team,
new_members,
permission=plug.TeamPermission.PULL)
for repo in plug.cli.io.progress_bar(
api.get_repos(), desc="Granting read access to repos"):
api.assign_repo(
repo=repo,
team=teachers_team,
permission=plug.TeamPermission.PULL,
)
msg = (f"Added {', '.join(new_members)} to the '{TEACHERS_TEAM_NAME}' "
"team")
return plug.Result(name="add-teachers",
status=plug.Status.SUCCESS,
msg=msg)
def _discover_repos(student_teams: List[plug.StudentTeam],
api: plug.PlatformAPI) -> Iterable[plug.StudentRepo]:
student_teams_dict = {t.name: t for t in student_teams}
fetched_teams = progresswrappers.get_teams(student_teams,
api,
desc="Discovering team repos")
for team in fetched_teams:
repos = api.get_team_repos(team)
yield from (plug.StudentRepo(
name=repo.name,
url=repo.url,
team=student_teams_dict[team.name],
) for repo in repos)
def _get_issue_generator(
repos: Iterable[plug.StudentRepo],
title_regex: str,
author: Optional[str],
state: plug.IssueState,
double_blind_key: Optional[str],
api: plug.PlatformAPI,
) -> Iterable[Tuple[plug.StudentRepo, Iterable[plug.Issue]]]:
for repo in repos:
if double_blind_key:
team_name = _hash_if_key(repo.team.name, double_blind_key)
repo_name = _hash_if_key(repo.name, double_blind_key)
platform_repo = api.get_repo(repo_name, team_name)
else:
platform_repo = api.get_repo(repo.name, repo.team.name)
yield repo, [
issue for issue in api.get_repo_issues(platform_repo)
if re.match(title_regex, issue.title) and (
state == plug.IssueState.ALL or state == issue.state) and (
not author or issue.author == author)
]
def open_issue(
issue: plug.Issue,
assignment_names: Iterable[str],
teams: Iterable[plug.StudentTeam],
api: plug.PlatformAPI,
) -> None:
"""Open an issue in student repos.
Args:
assignment_names: Names of assignments.
teams: Team objects specifying student groups.
issue: An issue to open.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
repo_urls = api.get_repo_urls(team_names=[t.name for t in teams],
assignment_names=assignment_names)
repos = progresswrappers.get_repos(repo_urls, api)
for repo in repos:
issue = api.create_issue(issue.title, issue.body, repo)
msg = f"Opened issue {repo.name}/#{issue.number}-'{issue.title}'"
repos.write(msg) # type: ignore
plug.log.info(msg)
def _create_or_fetch_repo(
name: str,
description: str,
private: bool,
api: plug.PlatformAPI,
team: Optional[plug.Team] = None,
) -> Tuple[bool, plug.Repo]:
try:
return (
True,
api.create_repo(
name, description=description, private=private, team=team
),
)
except plug.PlatformError:
team_name = team.name if team else None
repo = api.get_repo(repo_name=name, team_name=team_name)
if team:
api.assign_repo(
team=team, repo=repo, permission=plug.TeamPermission.PUSH
)
return False, repo
def migrate_repos(
template_repo_urls: plug.types.SizedIterable[str], api: plug.PlatformAPI
) -> None:
"""Migrate a repository from an arbitrary URL to the target organization.
The new repository is added to the master_repos team, which is created if
it does not already exist.
Args:
template_repo_urls: Local urls to repos to migrate.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
local_templates = [
plug.TemplateRepo(name=urlutil.extract_repo_name(url), url=url)
for url in template_repo_urls
]
create_repo_it = plug.cli.io.progress_bar(
(
_create_or_fetch_repo(
local.name, description="", private=True, api=api
)
for local in local_templates
),
desc="Creating remote repos",
total=len(template_repo_urls),
)
with tempfile.TemporaryDirectory() as tmpdir:
workdir = pathlib.Path(tmpdir)
_clone_all(local_templates, cwd=workdir, api=api)
remote_templates = [
plug.TemplateRepo(
name=repo.name, url=repo.url, _path=workdir / repo.name
)
for _, repo in create_repo_it
]
git.push(
[
PushSpec(
local_path=template_repo.path,
repo_url=api.insert_auth(template_repo.url),
branch=git.active_branch(template_repo.path),
)
for template_repo in remote_templates
]
)
plug.echo("Done!")
def purge_review_teams(
assignment_names: Iterable[str],
students: Iterable[plug.StudentTeam],
api: plug.PlatformAPI,
) -> None:
"""Delete all review teams associated with the given assignment names and
student teams.
Args:
assignment_names: Names of assignments.
students: An iterble of student teams.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
review_team_names = [
plug.generate_review_team_name(student, assignment_name)
for student in students for assignment_name in assignment_names
]
teams = progresswrappers.get_teams(review_team_names,
api,
desc="Deleting review teams")
for team in teams:
api.delete_team(team)
plug.log.info(f"Deleted {team.name}")
def _open_issue_by_urls(repo_urls: Iterable[str], issue: plug.Issue,
api: plug.PlatformAPI) -> None:
"""Open issues in the repos designated by the repo_urls.
Args:
repo_urls: URLs to repos in which to open an issue.
issue: An issue to open.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
repos = progresswrappers.get_repos(repo_urls, api)
for repo in repos:
issue = api.create_issue(issue.title, issue.body, repo)
msg = f"Opened issue {repo.name}/#{issue.number}-'{issue.title}'"
repos.write(msg) # type: ignore
plug.log.info(msg)