def migrate_repos(
template_repo_urls: plug.types.SizedIterable[str], api: plug.PlatformAPI
) -> None:
"""Migrate a repository from an arbitrary URL to the target organization.
The new repository is added to the master_repos team, which is created if
it does not already exist.
Args:
template_repo_urls: Local urls to repos to migrate.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
local_templates = [
plug.TemplateRepo(name=urlutil.extract_repo_name(url), url=url)
for url in template_repo_urls
]
create_repo_it = plug.cli.io.progress_bar(
(
_create_or_fetch_repo(
local.name, description="", private=True, api=api
)
for local in local_templates
),
desc="Creating remote repos",
total=len(template_repo_urls),
)
with tempfile.TemporaryDirectory() as tmpdir:
workdir = pathlib.Path(tmpdir)
_clone_all(local_templates, cwd=workdir, api=api)
remote_templates = [
plug.TemplateRepo(
name=repo.name, url=repo.url, _path=workdir / repo.name
)
for _, repo in create_repo_it
]
git.push(
[
PushSpec(
local_path=template_repo.path,
repo_url=api.insert_auth(template_repo.url),
branch=git.active_branch(template_repo.path),
)
for template_repo in remote_templates
]
)
plug.echo("Done!")
def test_tries_all_calls_when_repos_up_to_date(self, env_setup,
push_tuples, aio_subproc):
aio_subproc.process.stderr = b"Everything up-to-date"
expected_calls = [
call(
*f"git push {pt.repo_url}".split(),
pt.branch,
cwd=os.path.abspath(pt.local_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) for pt in push_tuples
]
git.push(push_tuples)
aio_subproc.create_subprocess.assert_has_calls(expected_calls)
def setup_student_repos(
template_repo_urls: Iterable[str],
teams: Iterable[plug.StudentTeam],
api: plug.PlatformAPI,
) -> Mapping[str, List[plug.Result]]:
"""Setup student repositories based on master repo templates. Performs three
primary tasks:
1. Create the specified teams on the target platform and add the
specified members to their teams. If a team already exists, it is left
as-is. If a student is already in a team they are assigned to, nothing
happens. If no account exists for some specified username, that
particular student is ignored, but any associated teams are still
created (even if a missing user is the only member of that team).
2. For each master repository, create one student repo per team and add
it to the corresponding student team. If a repository already exists,
it is skipped.
3. Push files from the master repos to the corresponding student repos.
Args:
template_repo_urls: URLs to master repos.
teams: An iterable of student teams specifying the teams to be setup.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
"""
teams = list(teams)
with tempfile.TemporaryDirectory() as tmpdir:
workdir = pathlib.Path(tmpdir)
template_repos = [
plug.TemplateRepo(
name=urlutil.extract_repo_name(url),
url=url,
_path=workdir / api.extract_repo_name(url),
)
for url in template_repo_urls
]
plug.log.info("Cloning into master repos ...")
_clone_all(template_repos, cwd=workdir, api=api)
pre_setup_results = plugin.execute_setup_tasks(
template_repos, api, cwd=pathlib.Path(tmpdir)
)
platform_teams = _create_platform_teams(teams, api)
to_push, preexisting = _create_state_separated_push_tuples(
platform_teams, template_repos, api
)
successful_pts, _ = git.push(push_tuples=to_push)
post_setup_results = _execute_post_setup_hook(
successful_pts, preexisting, api
)
return _combine_dicts(pre_setup_results, post_setup_results)
def update_student_repos(
template_repo_urls: plug.types.SizedIterable[str],
teams: plug.types.SizedIterable[plug.StudentTeam],
api: plug.PlatformAPI,
issue: Optional[plug.Issue] = None,
) -> Mapping[str, List[plug.Result]]:
"""Attempt to update all student repos related to one of the master repos.
Args:
template_repo_urls: URLs to master repos. Must be in the organization
that the api is set up for.
teams: An iterable of student teams.
api: An implementation of :py:class:`repobee_plug.PlatformAPI` used to
interface with the platform (e.g. GitHub or GitLab) instance.
issue: An optional issue to open in repos to which pushing fails.
"""
if len(set(template_repo_urls)) != len(template_repo_urls):
raise ValueError("template_repo_urls contains duplicates")
with tempfile.TemporaryDirectory() as tmpdir:
workdir = pathlib.Path(tmpdir)
template_repos = [
plug.TemplateRepo(
name=urlutil.extract_repo_name(url),
url=url,
_path=workdir / api.extract_repo_name(url),
)
for url in template_repo_urls
]
plug.log.info("Cloning into master repos ...")
_clone_all(template_repos, cwd=workdir, api=api)
hook_results = plugin.execute_setup_tasks(
template_repos, api, cwd=pathlib.Path(tmpdir)
)
push_tuple_iter = _create_update_push_tuples(
teams, template_repos, api
)
push_tuple_iter_progress = plug.cli.io.progress_bar(
push_tuple_iter,
desc="Setting up student repos",
total=len(teams) * len(template_repos),
)
successful_pts, failed_pts = git.push(
push_tuples=push_tuple_iter_progress
)
if failed_pts and issue:
plug.echo("Opening issue in repos to which push failed")
urls_without_auth = [
re.sub("https://.*?@", "https://", pt.repo_url)
for pt in failed_pts
]
_open_issue_by_urls(urls_without_auth, issue, api)
plug.log.info("Done!")
return hook_results
def test_stops_retrying_when_failed_pushes_succeed(self, env_setup,
push_tuples, mocker):
tried = False
fail_pt = push_tuples[1]
async def raise_once(pt):
nonlocal tried
if not tried and pt == fail_pt:
tried = True
raise exception.PushFailedError("Push failed", 128,
b"some error", pt.repo_url)
expected_num_calls = len(push_tuples) + 1 # one retry
async def raise_(pt):
raise exception.PushFailedError("Push failed", 128, b"some error",
pt.repo_url)
async_push_mock = mocker.patch("_repobee.git._push_async",
side_effect=raise_once)
git.push(push_tuples, tries=10)
assert len(async_push_mock.call_args_list) == expected_num_calls
def test(self, env_setup, push_tuples, aio_subproc):
"""Test that push works as expected when no exceptions are thrown by
tasks.
"""
expected_calls = [
call(
*f"git push {url} {branch}".split(),
cwd=os.path.abspath(local_repo),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) for local_repo, url, branch in push_tuples
]
successful_pts, failed_pts = git.push(push_tuples)
assert not failed_pts
assert successful_pts == push_tuples
aio_subproc.create_subprocess.assert_has_calls(expected_calls)
def test_tries_all_calls_despite_exceptions(self, env_setup, push_tuples,
mocker):
"""Test that push tries to push all push tuple values even if there
are exceptions.
"""
tries = 3
expected_calls = [
call(pt) for pt in sorted(push_tuples, key=lambda pt: pt.repo_url)
] * tries
async def raise_(pt):
raise exception.PushFailedError("Push failed", 128, b"some error",
pt.repo_url)
mocker.patch("_repobee.git._push_async", side_effect=raise_)
successful_pts, failed_pts = git.push(push_tuples, tries=tries)
assert not successful_pts
assert failed_pts == push_tuples
git._push_async.assert_has_calls(expected_calls, any_order=True)
def test_push_raises_when_tries_is_less_than_one(self, env_setup,
push_tuples, tries):
with pytest.raises(ValueError) as exc_info:
git.push(push_tuples, tries=tries)
assert "tries must be larger than 0" in str(exc_info.value)