def _print_status(lp: LaunchpadClient):
if lp.has_outstanding_build():
build_status = lp.get_build_status()
for arch, status in build_status.items():
echo.info(f"Build status for arch {arch}: {status}")
else:
echo.info("No build found.")
def _monitor_build(lp: LaunchpadClient) -> None:
target_list = humanize_list(lp.architectures, "and", "{}")
echo.info(
f"Building snap package for {target_list}. This may take some time to finish."
)
lp.monitor_build()
echo.info("Build complete.")
lp.cleanup()
def setUp(self):
super().setUp()
self._project = self._make_snapcraft_project()
self.lp = LaunchpadImpl()
self.fake_login_with = fixtures.MockPatch(
"launchpadlib.launchpad.Launchpad.login_with",
return_value=self.lp)
self.useFixture(self.fake_login_with)
self.lpc = LaunchpadClient(project=self._project,
build_id="id",
architectures=[])
def _start_build(
*, lp: LaunchpadClient, project: Project, build_id: str, package_all_sources: bool
) -> None:
# Pull/update sources for project.
worktree_dir = BaseDirectory.save_data_path("snapcraft", "remote-build", build_id)
wt = WorkTree(worktree_dir, project, package_all_sources=package_all_sources)
repo_dir = wt.prepare_repository()
lp.push_source_tree(repo_dir)
# Start building.
lp.start_build()
echo.info("If interrupted, resume with: 'snapcraft remote-build --recover'")
def setUp(self):
super().setUp()
self._project = self._make_snapcraft_project()
self.lp = LaunchpadImpl()
self.fake_login_with = fixtures.MockPatch(
"launchpadlib.launchpad.Launchpad.login_with",
return_value=self.lp)
self.useFixture(self.fake_login_with)
self.mock_git_class = mock.Mock(spec=Git)
self.mock_git_class.check_command_installed.return_value = True
self.lpc = LaunchpadClient(
project=self._project,
build_id="id",
architectures=[],
git_class=self.mock_git_class,
running_snapcraft_version="100.1234",
)
def setUp(self):
super().setUp()
self._project = self._make_snapcraft_project()
self.lpc = LaunchpadClient(
project=self._project, build_id="id", user="******", architectures=[]
)
class LaunchpadTestCase(unit.TestCase):
def setUp(self):
super().setUp()
self._project = self._make_snapcraft_project()
self.lpc = LaunchpadClient(
project=self._project, build_id="id", user="******", architectures=[]
)
@mock.patch("launchpadlib.launchpad.Launchpad.login_with")
def test_login(self, mock_login):
self.lpc.login()
self.assertThat(self.lpc.user, Equals("user"))
mock_login.assert_called_with(
"snapcraft remote-build {}".format(snapcraft.__version__),
"production",
mock.ANY,
credentials_file=mock.ANY,
version="devel",
)
@mock.patch("launchpadlib.launchpad.Launchpad.login_with")
def test_login_connection_issues(self, mock_login):
mock_login.side_effect = ConnectionRefusedError()
self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login)
mock_login.side_effect = TimeoutError()
self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login)
@mock.patch("snapcraft.internal.remote_build.LaunchpadClient.login")
def test_load_connection_issues(self, mock_login):
self.lpc._lp = LaunchpadImpl()
# ConnectionRefusedError should surface.
self.lpc._lp._load_mock.side_effect = ConnectionRefusedError
self.assertRaises(ConnectionRefusedError, self.lpc._lp_load_url, "foo")
# Load URL should work OK after single connection reset.
self.lpc._lp._load_mock.side_effect = [ConnectionResetError, None]
self.lpc._lp_load_url(url="foo")
mock_login.assert_called()
# Load URL should fail with two connection resets.
self.lpc._lp._load_mock.side_effect = [
ConnectionResetError,
ConnectionResetError,
]
self.assertRaises(ConnectionResetError, self.lpc._lp_load_url, "foo")
mock_login.assert_called()
def test_create_snap(self):
self.lpc._lp = LaunchpadImpl()
self.lpc._create_snap()
self.lpc._lp.snaps.new_mock.assert_called_with(
auto_build=False,
auto_build_archive="/ubuntu/+archive/primary",
auto_build_pocket="Updates",
git_path="master",
git_repository_url="https://[email protected]/~user/+git/id/",
name="id",
owner="/~user",
)
def test_create_snap_with_archs(self):
self.lpc._lp = LaunchpadImpl()
self.lpc.architectures = ["arch1", "arch2"]
self.lpc._create_snap()
self.lpc._lp.snaps.new_mock.assert_called_with(
auto_build=False,
auto_build_archive="/ubuntu/+archive/primary",
auto_build_pocket="Updates",
git_path="master",
git_repository_url="https://[email protected]/~user/+git/id/",
name="id",
owner="/~user",
processors=["/+processors/arch1", "/+processors/arch2"],
)
def test_delete_snap(self):
self.lpc._lp = LaunchpadImpl()
self.lpc._delete_snap()
self.lpc._lp.snaps.getByName_mock.assert_called_with(name="id", owner="/~user")
def test_start_build(self):
self.lpc._lp = LaunchpadImpl()
self.lpc.start_build()
@mock.patch(
"tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds",
return_value=SnapBuildReqImpl(
status="Failed", error_message="snapcraft.yaml not found..."
),
)
def test_start_build_error(self, mock_rb):
self.lpc._lp = LaunchpadImpl()
raised = self.assertRaises(errors.RemoteBuilderError, self.lpc.start_build)
self.assertThat(str(raised), Contains("snapcraft.yaml not found..."))
@mock.patch(
"tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds",
return_value=SnapBuildReqImpl(status="Pending", error_message=""),
)
def test_start_build_error_timeout(self, mock_rb):
self.lpc._lp = LaunchpadImpl()
raised = self.assertRaises(
errors.RemoteBuilderNotReadyError, self.lpc.start_build, timeout=0
)
self.assertThat(str(raised), Contains("is not ready"))
@mock.patch("snapcraft.internal.remote_build.LaunchpadClient._download_file")
def test_monitor_build(self, mock_download_file):
open("test_i386.txt", "w").close()
open("test_i386.1.txt", "w").close()
self.lpc._lp = LaunchpadImpl()
self.lpc.start_build()
self.lpc.monitor_build(interval=0)
mock_download_file.assert_has_calls(
[
mock.call(url="url_for/snap_file_i386.snap", dst="snap_file_i386.snap"),
mock.call(
url="url_for/build_log_file_1", gunzip=True, dst="test_i386.2.txt"
),
mock.call(
url="url_for/build_log_file_2", gunzip=True, dst="test_amd64.txt"
),
]
)
@mock.patch("snapcraft.internal.remote_build.LaunchpadClient._download_file")
@mock.patch(
"tests.unit.remote_build.test_launchpad.BuildImpl.getFileUrls", return_value=[]
)
@mock.patch("logging.Logger.error")
def test_monitor_build_error(self, mock_log, mock_urls, mock_download_file):
self.lpc._lp = LaunchpadImpl()
self.lpc.start_build()
self.lpc.monitor_build(interval=0)
mock_download_file.assert_has_calls(
[
mock.call(
url="url_for/build_log_file_2", gunzip=True, dst="test_amd64.txt"
)
]
)
mock_log.assert_called_with("Build failed for arch 'amd64'.")
def test_get_build_status(self):
self.lpc._lp = LaunchpadImpl()
self.lpc.start_build()
build_status = self.lpc.get_build_status()
self.assertThat(
build_status,
Equals({"i386": "Successfully built", "amd64": "Failed to build"}),
)
def _make_snapcraft_project(self):
yaml = textwrap.dedent(
"""\
name: test
base: core18
version: "1.0"
summary: test
description: test
confinement: strict
grade: stable
"""
)
snapcraft_yaml_file_path = self.make_snapcraft_yaml(yaml)
project = snapcraft.project.Project(
snapcraft_yaml_file_path=snapcraft_yaml_file_path
)
return project
def test_git_repository_creation(self):
source_testdir = self.useFixture(TestDir())
source_testdir.create_file("foo")
repo_dir = source_testdir.path
self.assertFalse(os.path.exists(os.path.join(repo_dir, ".git")))
self.lpc._gitify_repository(repo_dir)
self.assertTrue(os.path.exists(os.path.join(repo_dir, ".git")))
@mock.patch("snapcraft.internal.sources.Git.push", return_value=None)
def test_push_source_tree(self, mock_push):
self.lpc._lp = LaunchpadImpl()
source_testdir = self.useFixture(TestDir())
source_testdir.create_file("foo")
repo_dir = source_testdir.path
self.lpc.push_source_tree(repo_dir)
mock_push.assert_called_with(
"https://*****:*****@git.launchpad.net/~user/+git/id/",
"HEAD:master",
force=True,
)
@mock.patch(
"snapcraft.internal.sources.Git.push",
side_effect=SnapcraftPullError(
command="git push HEAD:master https://user:access-token@url", exit_code=128
),
)
def test_push_source_tree_error(self, mock_push):
self.lpc._lp = LaunchpadImpl()
source_testdir = self.useFixture(TestDir())
source_testdir.create_file("foo")
repo_dir = source_testdir.path
self.assertRaises(
errors.LaunchpadGitPushError, self.lpc.push_source_tree, repo_dir
)
mock_push.assert_called_with(
"https://*****:*****@git.launchpad.net/~user/+git/id/",
"HEAD:master",
force=True,
)
def remote_build(
recover: bool,
status: bool,
build_on: str,
launchpad_accept_public_upload: bool,
launchpad_timeout: int,
package_all_sources: bool,
echoer=echo,
) -> None:
"""Dispatch a snap for remote build.
Command remote-build sends the current project to be built remotely. After the build
is complete, packages for each architecture are retrieved and will be available in
the local filesystem.
If not specified in the snapcraft.yaml file, the list of architectures to build
can be set using the --build-on option. If both are specified, an error will occur.
Interrupted remote builds can be resumed using the --recover option, followed by
the build number informed when the remote build was originally dispatched. The
current state of the remote build for each architecture can be checked using the
--status option.
\b
Examples:
snapcraft remote-build
snapcraft remote-build --build-on=amd64
snapcraft remote-build --build-on=amd64,arm64,armhf,i386,ppc64el,s390x
snapcraft remote-build --recover
snapcraft remote-build --status
"""
if not launchpad_accept_public_upload:
raise errors.AcceptPublicUploadError()
echo.warning(
"snapcraft remote-build is experimental and is subject to change - use with caution."
)
project = get_project()
# TODO: use project.is_legacy() when available.
base = project.info.get_build_base()
if base is None:
raise errors.BaseRequiredError()
# Use a hash of current working directory to distinguish between other
# potential project builds occurring in parallel elsewhere.
project_hash = project._get_project_directory_hash()
build_id = f"snapcraft-{project.info.name}-{project_hash}"
architectures = _determine_architectures(project, build_on)
# Calculate timeout timestamp, if specified.
if launchpad_timeout > 0:
deadline = int(time.time()) + launchpad_timeout
else:
deadline = 0
lp = LaunchpadClient(
project=project,
build_id=build_id,
architectures=architectures,
deadline=deadline,
)
if status:
_print_status(lp)
return
if lp.has_outstanding_build():
echo.info("Found previously started build.")
_print_status(lp)
# If recovery specified, monitor build and exit.
if recover or echo.confirm("Do you wish to recover this build?",
default=True):
_monitor_build(lp)
return
# Otherwise clean running build before we start a new one.
_clean_build(lp)
_start_build(
lp=lp,
project=project,
build_id=build_id,
package_all_sources=package_all_sources,
)
_monitor_build(lp)
def _clean_build(lp: LaunchpadClient):
echo.info("Cleaning existing builds and artifacts...")
lp.cleanup()
echo.info("Done.")
class LaunchpadTestCase(unit.TestCase):
def setUp(self):
super().setUp()
self._project = self._make_snapcraft_project()
self.lp = LaunchpadImpl()
self.fake_login_with = fixtures.MockPatch(
"launchpadlib.launchpad.Launchpad.login_with",
return_value=self.lp)
self.useFixture(self.fake_login_with)
self.mock_git_class = mock.Mock(spec=Git)
self.mock_git_class.check_command_installed.return_value = True
self.lpc = LaunchpadClient(
project=self._project,
build_id="id",
architectures=[],
git_class=self.mock_git_class,
running_snapcraft_version="100.1234",
)
def test_login(self):
self.assertThat(self.lpc.user, Equals("user"))
self.fake_login_with.mock.assert_called_with(
"snapcraft remote-build {}".format(snapcraft.__version__),
"production",
mock.ANY,
credentials_file=mock.ANY,
version="devel",
)
def test_login_connection_issues(self):
self.fake_login_with.mock.side_effect = ConnectionRefusedError()
self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login)
self.fake_login_with.mock.side_effect = TimeoutError()
self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login)
def test_load_connection_refused(self):
# ConnectionRefusedError should surface.
self.fake_login_with.mock.reset_mock()
self.lpc._lp._load_mock.side_effect = ConnectionRefusedError
self.assertRaises(ConnectionRefusedError, self.lpc._lp_load_url, "foo")
self.fake_login_with.mock.assert_not_called()
def test_load_connection_reset_once(self):
# Load URL should work OK after single connection reset.
self.fake_login_with.mock.reset_mock()
self.lpc._lp._load_mock.side_effect = [ConnectionResetError, None]
self.lpc._lp_load_url(url="foo")
self.fake_login_with.mock.assert_called()
def test_load_connection_reset_twice(self):
# Load URL should fail with two connection resets.
self.fake_login_with.mock.reset_mock()
self.lpc._lp._load_mock.side_effect = [
ConnectionResetError,
ConnectionResetError,
]
self.assertRaises(ConnectionResetError, self.lpc._lp_load_url, "foo")
self.fake_login_with.mock.assert_called()
def test_create_snap(self):
self.lpc._create_snap()
self.lpc._lp.snaps.new_mock.assert_called_with(
auto_build=False,
auto_build_archive="/ubuntu/+archive/primary",
auto_build_pocket="Updates",
git_path="master",
git_repository_url="https://[email protected]/~user/+git/id/",
name="id",
owner="/~user",
)
def test_create_snap_with_archs(self):
self.lpc.architectures = ["arch1", "arch2"]
self.lpc._create_snap()
self.lpc._lp.snaps.new_mock.assert_called_with(
auto_build=False,
auto_build_archive="/ubuntu/+archive/primary",
auto_build_pocket="Updates",
git_path="master",
git_repository_url="https://[email protected]/~user/+git/id/",
name="id",
owner="/~user",
processors=["/+processors/arch1", "/+processors/arch2"],
)
def test_delete_snap(self):
self.lpc._delete_snap()
self.lpc._lp.snaps.getByName_mock.assert_called_with(name="id",
owner="/~user")
def test_start_build(self):
self.lpc.start_build()
@mock.patch(
"tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds",
return_value=SnapBuildReqImpl(
status="Failed", error_message="snapcraft.yaml not found..."),
)
def test_start_build_error(self, mock_rb):
raised = self.assertRaises(errors.RemoteBuilderError,
self.lpc.start_build)
self.assertThat(str(raised), Contains("snapcraft.yaml not found..."))
@mock.patch(
"tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds",
return_value=SnapBuildReqImpl(status="Pending", error_message=""),
)
@mock.patch("time.time", return_value=500)
def test_start_build_error_timeout(self, mock_time, mock_rb):
self.lpc.deadline = 499
raised = self.assertRaises(errors.RemoteBuildTimeoutError,
self.lpc.start_build)
self.assertThat(str(raised),
Equals("Remote build exceeded configured timeout."))
def test_issue_build_request_defaults(self):
fake_snap = mock.MagicMock()
self.lpc._issue_build_request(fake_snap)
self.assertThat(
fake_snap.mock_calls,
Equals([
mock.call.requestBuilds(
archive="main_archive",
channels={
"core18": "stable",
"snapcraft": "stable"
},
pocket="Updates",
)
]),
)
@mock.patch(
"snapcraft.internal.remote_build.LaunchpadClient._download_file")
def test_monitor_build(self, mock_download_file):
open("test_i386.txt", "w").close()
open("test_i386.1.txt", "w").close()
self.lpc.start_build()
self.lpc.monitor_build(interval=0)
self.assertThat(
mock_download_file.mock_calls,
Equals([
mock.call(dst="snap_file_i386.snap",
url="url_for/snap_file_i386.snap"),
mock.call(
dst="test_i386.2.txt",
gunzip=True,
url="url_for/build_log_file_1",
),
mock.call(dst="snap_file_amd64.snap",
url="url_for/snap_file_amd64.snap"),
mock.call(
dst="test_amd64.txt",
gunzip=True,
url="url_for/build_log_file_2",
),
mock.call(dst="snap_file_amd64.snap",
url="url_for/snap_file_amd64.snap"),
]),
)
@mock.patch(
"snapcraft.internal.remote_build.LaunchpadClient._download_file")
@mock.patch("tests.unit.remote_build.test_launchpad.BuildImpl.getFileUrls",
return_value=[])
@mock.patch("logging.Logger.error")
def test_monitor_build_error(self, mock_log, mock_urls,
mock_download_file):
self.lpc.start_build()
self.lpc.monitor_build(interval=0)
mock_download_file.assert_has_calls([
mock.call(url="url_for/build_log_file_2",
gunzip=True,
dst="test_amd64.txt")
])
self.assertThat(
mock_log.mock_calls,
Equals([
mock.call("Snap file not available for arch 'i386'."),
mock.call("Snap file not available for arch 'amd64'."),
mock.call("Build failed for arch 'amd64'."),
mock.call("Snap file not available for arch 'arm64'."),
mock.call("Build failed for arch 'arm64'."),
]),
)
@mock.patch(
"snapcraft.internal.remote_build.LaunchpadClient._download_file")
@mock.patch("time.time", return_value=500)
def test_monitor_build_error_timeout(self, mock_time, mock_rb):
self.lpc.deadline = 499
self.lpc.start_build()
raised = self.assertRaises(errors.RemoteBuildTimeoutError,
self.lpc.monitor_build,
interval=0)
self.assertThat(str(raised),
Equals("Remote build exceeded configured timeout."))
def test_get_build_status(self):
self.lpc.start_build()
build_status = self.lpc.get_build_status()
self.assertThat(
build_status,
Equals({
"amd64": "Failed to build",
"arm64": "Failed to build",
"i386": "Successfully built",
}),
)
def _make_snapcraft_project(self):
yaml = textwrap.dedent("""\
name: test
base: core18
version: "1.0"
summary: test
description: test
confinement: strict
grade: stable
""")
snapcraft_yaml_file_path = self.make_snapcraft_yaml(yaml)
project = snapcraft.project.Project(
snapcraft_yaml_file_path=snapcraft_yaml_file_path)
return project
def test_git_repository_creation(self):
source_testdir = self.useFixture(TestDir())
source_testdir.create_file("foo")
repo_dir = source_testdir.path
self.lpc._gitify_repository(repo_dir)
assert self.mock_git_class.mock_calls == [
mock.call.check_command_installed(),
mock.call(repo_dir, repo_dir, silent=True),
mock.call().init(),
mock.call().add("foo"),
mock.call().commit("committed by snapcraft version: 100.1234"),
]
def test_push_source_tree(self):
source_testdir = self.useFixture(TestDir())
repo_dir = source_testdir.path
self.lpc.push_source_tree(repo_dir)
self.mock_git_class.assert_has_calls == [
mock.call(
"https://*****:*****@git.launchpad.net/~user/+git/id/",
"HEAD:master",
force=True,
)
]
def test_push_source_tree_error(self):
self.mock_git_class.return_value.push.side_effect = (
SnapcraftPullError(
command="git push HEAD:master https://user:access-token@url",
exit_code=128,
), )
source_testdir = self.useFixture(TestDir())
repo_dir = source_testdir.path
self.assertRaises(errors.LaunchpadGitPushError,
self.lpc.push_source_tree, repo_dir)
def remote_build(
recover: bool,
status: bool,
build_on: str,
build_id: str,
launchpad_accept_public_upload: bool,
launchpad_timeout: int,
package_all_sources: bool,
echoer=echo,
) -> None:
"""Dispatch a snap for remote build.
Command remote-build sends the current project to be built remotely. After the build
is complete, packages for each architecture are retrieved and will be available in
the local filesystem.
If not specified in the snapcraft.yaml file, the list of architectures to build
can be set using the --build-on option. If both are specified, an error will occur.
Interrupted remote builds can be resumed using the --recover option, followed by
the build number informed when the remote build was originally dispatched. The
current state of the remote build for each architecture can be checked using the
--status option.
\b
Examples:
snapcraft remote-build
snapcraft remote-build --build-on=amd64
snapcraft remote-build --build-on=amd64,arm64,armhf,i386,ppc64el,s390x
snapcraft remote-build --recover
snapcraft remote-build --recover --build-id snapcraft-my-snap-b98a6bd3
snapcraft remote-build --status
snapcraft remote-build --status --build-id snapcraft-my-snap-b98a6bd3
"""
if os.getenv("SUDO_USER") and os.geteuid() == 0:
echo.warning(
"Running with 'sudo' may cause permission errors and is discouraged."
)
echo.warning(
"snapcraft remote-build is experimental and is subject to change - use with caution."
)
project = get_project()
try:
project._get_build_base()
except RuntimeError:
raise errors.BaseRequiredError()
if not build_id:
# If the option wasn't provided, use the project directory hash
# to create one unique to the status of the working tree, allowing
# us to distinguish between multiple builds for this project
# (and others).
project_hash = project._get_project_directory_hash()
build_id = f"snapcraft-{project.info.name}-{project_hash}"
echo.info(f"Using build ID {build_id}")
architectures = _determine_architectures(project, build_on)
# Calculate timeout timestamp, if specified.
if launchpad_timeout > 0:
deadline = int(time.time()) + launchpad_timeout
else:
deadline = 0
lp = LaunchpadClient(
project=project,
build_id=build_id,
architectures=architectures,
deadline=deadline,
)
if status:
_print_status(lp)
return
has_outstanding_build = lp.has_outstanding_build()
if recover and not has_outstanding_build:
echo.info("No build found.")
return
elif has_outstanding_build:
echo.info("Found previously started build.")
_print_status(lp)
# If recovery specified, monitor build and exit.
if recover or echo.confirm("Do you wish to recover this build?",
default=True):
_monitor_build(lp)
return
# Otherwise clean running build before we start a new one.
_clean_build(lp)
_check_launchpad_acceptance(launchpad_accept_public_upload)
_start_build(
lp=lp,
project=project,
build_id=build_id,
package_all_sources=package_all_sources,
)
_monitor_build(lp)