def _print_status(lp: LaunchpadClient): if lp.has_outstanding_build(): build_status = lp.get_build_status() for arch, status in build_status.items(): echo.info(f"Build status for arch {arch}: {status}") else: echo.info("No build found.")
def _monitor_build(lp: LaunchpadClient) -> None: target_list = humanize_list(lp.architectures, "and", "{}") echo.info( f"Building snap package for {target_list}. This may take some time to finish." ) lp.monitor_build() echo.info("Build complete.") lp.cleanup()
def setUp(self): super().setUp() self._project = self._make_snapcraft_project() self.lp = LaunchpadImpl() self.fake_login_with = fixtures.MockPatch( "launchpadlib.launchpad.Launchpad.login_with", return_value=self.lp) self.useFixture(self.fake_login_with) self.lpc = LaunchpadClient(project=self._project, build_id="id", architectures=[])
def _start_build( *, lp: LaunchpadClient, project: Project, build_id: str, package_all_sources: bool ) -> None: # Pull/update sources for project. worktree_dir = BaseDirectory.save_data_path("snapcraft", "remote-build", build_id) wt = WorkTree(worktree_dir, project, package_all_sources=package_all_sources) repo_dir = wt.prepare_repository() lp.push_source_tree(repo_dir) # Start building. lp.start_build() echo.info("If interrupted, resume with: 'snapcraft remote-build --recover'")
def setUp(self): super().setUp() self._project = self._make_snapcraft_project() self.lp = LaunchpadImpl() self.fake_login_with = fixtures.MockPatch( "launchpadlib.launchpad.Launchpad.login_with", return_value=self.lp) self.useFixture(self.fake_login_with) self.mock_git_class = mock.Mock(spec=Git) self.mock_git_class.check_command_installed.return_value = True self.lpc = LaunchpadClient( project=self._project, build_id="id", architectures=[], git_class=self.mock_git_class, running_snapcraft_version="100.1234", )
def setUp(self): super().setUp() self._project = self._make_snapcraft_project() self.lpc = LaunchpadClient( project=self._project, build_id="id", user="******", architectures=[] )
class LaunchpadTestCase(unit.TestCase): def setUp(self): super().setUp() self._project = self._make_snapcraft_project() self.lpc = LaunchpadClient( project=self._project, build_id="id", user="******", architectures=[] ) @mock.patch("launchpadlib.launchpad.Launchpad.login_with") def test_login(self, mock_login): self.lpc.login() self.assertThat(self.lpc.user, Equals("user")) mock_login.assert_called_with( "snapcraft remote-build {}".format(snapcraft.__version__), "production", mock.ANY, credentials_file=mock.ANY, version="devel", ) @mock.patch("launchpadlib.launchpad.Launchpad.login_with") def test_login_connection_issues(self, mock_login): mock_login.side_effect = ConnectionRefusedError() self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login) mock_login.side_effect = TimeoutError() self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login) @mock.patch("snapcraft.internal.remote_build.LaunchpadClient.login") def test_load_connection_issues(self, mock_login): self.lpc._lp = LaunchpadImpl() # ConnectionRefusedError should surface. self.lpc._lp._load_mock.side_effect = ConnectionRefusedError self.assertRaises(ConnectionRefusedError, self.lpc._lp_load_url, "foo") # Load URL should work OK after single connection reset. self.lpc._lp._load_mock.side_effect = [ConnectionResetError, None] self.lpc._lp_load_url(url="foo") mock_login.assert_called() # Load URL should fail with two connection resets. self.lpc._lp._load_mock.side_effect = [ ConnectionResetError, ConnectionResetError, ] self.assertRaises(ConnectionResetError, self.lpc._lp_load_url, "foo") mock_login.assert_called() def test_create_snap(self): self.lpc._lp = LaunchpadImpl() self.lpc._create_snap() self.lpc._lp.snaps.new_mock.assert_called_with( auto_build=False, auto_build_archive="/ubuntu/+archive/primary", auto_build_pocket="Updates", git_path="master", git_repository_url="https://[email protected]/~user/+git/id/", name="id", owner="/~user", ) def test_create_snap_with_archs(self): self.lpc._lp = LaunchpadImpl() self.lpc.architectures = ["arch1", "arch2"] self.lpc._create_snap() self.lpc._lp.snaps.new_mock.assert_called_with( auto_build=False, auto_build_archive="/ubuntu/+archive/primary", auto_build_pocket="Updates", git_path="master", git_repository_url="https://[email protected]/~user/+git/id/", name="id", owner="/~user", processors=["/+processors/arch1", "/+processors/arch2"], ) def test_delete_snap(self): self.lpc._lp = LaunchpadImpl() self.lpc._delete_snap() self.lpc._lp.snaps.getByName_mock.assert_called_with(name="id", owner="/~user") def test_start_build(self): self.lpc._lp = LaunchpadImpl() self.lpc.start_build() @mock.patch( "tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds", return_value=SnapBuildReqImpl( status="Failed", error_message="snapcraft.yaml not found..." ), ) def test_start_build_error(self, mock_rb): self.lpc._lp = LaunchpadImpl() raised = self.assertRaises(errors.RemoteBuilderError, self.lpc.start_build) self.assertThat(str(raised), Contains("snapcraft.yaml not found...")) @mock.patch( "tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds", return_value=SnapBuildReqImpl(status="Pending", error_message=""), ) def test_start_build_error_timeout(self, mock_rb): self.lpc._lp = LaunchpadImpl() raised = self.assertRaises( errors.RemoteBuilderNotReadyError, self.lpc.start_build, timeout=0 ) self.assertThat(str(raised), Contains("is not ready")) @mock.patch("snapcraft.internal.remote_build.LaunchpadClient._download_file") def test_monitor_build(self, mock_download_file): open("test_i386.txt", "w").close() open("test_i386.1.txt", "w").close() self.lpc._lp = LaunchpadImpl() self.lpc.start_build() self.lpc.monitor_build(interval=0) mock_download_file.assert_has_calls( [ mock.call(url="url_for/snap_file_i386.snap", dst="snap_file_i386.snap"), mock.call( url="url_for/build_log_file_1", gunzip=True, dst="test_i386.2.txt" ), mock.call( url="url_for/build_log_file_2", gunzip=True, dst="test_amd64.txt" ), ] ) @mock.patch("snapcraft.internal.remote_build.LaunchpadClient._download_file") @mock.patch( "tests.unit.remote_build.test_launchpad.BuildImpl.getFileUrls", return_value=[] ) @mock.patch("logging.Logger.error") def test_monitor_build_error(self, mock_log, mock_urls, mock_download_file): self.lpc._lp = LaunchpadImpl() self.lpc.start_build() self.lpc.monitor_build(interval=0) mock_download_file.assert_has_calls( [ mock.call( url="url_for/build_log_file_2", gunzip=True, dst="test_amd64.txt" ) ] ) mock_log.assert_called_with("Build failed for arch 'amd64'.") def test_get_build_status(self): self.lpc._lp = LaunchpadImpl() self.lpc.start_build() build_status = self.lpc.get_build_status() self.assertThat( build_status, Equals({"i386": "Successfully built", "amd64": "Failed to build"}), ) def _make_snapcraft_project(self): yaml = textwrap.dedent( """\ name: test base: core18 version: "1.0" summary: test description: test confinement: strict grade: stable """ ) snapcraft_yaml_file_path = self.make_snapcraft_yaml(yaml) project = snapcraft.project.Project( snapcraft_yaml_file_path=snapcraft_yaml_file_path ) return project def test_git_repository_creation(self): source_testdir = self.useFixture(TestDir()) source_testdir.create_file("foo") repo_dir = source_testdir.path self.assertFalse(os.path.exists(os.path.join(repo_dir, ".git"))) self.lpc._gitify_repository(repo_dir) self.assertTrue(os.path.exists(os.path.join(repo_dir, ".git"))) @mock.patch("snapcraft.internal.sources.Git.push", return_value=None) def test_push_source_tree(self, mock_push): self.lpc._lp = LaunchpadImpl() source_testdir = self.useFixture(TestDir()) source_testdir.create_file("foo") repo_dir = source_testdir.path self.lpc.push_source_tree(repo_dir) mock_push.assert_called_with( "https://*****:*****@git.launchpad.net/~user/+git/id/", "HEAD:master", force=True, ) @mock.patch( "snapcraft.internal.sources.Git.push", side_effect=SnapcraftPullError( command="git push HEAD:master https://user:access-token@url", exit_code=128 ), ) def test_push_source_tree_error(self, mock_push): self.lpc._lp = LaunchpadImpl() source_testdir = self.useFixture(TestDir()) source_testdir.create_file("foo") repo_dir = source_testdir.path self.assertRaises( errors.LaunchpadGitPushError, self.lpc.push_source_tree, repo_dir ) mock_push.assert_called_with( "https://*****:*****@git.launchpad.net/~user/+git/id/", "HEAD:master", force=True, )
def remote_build( recover: bool, status: bool, build_on: str, launchpad_accept_public_upload: bool, launchpad_timeout: int, package_all_sources: bool, echoer=echo, ) -> None: """Dispatch a snap for remote build. Command remote-build sends the current project to be built remotely. After the build is complete, packages for each architecture are retrieved and will be available in the local filesystem. If not specified in the snapcraft.yaml file, the list of architectures to build can be set using the --build-on option. If both are specified, an error will occur. Interrupted remote builds can be resumed using the --recover option, followed by the build number informed when the remote build was originally dispatched. The current state of the remote build for each architecture can be checked using the --status option. \b Examples: snapcraft remote-build snapcraft remote-build --build-on=amd64 snapcraft remote-build --build-on=amd64,arm64,armhf,i386,ppc64el,s390x snapcraft remote-build --recover snapcraft remote-build --status """ if not launchpad_accept_public_upload: raise errors.AcceptPublicUploadError() echo.warning( "snapcraft remote-build is experimental and is subject to change - use with caution." ) project = get_project() # TODO: use project.is_legacy() when available. base = project.info.get_build_base() if base is None: raise errors.BaseRequiredError() # Use a hash of current working directory to distinguish between other # potential project builds occurring in parallel elsewhere. project_hash = project._get_project_directory_hash() build_id = f"snapcraft-{project.info.name}-{project_hash}" architectures = _determine_architectures(project, build_on) # Calculate timeout timestamp, if specified. if launchpad_timeout > 0: deadline = int(time.time()) + launchpad_timeout else: deadline = 0 lp = LaunchpadClient( project=project, build_id=build_id, architectures=architectures, deadline=deadline, ) if status: _print_status(lp) return if lp.has_outstanding_build(): echo.info("Found previously started build.") _print_status(lp) # If recovery specified, monitor build and exit. if recover or echo.confirm("Do you wish to recover this build?", default=True): _monitor_build(lp) return # Otherwise clean running build before we start a new one. _clean_build(lp) _start_build( lp=lp, project=project, build_id=build_id, package_all_sources=package_all_sources, ) _monitor_build(lp)
def _clean_build(lp: LaunchpadClient): echo.info("Cleaning existing builds and artifacts...") lp.cleanup() echo.info("Done.")
class LaunchpadTestCase(unit.TestCase): def setUp(self): super().setUp() self._project = self._make_snapcraft_project() self.lp = LaunchpadImpl() self.fake_login_with = fixtures.MockPatch( "launchpadlib.launchpad.Launchpad.login_with", return_value=self.lp) self.useFixture(self.fake_login_with) self.mock_git_class = mock.Mock(spec=Git) self.mock_git_class.check_command_installed.return_value = True self.lpc = LaunchpadClient( project=self._project, build_id="id", architectures=[], git_class=self.mock_git_class, running_snapcraft_version="100.1234", ) def test_login(self): self.assertThat(self.lpc.user, Equals("user")) self.fake_login_with.mock.assert_called_with( "snapcraft remote-build {}".format(snapcraft.__version__), "production", mock.ANY, credentials_file=mock.ANY, version="devel", ) def test_login_connection_issues(self): self.fake_login_with.mock.side_effect = ConnectionRefusedError() self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login) self.fake_login_with.mock.side_effect = TimeoutError() self.assertRaises(errors.LaunchpadHttpsError, self.lpc.login) def test_load_connection_refused(self): # ConnectionRefusedError should surface. self.fake_login_with.mock.reset_mock() self.lpc._lp._load_mock.side_effect = ConnectionRefusedError self.assertRaises(ConnectionRefusedError, self.lpc._lp_load_url, "foo") self.fake_login_with.mock.assert_not_called() def test_load_connection_reset_once(self): # Load URL should work OK after single connection reset. self.fake_login_with.mock.reset_mock() self.lpc._lp._load_mock.side_effect = [ConnectionResetError, None] self.lpc._lp_load_url(url="foo") self.fake_login_with.mock.assert_called() def test_load_connection_reset_twice(self): # Load URL should fail with two connection resets. self.fake_login_with.mock.reset_mock() self.lpc._lp._load_mock.side_effect = [ ConnectionResetError, ConnectionResetError, ] self.assertRaises(ConnectionResetError, self.lpc._lp_load_url, "foo") self.fake_login_with.mock.assert_called() def test_create_snap(self): self.lpc._create_snap() self.lpc._lp.snaps.new_mock.assert_called_with( auto_build=False, auto_build_archive="/ubuntu/+archive/primary", auto_build_pocket="Updates", git_path="master", git_repository_url="https://[email protected]/~user/+git/id/", name="id", owner="/~user", ) def test_create_snap_with_archs(self): self.lpc.architectures = ["arch1", "arch2"] self.lpc._create_snap() self.lpc._lp.snaps.new_mock.assert_called_with( auto_build=False, auto_build_archive="/ubuntu/+archive/primary", auto_build_pocket="Updates", git_path="master", git_repository_url="https://[email protected]/~user/+git/id/", name="id", owner="/~user", processors=["/+processors/arch1", "/+processors/arch2"], ) def test_delete_snap(self): self.lpc._delete_snap() self.lpc._lp.snaps.getByName_mock.assert_called_with(name="id", owner="/~user") def test_start_build(self): self.lpc.start_build() @mock.patch( "tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds", return_value=SnapBuildReqImpl( status="Failed", error_message="snapcraft.yaml not found..."), ) def test_start_build_error(self, mock_rb): raised = self.assertRaises(errors.RemoteBuilderError, self.lpc.start_build) self.assertThat(str(raised), Contains("snapcraft.yaml not found...")) @mock.patch( "tests.unit.remote_build.test_launchpad.SnapImpl.requestBuilds", return_value=SnapBuildReqImpl(status="Pending", error_message=""), ) @mock.patch("time.time", return_value=500) def test_start_build_error_timeout(self, mock_time, mock_rb): self.lpc.deadline = 499 raised = self.assertRaises(errors.RemoteBuildTimeoutError, self.lpc.start_build) self.assertThat(str(raised), Equals("Remote build exceeded configured timeout.")) def test_issue_build_request_defaults(self): fake_snap = mock.MagicMock() self.lpc._issue_build_request(fake_snap) self.assertThat( fake_snap.mock_calls, Equals([ mock.call.requestBuilds( archive="main_archive", channels={ "core18": "stable", "snapcraft": "stable" }, pocket="Updates", ) ]), ) @mock.patch( "snapcraft.internal.remote_build.LaunchpadClient._download_file") def test_monitor_build(self, mock_download_file): open("test_i386.txt", "w").close() open("test_i386.1.txt", "w").close() self.lpc.start_build() self.lpc.monitor_build(interval=0) self.assertThat( mock_download_file.mock_calls, Equals([ mock.call(dst="snap_file_i386.snap", url="url_for/snap_file_i386.snap"), mock.call( dst="test_i386.2.txt", gunzip=True, url="url_for/build_log_file_1", ), mock.call(dst="snap_file_amd64.snap", url="url_for/snap_file_amd64.snap"), mock.call( dst="test_amd64.txt", gunzip=True, url="url_for/build_log_file_2", ), mock.call(dst="snap_file_amd64.snap", url="url_for/snap_file_amd64.snap"), ]), ) @mock.patch( "snapcraft.internal.remote_build.LaunchpadClient._download_file") @mock.patch("tests.unit.remote_build.test_launchpad.BuildImpl.getFileUrls", return_value=[]) @mock.patch("logging.Logger.error") def test_monitor_build_error(self, mock_log, mock_urls, mock_download_file): self.lpc.start_build() self.lpc.monitor_build(interval=0) mock_download_file.assert_has_calls([ mock.call(url="url_for/build_log_file_2", gunzip=True, dst="test_amd64.txt") ]) self.assertThat( mock_log.mock_calls, Equals([ mock.call("Snap file not available for arch 'i386'."), mock.call("Snap file not available for arch 'amd64'."), mock.call("Build failed for arch 'amd64'."), mock.call("Snap file not available for arch 'arm64'."), mock.call("Build failed for arch 'arm64'."), ]), ) @mock.patch( "snapcraft.internal.remote_build.LaunchpadClient._download_file") @mock.patch("time.time", return_value=500) def test_monitor_build_error_timeout(self, mock_time, mock_rb): self.lpc.deadline = 499 self.lpc.start_build() raised = self.assertRaises(errors.RemoteBuildTimeoutError, self.lpc.monitor_build, interval=0) self.assertThat(str(raised), Equals("Remote build exceeded configured timeout.")) def test_get_build_status(self): self.lpc.start_build() build_status = self.lpc.get_build_status() self.assertThat( build_status, Equals({ "amd64": "Failed to build", "arm64": "Failed to build", "i386": "Successfully built", }), ) def _make_snapcraft_project(self): yaml = textwrap.dedent("""\ name: test base: core18 version: "1.0" summary: test description: test confinement: strict grade: stable """) snapcraft_yaml_file_path = self.make_snapcraft_yaml(yaml) project = snapcraft.project.Project( snapcraft_yaml_file_path=snapcraft_yaml_file_path) return project def test_git_repository_creation(self): source_testdir = self.useFixture(TestDir()) source_testdir.create_file("foo") repo_dir = source_testdir.path self.lpc._gitify_repository(repo_dir) assert self.mock_git_class.mock_calls == [ mock.call.check_command_installed(), mock.call(repo_dir, repo_dir, silent=True), mock.call().init(), mock.call().add("foo"), mock.call().commit("committed by snapcraft version: 100.1234"), ] def test_push_source_tree(self): source_testdir = self.useFixture(TestDir()) repo_dir = source_testdir.path self.lpc.push_source_tree(repo_dir) self.mock_git_class.assert_has_calls == [ mock.call( "https://*****:*****@git.launchpad.net/~user/+git/id/", "HEAD:master", force=True, ) ] def test_push_source_tree_error(self): self.mock_git_class.return_value.push.side_effect = ( SnapcraftPullError( command="git push HEAD:master https://user:access-token@url", exit_code=128, ), ) source_testdir = self.useFixture(TestDir()) repo_dir = source_testdir.path self.assertRaises(errors.LaunchpadGitPushError, self.lpc.push_source_tree, repo_dir)
def remote_build( recover: bool, status: bool, build_on: str, build_id: str, launchpad_accept_public_upload: bool, launchpad_timeout: int, package_all_sources: bool, echoer=echo, ) -> None: """Dispatch a snap for remote build. Command remote-build sends the current project to be built remotely. After the build is complete, packages for each architecture are retrieved and will be available in the local filesystem. If not specified in the snapcraft.yaml file, the list of architectures to build can be set using the --build-on option. If both are specified, an error will occur. Interrupted remote builds can be resumed using the --recover option, followed by the build number informed when the remote build was originally dispatched. The current state of the remote build for each architecture can be checked using the --status option. \b Examples: snapcraft remote-build snapcraft remote-build --build-on=amd64 snapcraft remote-build --build-on=amd64,arm64,armhf,i386,ppc64el,s390x snapcraft remote-build --recover snapcraft remote-build --recover --build-id snapcraft-my-snap-b98a6bd3 snapcraft remote-build --status snapcraft remote-build --status --build-id snapcraft-my-snap-b98a6bd3 """ if os.getenv("SUDO_USER") and os.geteuid() == 0: echo.warning( "Running with 'sudo' may cause permission errors and is discouraged." ) echo.warning( "snapcraft remote-build is experimental and is subject to change - use with caution." ) project = get_project() try: project._get_build_base() except RuntimeError: raise errors.BaseRequiredError() if not build_id: # If the option wasn't provided, use the project directory hash # to create one unique to the status of the working tree, allowing # us to distinguish between multiple builds for this project # (and others). project_hash = project._get_project_directory_hash() build_id = f"snapcraft-{project.info.name}-{project_hash}" echo.info(f"Using build ID {build_id}") architectures = _determine_architectures(project, build_on) # Calculate timeout timestamp, if specified. if launchpad_timeout > 0: deadline = int(time.time()) + launchpad_timeout else: deadline = 0 lp = LaunchpadClient( project=project, build_id=build_id, architectures=architectures, deadline=deadline, ) if status: _print_status(lp) return has_outstanding_build = lp.has_outstanding_build() if recover and not has_outstanding_build: echo.info("No build found.") return elif has_outstanding_build: echo.info("Found previously started build.") _print_status(lp) # If recovery specified, monitor build and exit. if recover or echo.confirm("Do you wish to recover this build?", default=True): _monitor_build(lp) return # Otherwise clean running build before we start a new one. _clean_build(lp) _check_launchpad_acceptance(launchpad_accept_public_upload) _start_build( lp=lp, project=project, build_id=build_id, package_all_sources=package_all_sources, ) _monitor_build(lp)