def _upload_delta( store_client, *, snap_name: str, snap_filename: str, source_snap: str, built_at: str, channels: Optional[List[str]] = None, ) -> Dict[str, Any]: delta_format = "xdelta3" logger.debug("Found cached source snap {}.".format(source_snap)) target_snap = os.path.join(os.getcwd(), snap_filename) try: xdelta_generator = deltas.XDelta3Generator(source_path=source_snap, target_path=target_snap) delta_filename = xdelta_generator.make_delta() except (DeltaGenerationError, DeltaGenerationTooBigError, ToolMissingError) as e: raise storeapi.errors.StoreDeltaApplicationError(str(e)) snap_hashes = { "source_hash": calculate_sha3_384(source_snap), "target_hash": calculate_sha3_384(target_snap), "delta_hash": calculate_sha3_384(delta_filename), } try: logger.debug("Uploading delta {!r}.".format(delta_filename)) delta_tracker = store_client.upload( snap_name=snap_name, snap_filename=delta_filename, built_at=built_at, channels=channels, delta_format=delta_format, source_hash=snap_hashes["source_hash"], target_hash=snap_hashes["target_hash"], delta_hash=snap_hashes["delta_hash"], ) result = delta_tracker.track() delta_tracker.raise_for_code() except storeapi.errors.StoreReviewError as e: if e.code == "processing_upload_delta_error": raise storeapi.errors.StoreDeltaApplicationError(str(e)) else: raise except storeapi.errors.StoreServerError as e: raise storeapi.errors.StoreUploadError(snap_name, e.response) finally: if os.path.isfile(delta_filename): try: os.remove(delta_filename) except OSError: logger.warning( "Unable to remove delta {}.".format(delta_filename)) return result
def _push_delta(snap_name, snap_filename, source_snap, built_at): store = storeapi.StoreClient() delta_format = "xdelta3" logger.debug("Found cached source snap {}.".format(source_snap)) target_snap = os.path.join(os.getcwd(), snap_filename) try: xdelta_generator = deltas.XDelta3Generator(source_path=source_snap, target_path=target_snap) delta_filename = xdelta_generator.make_delta() except (DeltaGenerationError, DeltaGenerationTooBigError, DeltaToolError) as e: raise storeapi.errors.StoreDeltaApplicationError(str(e)) snap_hashes = { "source_hash": calculate_sha3_384(source_snap), "target_hash": calculate_sha3_384(target_snap), "delta_hash": calculate_sha3_384(delta_filename), } try: logger.debug("Pushing delta {!r}.".format(delta_filename)) with _requires_login(): delta_tracker = store.upload( snap_name, delta_filename, delta_format=delta_format, source_hash=snap_hashes["source_hash"], target_hash=snap_hashes["target_hash"], delta_hash=snap_hashes["delta_hash"], built_at=built_at, ) result = delta_tracker.track() delta_tracker.raise_for_code() except storeapi.errors.StoreReviewError as e: if e.code == "processing_upload_delta_error": raise storeapi.errors.StoreDeltaApplicationError(str(e)) else: raise except storeapi.errors.StoreServerError as e: raise storeapi.errors.StorePushError(snap_name, e.response) finally: if os.path.isfile(delta_filename): try: os.remove(delta_filename) except OSError: logger.warning( "Unable to remove delta {}.".format(delta_filename)) return result
def test_push_revision_prune_snap_cache(self): snap_revision = 9 patcher = mock.patch('snapcraft.storeapi.StoreClient.push_precheck') patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch.object(storeapi.StoreClient, 'get_snap_revisions') mock_release = patcher.start() self.addCleanup(patcher.stop) mock_release.return_value = [snap_revision] mock_tracker = mock.Mock(storeapi.StatusTracker) mock_tracker.track.return_value = { 'code': 'ready_to_release', 'processed': True, 'can_release': True, 'url': '/fake/url', 'revision': snap_revision, } patcher = mock.patch.object(storeapi.StoreClient, 'upload') mock_upload = patcher.start() self.addCleanup(patcher.stop) mock_upload.return_value = mock_tracker deb_arch = 'amd64' snap_cache = os.path.join( BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'basic', 'snap_hashes', deb_arch ) os.makedirs(snap_cache) for cached_snap in self.cached_snaps: cached_snap = cached_snap.format(deb_arch) open(os.path.join(snap_cache, cached_snap), 'a').close() # Upload with mock.patch('snapcraft.storeapi.StatusTracker'): result = self.run_command(['push', self.snap_file]) self.assertThat(result.exit_code, Equals(0)) real_cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file) ) self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists()) for snap in self.cached_snaps: snap = snap.format(deb_arch) self.assertThat(os.path.join(snap_cache, snap), Not(FileExists())) self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_push_revision_cached_with_experimental_deltas(self): self.useFixture(fixture_setup.FakeTerminal()) if self.enable_deltas: self.useFixture(fixture_setup.DeltaUploads()) # Create a snap main(['init']) main(['snap']) snap_file = glob.glob('*.snap')[0] # Upload with mock.patch('snapcraft.storeapi.StatusTracker'): main(['push', snap_file]) snap_cache = os.path.join( BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'my-snap-name', 'snap_hashes', self.deb_arch, ) cached_snap = os.path.join(snap_cache, file_utils.calculate_sha3_384(snap_file)) if self.enable_deltas: self.assertThat(cached_snap, FileExists()) else: self.assertThat(cached_snap, Not(FileExists()))
def _download_multipass(dl_dir: str, echoer) -> str: """Creates temporary Downloads installer to temp directory.""" dl_url = _fetch_installer_url() dl_basename = os.path.basename(dl_url) dl_path = os.path.join(dl_dir, dl_basename) echoer.info("Downloading Multipass installer...\n{} -> {}".format( dl_url, dl_path)) try: request = requests.get(dl_url, stream=True, allow_redirects=True) request.raise_for_status() download_requests_stream(request, dl_path) except requests.RequestException as e: raise ProviderMultipassDownloadFailed(_requests_exception_hint(e)) digest = calculate_sha3_384(dl_path) if digest != _MULTIPASS_DL_SHA3_384: raise ProviderMultipassDownloadFailed( "download failed verification (expected={} but found={})".format( _MULTIPASS_DL_SHA3_384, digest)) echoer.info("Verified installer successfully...") return dl_path
def test_core_setup_if_docker_env(self, dockerenv_fake, download_mock): dockerenv_file = os.path.join(self.tempdir, "dockerenv") os.makedirs(self.tempdir) open(dockerenv_file, "w").close() dockerenv_fake.return_value = dockerenv_file project_config = self.make_snapcraft_project(confinement="classic") core_snap = self.create_core_snap(project_config.project.deb_arch) core_snap_hash = calculate_sha3_384(core_snap) download_mock.return_value = core_snap_hash self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap) lifecycle.execute(steps.PULL, project_config) regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format( os.path.dirname(self.core_path), self.core_path, core_snap_hash ) self.assertThat( self.witness_path, FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)), ) download_mock.assert_called_once_with( "core", "stable", os.path.join(self.tempdir, "core.snap"), project_config.project.deb_arch, "", )
def test_core_setup_with_env_var(self, download_mock): self.useFixture(fixtures.EnvironmentVariable("SNAPCRAFT_SETUP_CORE", "1")) project_config = self.make_snapcraft_project(confinement="classic") core_snap = self.create_core_snap(project_config.project.deb_arch) core_snap_hash = calculate_sha3_384(core_snap) download_mock.return_value = core_snap_hash self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap) lifecycle.execute(steps.PULL, project_config) regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format( os.path.dirname(self.core_path), self.core_path, core_snap_hash ) self.assertThat( self.witness_path, FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)), ) download_mock.assert_called_once_with( "core", "stable", os.path.join(self.tempdir, "core.snap"), project_config.project.deb_arch, "", )
def test_core_setup_with_env_var(self, download_mock): self.useFixture(fixtures.EnvironmentVariable("SNAPCRAFT_SETUP_CORE", "1")) project_config = self.make_snapcraft_project(confinement="classic") core_snap = self.create_core_snap(project_config.project.deb_arch) core_snap_hash = calculate_sha3_384(core_snap) download_mock.return_value = core_snap_hash self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap) lifecycle.execute(steps.PULL, project_config) regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format( os.path.dirname(self.core_path), self.core_path, core_snap_hash ) self.assertThat( self.witness_path, FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)), ) download_mock.assert_called_once_with( "core", "stable", os.path.join(self.tempdir, "core.snap"), project_config.project.deb_arch, "", )
def test_core_setup_if_docker_env(self, dockerenv_fake, download_mock): dockerenv_file = os.path.join(self.tempdir, "dockerenv") os.makedirs(self.tempdir) open(dockerenv_file, "w").close() dockerenv_fake.return_value = dockerenv_file project_config = self.make_snapcraft_project(confinement="classic") core_snap = self.create_core_snap(project_config.project.deb_arch) core_snap_hash = calculate_sha3_384(core_snap) download_mock.return_value = core_snap_hash self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap) lifecycle.execute(steps.PULL, project_config) regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format( os.path.dirname(self.core_path), self.core_path, core_snap_hash ) self.assertThat( self.witness_path, FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)), ) download_mock.assert_called_once_with( "core", "stable", os.path.join(self.tempdir, "core.snap"), project_config.project.deb_arch, "", )
def test_push_revision_prune_snap_cache(self): self.useFixture(fixture_setup.FakeTerminal()) self.useFixture(fixture_setup.DeltaUploads()) snap_revision = 9 patcher = mock.patch('snapcraft.storeapi.StoreClient.push_precheck') patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch.object(storeapi.StoreClient, 'get_snap_history') mock_release = patcher.start() self.addCleanup(patcher.stop) mock_release.return_value = [snap_revision] mock_tracker = mock.Mock(storeapi.StatusTracker) mock_tracker.track.return_value = { 'code': 'ready_to_release', 'processed': True, 'can_release': True, 'url': '/fake/url', 'revision': snap_revision, } patcher = mock.patch.object(storeapi.StoreClient, 'upload') mock_upload = patcher.start() self.addCleanup(patcher.stop) mock_upload.return_value = mock_tracker deb_arch = snapcraft.ProjectOptions().deb_arch snap_cache = os.path.join(BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'my-snap-name', 'snap_hashes', deb_arch) os.makedirs(snap_cache) for cached_snap in self.cached_snaps: cached_snap = cached_snap.format(deb_arch) open(os.path.join(snap_cache, cached_snap), 'a').close() # Create a snap main(['init']) main(['snap']) snap_file = glob.glob('*.snap')[0] # Upload with mock.patch('snapcraft.storeapi.StatusTracker'): main(['push', snap_file]) real_cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(snap_file)) self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists()) for snap in self.cached_snaps: snap = snap.format(deb_arch) self.assertThat(os.path.join(snap_cache, snap), Not(FileExists())) self.assertEqual(1, len(os.listdir(snap_cache)))
def push(snap_filename, release_channels=None): """Push a snap_filename to the store. If a cached snap is available, a delta will be generated from the cached snap to the new target snap and uploaded instead. In the case of a delta processing or upload failure, push will fall back to uploading the full snap. If release_channels is defined it also releases it to those channels if the store deems the uploaded snap as ready to release. """ if not os.path.exists(snap_filename): raise FileNotFoundError( 'The file {!r} does not exist.'.format(snap_filename)) snap_yaml = _get_data_from_snap_file(snap_filename) snap_name = snap_yaml['name'] store = storeapi.StoreClient() logger.info('Preparing to push {!r} to the store.'.format(snap_filename)) with _requires_login(): store.push_precheck(snap_name) snap_cache = cache.SnapCache(project_name=snap_name) arch = snap_yaml['architectures'][0] source_snap = snap_cache.get(deb_arch=arch) sha3_384_available = hasattr(hashlib, 'sha3_384') if sha3_384_available and source_snap: try: result = _push_delta(snap_name, snap_filename, source_snap) except StoreDeltaApplicationError as e: logger.warning( 'Error generating delta: {}\n' 'Falling back to pushing full snap...'.format(str(e))) result = _push_snap(snap_name, snap_filename) except storeapi.errors.StorePushError as e: store_error = e.error_list[0].get('message') logger.warning( 'Unable to push delta to store: {}\n' 'Falling back to pushing full snap...'.format(store_error)) result = _push_snap(snap_name, snap_filename) else: result = _push_snap(snap_name, snap_filename) # This is workaround until LP: #1599875 is solved if 'revision' in result: logger.info('Revision {!r} of {!r} created.'.format( result['revision'], snap_name)) snap_cache.cache(snap_filename=snap_filename) snap_cache.prune(deb_arch=arch, keep_hash=calculate_sha3_384(snap_filename)) else: logger.info('Pushing {!r}'.format(snap_name)) if release_channels: release(snap_name, result['revision'], release_channels)
def push(snap_filename, release_channels=None): """Push a snap_filename to the store. If a cached snap is available, a delta will be generated from the cached snap to the new target snap and uploaded instead. In the case of a delta processing or upload failure, push will fall back to uploading the full snap. If release_channels is defined it also releases it to those channels if the store deems the uploaded snap as ready to release. """ if not os.path.exists(snap_filename): raise FileNotFoundError( 'The file {!r} does not exist.'.format(snap_filename)) snap_yaml = _get_data_from_snap_file(snap_filename) snap_name = snap_yaml['name'] store = storeapi.StoreClient() logger.info('Pushing {!r} to the store.'.format(snap_filename)) with _requires_login(): store.push_precheck(snap_name) snap_cache = cache.SnapCache(project_name=snap_name) arch = snap_yaml['architectures'][0] source_snap = snap_cache.get(deb_arch=arch) sha3_384_available = hasattr(hashlib, 'sha3_384') if sha3_384_available and source_snap: try: result = _push_delta(snap_name, snap_filename, source_snap) except StoreDeltaApplicationError as e: logger.warning( 'Error generating delta: {}\n' 'Falling back to pushing full snap...'.format(str(e))) result = _push_snap(snap_name, snap_filename) except storeapi.errors.StorePushError as e: store_error = e.error_list[0].get('message') logger.warning( 'Unable to push delta to store: {}\n' 'Falling back to pushing full snap...'.format(store_error)) result = _push_snap(snap_name, snap_filename) else: result = _push_snap(snap_name, snap_filename) # This is workaround until LP: #1599875 is solved if 'revision' in result: logger.info('Revision {!r} of {!r} created.'.format( result['revision'], snap_name)) snap_cache.cache(snap_filename=snap_filename) snap_cache.prune(deb_arch=arch, keep_hash=calculate_sha3_384(snap_filename)) else: logger.info('Pushing {!r}'.format(snap_name)) if release_channels: release(snap_name, result['revision'], release_channels)
def test_snap_cache_get_latest(self): # Create snaps with open(os.path.join(self.path, "snapcraft.yaml"), "w") as f: f.write( """name: my-snap-name summary: test cached snap description: test cached snap architectures: ['{}'] confinement: devmode grade: devel version: '0.1' parts: my-part: plugin: nil """.format( self.deb_arch ) ) result = self.run_command(["snap"]) self.assertThat(result.exit_code, Equals(0)) snap_file = glob.glob("*0.1*.snap")[0] snap_cache = cache.SnapCache(project_name="my-snap-name") snap_cache.cache(snap_filename=snap_file) with open(os.path.join(self.path, "snapcraft.yaml"), "w") as f: f.write( """name: my-snap-name summary: test cached snap description: test cached snap architectures: ['{}'] confinement: devmode grade: devel version: '0.2' parts: my-part: plugin: nil """.format( self.deb_arch ) ) result = self.run_command(["snap"]) self.assertThat(result.exit_code, Equals(0)) snap_file_latest = glob.glob("*0.2*.snap")[0] snap_cache.cache(snap_filename=snap_file_latest) latest_hash = file_utils.calculate_sha3_384(snap_file_latest) # get latest latest_snap = snap_cache.get(deb_arch=self.deb_arch) expected_snap_path = os.path.join( snap_cache.snap_cache_root, self.deb_arch, latest_hash ) self.assertThat(latest_snap, Equals(expected_snap_path))
def _push_delta(snap_name, snap_filename, source_snap): store = storeapi.StoreClient() delta_format = 'xdelta3' logger.info('Found cached source snap {}.'.format(source_snap)) target_snap = os.path.join(os.getcwd(), snap_filename) try: xdelta_generator = deltas.XDelta3Generator(source_path=source_snap, target_path=target_snap) delta_filename = xdelta_generator.make_delta() except (DeltaGenerationError, DeltaGenerationTooBigError, DeltaToolError) as e: raise StoreDeltaApplicationError(str(e)) snap_hashes = { 'source_hash': calculate_sha3_384(source_snap), 'target_hash': calculate_sha3_384(target_snap), 'delta_hash': calculate_sha3_384(delta_filename) } try: logger.info('Pushing delta {}.'.format(delta_filename)) with _requires_login(): delta_tracker = store.upload( snap_name, delta_filename, delta_format=delta_format, source_hash=snap_hashes['source_hash'], target_hash=snap_hashes['target_hash'], delta_hash=snap_hashes['delta_hash']) result = delta_tracker.track() delta_tracker.raise_for_code() except storeapi.errors.StoreReviewError as e: if e.code == 'processing_upload_delta_error': raise StoreDeltaApplicationError else: raise finally: if os.path.isfile(delta_filename): try: os.remove(delta_filename) except OSError: logger.warning( 'Unable to remove delta {}.'.format(delta_filename)) return result
def push(snap_filename, release_channels=None): """Push a snap_filename to the store. If a cached snap is available, a delta will be generated from the cached snap to the new target snap and uploaded instead. In the case of a delta processing or upload failure, push will fall back to uploading the full snap. If release_channels is defined it also releases it to those channels if the store deems the uploaded snap as ready to release. """ snap_yaml = _get_data_from_snap_file(snap_filename) snap_name = snap_yaml["name"] built_at = snap_yaml.get("snapcraft-started-at") store = storeapi.StoreClient() logger.debug( "Run push precheck and verify cached data for {!r}.".format(snap_filename) ) with _requires_login(): store.push_precheck(snap_name) snap_cache = cache.SnapCache(project_name=snap_name) arch = "all" with contextlib.suppress(KeyError): arch = snap_yaml["architectures"][0] source_snap = snap_cache.get(deb_arch=arch) sha3_384_available = hasattr(hashlib, "sha3_384") if sha3_384_available and source_snap: try: result = _push_delta(snap_name, snap_filename, source_snap, built_at) except storeapi.errors.StoreDeltaApplicationError as e: logger.warning( "Error generating delta: {}\n" "Falling back to pushing full snap...".format(str(e)) ) result = _push_snap(snap_name, snap_filename, built_at) except storeapi.errors.StorePushError as e: store_error = e.error_list[0].get("message") logger.warning( "Unable to push delta to store: {}\n" "Falling back to pushing full snap...".format(store_error) ) result = _push_snap(snap_name, snap_filename, built_at) else: result = _push_snap(snap_name, snap_filename, built_at) logger.info("Revision {!r} of {!r} created.".format(result["revision"], snap_name)) snap_cache.cache(snap_filename=snap_filename) snap_cache.prune(deb_arch=arch, keep_hash=calculate_sha3_384(snap_filename)) if release_channels: release(snap_name, result["revision"], release_channels)
def _push_delta(snap_name, snap_filename, source_snap): store = storeapi.StoreClient() delta_format = 'xdelta3' logger.info('Found cached source snap {}.'.format(source_snap)) target_snap = os.path.join(os.getcwd(), snap_filename) try: xdelta_generator = deltas.XDelta3Generator( source_path=source_snap, target_path=target_snap) delta_filename = xdelta_generator.make_delta() except (DeltaGenerationError, DeltaGenerationTooBigError, DeltaToolError) as e: raise StoreDeltaApplicationError(str(e)) snap_hashes = {'source_hash': calculate_sha3_384(source_snap), 'target_hash': calculate_sha3_384(target_snap), 'delta_hash': calculate_sha3_384(delta_filename)} try: logger.info('Pushing delta {}.'.format(delta_filename)) with _requires_login(): delta_tracker = store.upload( snap_name, delta_filename, delta_format=delta_format, source_hash=snap_hashes['source_hash'], target_hash=snap_hashes['target_hash'], delta_hash=snap_hashes['delta_hash']) result = delta_tracker.track() delta_tracker.raise_for_code() except storeapi.errors.StoreReviewError as e: if e.code == 'processing_upload_delta_error': raise StoreDeltaApplicationError else: raise finally: if os.path.isfile(delta_filename): try: os.remove(delta_filename) except OSError: logger.warning( 'Unable to remove delta {}.'.format(delta_filename)) return result
def test_snap_cache(self): # cache snap snap_cache = cache.SnapCache(project_name='cache-test') cached_snap_path = snap_cache.cache(snap_filename=self.snap_path) expected_snap_path = os.path.join( snap_cache.snap_cache_root, 'amd64', file_utils.calculate_sha3_384(self.snap_path)) self.assertThat(cached_snap_path, Equals(expected_snap_path)) self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_latest(self): self.useFixture(fixture_setup.FakeTerminal()) # Create snaps with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f: f.write("""name: my-snap-name summary: test cached snap description: test cached snap architectures: ['{}'] confinement: devmode grade: devel version: '0.1' parts: my-part: plugin: nil """.format(self.deb_arch)) main(['snap']) snap_file = glob.glob('*0.1*.snap')[0] snap_cache = cache.SnapCache(project_name='my-snap-name') snap_cache.cache(snap_filename=snap_file) with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f: f.write("""name: my-snap-name summary: test cached snap description: test cached snap architectures: ['{}'] confinement: devmode grade: devel version: '0.2' parts: my-part: plugin: nil """.format(self.deb_arch)) main(['snap']) snap_file_latest = glob.glob('*0.2*.snap')[0] snap_cache.cache(snap_filename=snap_file_latest) latest_hash = file_utils.calculate_sha3_384(snap_file_latest) # get latest latest_snap = snap_cache.get(deb_arch=self.deb_arch) expected_snap_path = os.path.join( snap_cache.snap_cache_root, self.deb_arch, latest_hash ) self.assertEqual(expected_snap_path, latest_snap)
def test_push_revision_cached_with_experimental_deltas(self): # Upload with mock.patch('snapcraft.storeapi._status_tracker.' 'StatusTracker'): result = self.run_command(['push', self.snap_file]) self.assertThat(result.exit_code, Equals(0)) snap_cache = os.path.join(BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'basic', 'snap_hashes', 'amd64') cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file)) self.assertThat(cached_snap, FileExists())
def test_push_revision_prune_snap_cache(self): snap_revision = 9 patcher = mock.patch.object(storeapi.StoreClient, "get_snap_revisions") mock_release = patcher.start() self.addCleanup(patcher.stop) mock_release.return_value = [snap_revision] mock_tracker = mock.Mock(storeapi._status_tracker.StatusTracker) mock_tracker.track.return_value = { "code": "ready_to_release", "processed": True, "can_release": True, "url": "/fake/url", "revision": snap_revision, } patcher = mock.patch.object(storeapi.StoreClient, "upload") mock_upload = patcher.start() self.addCleanup(patcher.stop) mock_upload.return_value = mock_tracker deb_arch = "amd64" snap_cache = os.path.join( BaseDirectory.xdg_cache_home, "snapcraft", "projects", "basic", "snap_hashes", deb_arch, ) os.makedirs(snap_cache) for cached_snap in self.cached_snaps: cached_snap = cached_snap.format(deb_arch) open(os.path.join(snap_cache, cached_snap), "a").close() # Upload with mock.patch("snapcraft.storeapi._status_tracker.StatusTracker"): result = self.run_command(["push", self.snap_file]) self.assertThat(result.exit_code, Equals(0)) real_cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file)) self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists()) for snap in self.cached_snaps: snap = snap.format(deb_arch) self.assertThat(os.path.join(snap_cache, snap), Not(FileExists())) self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_push_revision_prune_snap_cache(self): snap_revision = 9 patcher = mock.patch.object(storeapi.StoreClient, "get_snap_revisions") mock_release = patcher.start() self.addCleanup(patcher.stop) mock_release.return_value = [snap_revision] mock_tracker = mock.Mock(storeapi._status_tracker.StatusTracker) mock_tracker.track.return_value = { "code": "ready_to_release", "processed": True, "can_release": True, "url": "/fake/url", "revision": snap_revision, } patcher = mock.patch.object(storeapi.StoreClient, "upload") mock_upload = patcher.start() self.addCleanup(patcher.stop) mock_upload.return_value = mock_tracker deb_arch = "amd64" snap_cache = os.path.join( BaseDirectory.xdg_cache_home, "snapcraft", "projects", "basic", "snap_hashes", deb_arch, ) os.makedirs(snap_cache) for cached_snap in self.cached_snaps: cached_snap = cached_snap.format(deb_arch) open(os.path.join(snap_cache, cached_snap), "a").close() # Upload with mock.patch("snapcraft.storeapi._status_tracker.StatusTracker"): result = self.run_command(["push", self.snap_file]) self.assertThat(result.exit_code, Equals(0)) real_cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file) ) self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists()) for snap in self.cached_snaps: snap = snap.format(deb_arch) self.assertThat(os.path.join(snap_cache, snap), Not(FileExists())) self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_snap_cache(self): # cache snap snap_cache = cache.SnapCache(project_name="cache-test") cached_snap_path = snap_cache.cache(snap_filename=self.snap_path) expected_snap_path = os.path.join( snap_cache.snap_cache_root, "amd64", file_utils.calculate_sha3_384(self.snap_path), ) self.assertThat(cached_snap_path, Equals(expected_snap_path)) self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache(self): self.useFixture(fixture_setup.FakeTerminal()) # cache snap snap_cache = cache.SnapCache(project_name='cache-test') cached_snap_path = snap_cache.cache(snap_filename=self.snap_path) expected_snap_path = os.path.join( snap_cache.snap_cache_root, 'amd64', file_utils.calculate_sha3_384(self.snap_path)) self.assertEqual(expected_snap_path, cached_snap_path) self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_by_hash(self): snap_cache = cache.SnapCache(project_name="my-snap-name") snap_cache.cache(snap_filename=self.snap_path) # get hash of snap snap_hash = file_utils.calculate_sha3_384(self.snap_path) # get snap by hash snap = snap_cache.get(deb_arch="amd64", snap_hash=snap_hash) self.assertThat( snap, Equals(os.path.join(snap_cache.snap_cache_root, "amd64", snap_hash)) )
def test_snap_cache_get_by_hash(self): self.useFixture(fixture_setup.FakeTerminal()) snap_cache = cache.SnapCache(project_name='my-snap-name') snap_cache.cache(snap_filename=self.snap_path) # get hash of snap snap_hash = file_utils.calculate_sha3_384(self.snap_path) # get snap by hash snap = snap_cache.get(deb_arch='amd64', snap_hash=snap_hash) self.assertEqual( os.path.join(snap_cache.snap_cache_root, 'amd64', snap_hash), snap)
def test_snap_cache_get_latest(self): # Create snaps meta_dir = os.path.join(self.path, "prime", "meta") os.makedirs(meta_dir) with open(os.path.join(meta_dir, "snap.yaml"), "w") as f: print( dedent("""\ name: my-snap-name summary: test cached snap description: test cached snap confinement: devmode grade: devel version: '0.1' """), file=f, ) result = self.run_command(["pack", os.path.join(self.path, "prime")]) self.assertThat(result.exit_code, Equals(0)) snap_file = glob.glob("*0.1*.snap")[0] snap_cache = cache.SnapCache(project_name="my-snap-name") snap_cache.cache(snap_filename=snap_file) with open(os.path.join(meta_dir, "snap.yaml"), "w") as f: print( dedent("""\ name: my-snap-name summary: test cached snap description: test cached snap confinement: devmode grade: devel version: '0.2' """), file=f, ) result = self.run_command(["pack", os.path.join(self.path, "prime")]) self.assertThat(result.exit_code, Equals(0)) snap_file_latest = glob.glob("*0.2*.snap")[0] snap_cache.cache(snap_filename=snap_file_latest) latest_hash = file_utils.calculate_sha3_384(snap_file_latest) # get latest latest_snap = snap_cache.get(deb_arch="all") expected_snap_path = os.path.join(snap_cache.snap_cache_root, "all", latest_hash) self.assertThat(latest_snap, Equals(expected_snap_path))
def test_snap_cache_get_by_hash(self): self.useFixture(fixture_setup.FakeTerminal()) snap_cache = cache.SnapCache(project_name='my-snap-name') snap_cache.cache(snap_filename=self.snap_path) # get hash of snap snap_hash = file_utils.calculate_sha3_384(self.snap_path) # get snap by hash snap = snap_cache.get(deb_arch='amd64', snap_hash=snap_hash) self.assertThat( snap, Equals( os.path.join(snap_cache.snap_cache_root, 'amd64', snap_hash)) )
def test_push_revision_cached_with_experimental_deltas(self): # Upload result = self.run_command(["push", self.snap_file]) self.assertThat(result.exit_code, Equals(0)) snap_cache = os.path.join( BaseDirectory.xdg_cache_home, "snapcraft", "projects", "basic", "snap_hashes", "amd64", ) cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file)) self.assertThat(cached_snap, FileExists())
def test_snap_cache(self): self.useFixture(fixture_setup.FakeTerminal()) # cache snap snap_cache = cache.SnapCache(project_name='cache-test') cached_snap_path = snap_cache.cache(snap_filename=self.snap_path) expected_snap_path = os.path.join( snap_cache.snap_cache_root, 'amd64', file_utils.calculate_sha3_384(self.snap_path) ) self.assertThat( cached_snap_path, Equals(expected_snap_path)) self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_latest_no_architectures(self): # Create snaps meta_dir = os.path.join(self.path, "prime", "meta") os.makedirs(meta_dir) with open(os.path.join(meta_dir, "snap.yaml"), "w") as f: f.write( """name: my-snap-name summary: test cached snap description: test cached snap confinement: devmode grade: devel version: '0.1' """ ) result = self.run_command(["pack", os.path.join(self.path, "prime")]) self.assertThat(result.exit_code, Equals(0)) snap_file = glob.glob("*0.1*.snap")[0] snap_cache = cache.SnapCache(project_name="my-snap-name") snap_cache.cache(snap_filename=snap_file) with open(os.path.join(meta_dir, "snap.yaml"), "w") as f: f.write( """name: my-snap-name summary: test cached snap description: test cached snap confinement: devmode grade: devel version: '0.2' """ ) result = self.run_command(["pack", os.path.join(self.path, "prime")]) self.assertThat(result.exit_code, Equals(0)) snap_file_latest = glob.glob("*0.2*.snap")[0] snap_cache.cache(snap_filename=snap_file_latest) latest_hash = file_utils.calculate_sha3_384(snap_file_latest) # get latest latest_snap = snap_cache.get(deb_arch="all") expected_snap_path = os.path.join( snap_cache.snap_cache_root, "all", latest_hash ) self.assertThat(latest_snap, Equals(expected_snap_path))
def test_push_revision_cached_with_experimental_deltas(self): # Upload with mock.patch('snapcraft.storeapi.StatusTracker'): result = self.run_command(['push', self.snap_file]) self.assertThat(result.exit_code, Equals(0)) snap_cache = os.path.join( BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'basic', 'snap_hashes', 'amd64' ) cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file)) self.assertThat(cached_snap, FileExists())
def test_push_revision_cached_with_experimental_deltas(self): # Upload with mock.patch("snapcraft.storeapi._status_tracker.StatusTracker"): result = self.run_command(["push", self.snap_file]) self.assertThat(result.exit_code, Equals(0)) snap_cache = os.path.join( BaseDirectory.xdg_cache_home, "snapcraft", "projects", "basic", "snap_hashes", "amd64", ) cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file) ) self.assertThat(cached_snap, FileExists())
def test_push_revision_prune_snap_cache(self): snap_revision = 9 self.mock_tracker.track.return_value = { "code": "ready_to_release", "processed": True, "can_release": True, "url": "/fake/url", "revision": snap_revision, } deb_arch = "amd64" snap_cache = os.path.join( BaseDirectory.xdg_cache_home, "snapcraft", "projects", "basic", "snap_hashes", deb_arch, ) os.makedirs(snap_cache) for cached_snap in self.cached_snaps: cached_snap = cached_snap.format(deb_arch) open(os.path.join(snap_cache, cached_snap), "a").close() # Upload result = self.run_command(["push", self.snap_file]) self.assertThat(result.exit_code, Equals(0)) real_cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(self.snap_file)) self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists()) for snap in self.cached_snaps: snap = snap.format(deb_arch) self.assertThat(os.path.join(snap_cache, snap), Not(FileExists())) self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_snap_cache_get_latest_no_architectures(self): # Create snaps meta_dir = os.path.join(self.path, 'prime', 'meta') os.makedirs(meta_dir) with open(os.path.join(meta_dir, 'snap.yaml'), 'w') as f: f.write("""name: my-snap-name summary: test cached snap description: test cached snap confinement: devmode grade: devel version: '0.1' """) result = self.run_command(['pack', os.path.join(self.path, 'prime')]) self.assertThat(result.exit_code, Equals(0)) snap_file = glob.glob('*0.1*.snap')[0] snap_cache = cache.SnapCache(project_name='my-snap-name') snap_cache.cache(snap_filename=snap_file) with open(os.path.join(meta_dir, 'snap.yaml'), 'w') as f: f.write("""name: my-snap-name summary: test cached snap description: test cached snap confinement: devmode grade: devel version: '0.2' """) result = self.run_command(['pack', os.path.join(self.path, 'prime')]) self.assertThat(result.exit_code, Equals(0)) snap_file_latest = glob.glob('*0.2*.snap')[0] snap_cache.cache(snap_filename=snap_file_latest) latest_hash = file_utils.calculate_sha3_384(snap_file_latest) # get latest latest_snap = snap_cache.get(deb_arch='all') expected_snap_path = os.path.join(snap_cache.snap_cache_root, 'all', latest_hash) self.assertThat(latest_snap, Equals(expected_snap_path))
def test_core_setup(self, download_mock): core_snap = self._create_core_snap() core_snap_hash = calculate_sha3_384(core_snap) download_mock.return_value = core_snap_hash self.tempdir_mock.side_effect = self._setup_tempdir_side_effect( core_snap) self._create_classic_confined_snapcraft_yaml() lifecycle.execute('pull', self.project_options) regex = ('mkdir -p {}\n' 'unsquashfs -d {} .*{}\n').format( os.path.dirname(self.core_path), self.core_path, core_snap_hash) self.assertThat( self.witness_path, FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL))) download_mock.assert_called_once_with( 'core', 'stable', os.path.join(self.tempdir, 'core.snap'), self.project_options.deb_arch, '')
def test_snap_cache_get_by_hash(self): self.useFixture(fixture_setup.FakeTerminal()) # Create snap main(['init']) main(['snap']) snap_file = glob.glob('*.snap')[0] snap_cache = cache.SnapCache(project_name='my-snap-name') snap_cache.cache(snap_filename=snap_file) # get hash of snap snap_hash = file_utils.calculate_sha3_384(snap_file) # get snap by hash snap = snap_cache.get(deb_arch=self.deb_arch, snap_hash=snap_hash) self.assertEqual( os.path.join(snap_cache.snap_cache_root, self.deb_arch, snap_hash), snap )
def test_core_setup(self, download_mock): core_snap = self._create_core_snap() core_snap_hash = calculate_sha3_384(core_snap) download_mock.return_value = core_snap_hash self.tempdir_mock.side_effect = self._setup_tempdir_side_effect( core_snap) self._create_classic_confined_snapcraft_yaml() lifecycle.execute('pull', self.project_options) regex = ( 'mkdir -p {}\n' 'unsquashfs -d {} .*{}\n').format( os.path.dirname(self.core_path), self.core_path, core_snap_hash) self.assertThat( self.witness_path, FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL))) download_mock.assert_called_once_with( 'core', 'stable', os.path.join(self.tempdir, 'core.snap'), self.project_options.deb_arch, '')
def test_push_revision_cached_with_experimental_deltas(self): self.useFixture(fixture_setup.FakeTerminal()) # Create a snap main(['init']) main(['snap']) snap_file = glob.glob('*.snap')[0] # Upload with mock.patch('snapcraft.storeapi.StatusTracker'): main(['push', snap_file]) snap_cache = os.path.join( BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'my-snap-name', 'snap_hashes', self.deb_arch, ) cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(snap_file)) self.assertThat(cached_snap, FileExists())
def test_snap_cache(self): self.useFixture(fixture_setup.FakeTerminal()) # Create a snap main(['init']) main(['snap']) snap_file = glob.glob('*.snap')[0] snap_path = os.path.join(self.path, snap_file) # cache snap snap_cache = cache.SnapCache(project_name='cache-test') cached_snap_path = snap_cache.cache(snap_filename=snap_file) expected_snap_path = os.path.join( snap_cache.snap_cache_root, self.deb_arch, file_utils.calculate_sha3_384(snap_path) ) self.assertEqual( expected_snap_path, cached_snap_path ) self.assertTrue(os.path.isfile(cached_snap_path))
def _get_snap_cache_path(self, snap_filename): snap_hash = file_utils.calculate_sha3_384(snap_filename) arch = self._get_snap_deb_arch(snap_filename) os.makedirs(os.path.join(self.snap_cache_root, arch), exist_ok=True) return os.path.join(self.snap_cache_root, arch, snap_hash)
def _get_snap_cache_path(self, snap_filename): snap_hash = file_utils.calculate_sha3_384(snap_filename) arch = self._get_snap_deb_arch(snap_filename) os.makedirs(os.path.join(self.snap_cache_root, arch), exist_ok=True) return os.path.join(self.snap_cache_root, arch, snap_hash)
def test_push_revision_prune_snap_cache(self): self.useFixture(fixture_setup.FakeTerminal()) snap_revision = 9 patcher = mock.patch('snapcraft.storeapi.StoreClient.push_precheck') patcher.start() self.addCleanup(patcher.stop) patcher = mock.patch.object(storeapi.StoreClient, 'get_snap_revisions') mock_release = patcher.start() self.addCleanup(patcher.stop) mock_release.return_value = [snap_revision] mock_tracker = mock.Mock(storeapi.StatusTracker) mock_tracker.track.return_value = { 'code': 'ready_to_release', 'processed': True, 'can_release': True, 'url': '/fake/url', 'revision': snap_revision, } patcher = mock.patch.object(storeapi.StoreClient, 'upload') mock_upload = patcher.start() self.addCleanup(patcher.stop) mock_upload.return_value = mock_tracker deb_arch = snapcraft.ProjectOptions().deb_arch snap_cache = os.path.join( BaseDirectory.xdg_cache_home, 'snapcraft', 'projects', 'my-snap-name', 'snap_hashes', deb_arch ) os.makedirs(snap_cache) for cached_snap in self.cached_snaps: cached_snap = cached_snap.format(deb_arch) open(os.path.join(snap_cache, cached_snap), 'a').close() # Create a snap main(['init']) main(['snap']) snap_file = glob.glob('*.snap')[0] # Upload with mock.patch('snapcraft.storeapi.StatusTracker'): main(['push', snap_file]) real_cached_snap = os.path.join( snap_cache, file_utils.calculate_sha3_384(snap_file) ) self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists()) for snap in self.cached_snaps: snap = snap.format(deb_arch) self.assertThat(os.path.join(snap_cache, snap), Not(FileExists())) self.assertEqual(1, len(os.listdir(snap_cache)))
def upload(snap_filename, release_channels=None) -> Tuple[str, int]: """Upload a snap_filename to the store. If a cached snap is available, a delta will be generated from the cached snap to the new target snap and uploaded instead. In the case of a delta processing or upload failure, upload will fall back to uploading the full snap. If release_channels is defined it also releases it to those channels if the store deems the uploaded snap as ready to release. """ snap_yaml = _get_data_from_snap_file(snap_filename) snap_name = snap_yaml["name"] built_at = snap_yaml.get("snapcraft-started-at") logger.debug("Run upload precheck and verify cached data for {!r}.".format( snap_filename)) store_client = StoreClientCLI() store_client.upload_precheck(snap_name=snap_name) snap_cache = cache.SnapCache(project_name=snap_name) try: deb_arch = snap_yaml["architectures"][0] except KeyError: deb_arch = "all" source_snap = snap_cache.get(deb_arch=deb_arch) sha3_384_available = hasattr(hashlib, "sha3_384") result: Optional[Dict[str, Any]] = None if sha3_384_available and source_snap: try: result = _upload_delta( store_client, snap_name=snap_name, snap_filename=snap_filename, source_snap=source_snap, built_at=built_at, channels=release_channels, ) except storeapi.errors.StoreDeltaApplicationError as e: logger.warning("Error generating delta: {}\n" "Falling back to uploading full snap...".format( str(e))) except storeapi.errors.StoreUploadError as upload_error: logger.warning("Unable to upload delta to store: {}\n" "Falling back to uploading full snap...".format( upload_error.error_list)) if result is None: result = _upload_snap( store_client, snap_name=snap_name, snap_filename=snap_filename, built_at=built_at, channels=release_channels, ) snap_cache.cache(snap_filename=snap_filename) snap_cache.prune(deb_arch=deb_arch, keep_hash=calculate_sha3_384(snap_filename)) return snap_name, result["revision"]