def _upload_delta(
store_client,
*,
snap_name: str,
snap_filename: str,
source_snap: str,
built_at: str,
channels: Optional[List[str]] = None,
) -> Dict[str, Any]:
delta_format = "xdelta3"
logger.debug("Found cached source snap {}.".format(source_snap))
target_snap = os.path.join(os.getcwd(), snap_filename)
try:
xdelta_generator = deltas.XDelta3Generator(source_path=source_snap,
target_path=target_snap)
delta_filename = xdelta_generator.make_delta()
except (DeltaGenerationError, DeltaGenerationTooBigError,
ToolMissingError) as e:
raise storeapi.errors.StoreDeltaApplicationError(str(e))
snap_hashes = {
"source_hash": calculate_sha3_384(source_snap),
"target_hash": calculate_sha3_384(target_snap),
"delta_hash": calculate_sha3_384(delta_filename),
}
try:
logger.debug("Uploading delta {!r}.".format(delta_filename))
delta_tracker = store_client.upload(
snap_name=snap_name,
snap_filename=delta_filename,
built_at=built_at,
channels=channels,
delta_format=delta_format,
source_hash=snap_hashes["source_hash"],
target_hash=snap_hashes["target_hash"],
delta_hash=snap_hashes["delta_hash"],
)
result = delta_tracker.track()
delta_tracker.raise_for_code()
except storeapi.errors.StoreReviewError as e:
if e.code == "processing_upload_delta_error":
raise storeapi.errors.StoreDeltaApplicationError(str(e))
else:
raise
except storeapi.errors.StoreServerError as e:
raise storeapi.errors.StoreUploadError(snap_name, e.response)
finally:
if os.path.isfile(delta_filename):
try:
os.remove(delta_filename)
except OSError:
logger.warning(
"Unable to remove delta {}.".format(delta_filename))
return result
def _push_delta(snap_name, snap_filename, source_snap, built_at):
store = storeapi.StoreClient()
delta_format = "xdelta3"
logger.debug("Found cached source snap {}.".format(source_snap))
target_snap = os.path.join(os.getcwd(), snap_filename)
try:
xdelta_generator = deltas.XDelta3Generator(source_path=source_snap,
target_path=target_snap)
delta_filename = xdelta_generator.make_delta()
except (DeltaGenerationError, DeltaGenerationTooBigError,
DeltaToolError) as e:
raise storeapi.errors.StoreDeltaApplicationError(str(e))
snap_hashes = {
"source_hash": calculate_sha3_384(source_snap),
"target_hash": calculate_sha3_384(target_snap),
"delta_hash": calculate_sha3_384(delta_filename),
}
try:
logger.debug("Pushing delta {!r}.".format(delta_filename))
with _requires_login():
delta_tracker = store.upload(
snap_name,
delta_filename,
delta_format=delta_format,
source_hash=snap_hashes["source_hash"],
target_hash=snap_hashes["target_hash"],
delta_hash=snap_hashes["delta_hash"],
built_at=built_at,
)
result = delta_tracker.track()
delta_tracker.raise_for_code()
except storeapi.errors.StoreReviewError as e:
if e.code == "processing_upload_delta_error":
raise storeapi.errors.StoreDeltaApplicationError(str(e))
else:
raise
except storeapi.errors.StoreServerError as e:
raise storeapi.errors.StorePushError(snap_name, e.response)
finally:
if os.path.isfile(delta_filename):
try:
os.remove(delta_filename)
except OSError:
logger.warning(
"Unable to remove delta {}.".format(delta_filename))
return result
def test_push_revision_prune_snap_cache(self):
snap_revision = 9
patcher = mock.patch('snapcraft.storeapi.StoreClient.push_precheck')
patcher.start()
self.addCleanup(patcher.stop)
patcher = mock.patch.object(storeapi.StoreClient, 'get_snap_revisions')
mock_release = patcher.start()
self.addCleanup(patcher.stop)
mock_release.return_value = [snap_revision]
mock_tracker = mock.Mock(storeapi.StatusTracker)
mock_tracker.track.return_value = {
'code': 'ready_to_release',
'processed': True,
'can_release': True,
'url': '/fake/url',
'revision': snap_revision,
}
patcher = mock.patch.object(storeapi.StoreClient, 'upload')
mock_upload = patcher.start()
self.addCleanup(patcher.stop)
mock_upload.return_value = mock_tracker
deb_arch = 'amd64'
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
'snapcraft',
'projects',
'basic',
'snap_hashes',
deb_arch
)
os.makedirs(snap_cache)
for cached_snap in self.cached_snaps:
cached_snap = cached_snap.format(deb_arch)
open(os.path.join(snap_cache, cached_snap), 'a').close()
# Upload
with mock.patch('snapcraft.storeapi.StatusTracker'):
result = self.run_command(['push', self.snap_file])
self.assertThat(result.exit_code, Equals(0))
real_cached_snap = os.path.join(
snap_cache,
file_utils.calculate_sha3_384(self.snap_file)
)
self.assertThat(os.path.join(snap_cache, real_cached_snap),
FileExists())
for snap in self.cached_snaps:
snap = snap.format(deb_arch)
self.assertThat(os.path.join(snap_cache, snap),
Not(FileExists()))
self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_push_revision_cached_with_experimental_deltas(self):
self.useFixture(fixture_setup.FakeTerminal())
if self.enable_deltas:
self.useFixture(fixture_setup.DeltaUploads())
# Create a snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
# Upload
with mock.patch('snapcraft.storeapi.StatusTracker'):
main(['push', snap_file])
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
'snapcraft',
'projects',
'my-snap-name',
'snap_hashes',
self.deb_arch,
)
cached_snap = os.path.join(snap_cache,
file_utils.calculate_sha3_384(snap_file))
if self.enable_deltas:
self.assertThat(cached_snap, FileExists())
else:
self.assertThat(cached_snap, Not(FileExists()))
def _download_multipass(dl_dir: str, echoer) -> str:
"""Creates temporary Downloads installer to temp directory."""
dl_url = _fetch_installer_url()
dl_basename = os.path.basename(dl_url)
dl_path = os.path.join(dl_dir, dl_basename)
echoer.info("Downloading Multipass installer...\n{} -> {}".format(
dl_url, dl_path))
try:
request = requests.get(dl_url, stream=True, allow_redirects=True)
request.raise_for_status()
download_requests_stream(request, dl_path)
except requests.RequestException as e:
raise ProviderMultipassDownloadFailed(_requests_exception_hint(e))
digest = calculate_sha3_384(dl_path)
if digest != _MULTIPASS_DL_SHA3_384:
raise ProviderMultipassDownloadFailed(
"download failed verification (expected={} but found={})".format(
_MULTIPASS_DL_SHA3_384, digest))
echoer.info("Verified installer successfully...")
return dl_path
def test_core_setup_if_docker_env(self, dockerenv_fake, download_mock):
dockerenv_file = os.path.join(self.tempdir, "dockerenv")
os.makedirs(self.tempdir)
open(dockerenv_file, "w").close()
dockerenv_fake.return_value = dockerenv_file
project_config = self.make_snapcraft_project(confinement="classic")
core_snap = self.create_core_snap(project_config.project.deb_arch)
core_snap_hash = calculate_sha3_384(core_snap)
download_mock.return_value = core_snap_hash
self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap)
lifecycle.execute(steps.PULL, project_config)
regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format(
os.path.dirname(self.core_path), self.core_path, core_snap_hash
)
self.assertThat(
self.witness_path,
FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)),
)
download_mock.assert_called_once_with(
"core",
"stable",
os.path.join(self.tempdir, "core.snap"),
project_config.project.deb_arch,
"",
)
def test_core_setup_with_env_var(self, download_mock):
self.useFixture(fixtures.EnvironmentVariable("SNAPCRAFT_SETUP_CORE", "1"))
project_config = self.make_snapcraft_project(confinement="classic")
core_snap = self.create_core_snap(project_config.project.deb_arch)
core_snap_hash = calculate_sha3_384(core_snap)
download_mock.return_value = core_snap_hash
self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap)
lifecycle.execute(steps.PULL, project_config)
regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format(
os.path.dirname(self.core_path), self.core_path, core_snap_hash
)
self.assertThat(
self.witness_path,
FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)),
)
download_mock.assert_called_once_with(
"core",
"stable",
os.path.join(self.tempdir, "core.snap"),
project_config.project.deb_arch,
"",
)
def test_core_setup_with_env_var(self, download_mock):
self.useFixture(fixtures.EnvironmentVariable("SNAPCRAFT_SETUP_CORE", "1"))
project_config = self.make_snapcraft_project(confinement="classic")
core_snap = self.create_core_snap(project_config.project.deb_arch)
core_snap_hash = calculate_sha3_384(core_snap)
download_mock.return_value = core_snap_hash
self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap)
lifecycle.execute(steps.PULL, project_config)
regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format(
os.path.dirname(self.core_path), self.core_path, core_snap_hash
)
self.assertThat(
self.witness_path,
FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)),
)
download_mock.assert_called_once_with(
"core",
"stable",
os.path.join(self.tempdir, "core.snap"),
project_config.project.deb_arch,
"",
)
def test_core_setup_if_docker_env(self, dockerenv_fake, download_mock):
dockerenv_file = os.path.join(self.tempdir, "dockerenv")
os.makedirs(self.tempdir)
open(dockerenv_file, "w").close()
dockerenv_fake.return_value = dockerenv_file
project_config = self.make_snapcraft_project(confinement="classic")
core_snap = self.create_core_snap(project_config.project.deb_arch)
core_snap_hash = calculate_sha3_384(core_snap)
download_mock.return_value = core_snap_hash
self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(core_snap)
lifecycle.execute(steps.PULL, project_config)
regex = (".*mkdir -p {}\nunsquashfs -d {} .*{}\n").format(
os.path.dirname(self.core_path), self.core_path, core_snap_hash
)
self.assertThat(
self.witness_path,
FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)),
)
download_mock.assert_called_once_with(
"core",
"stable",
os.path.join(self.tempdir, "core.snap"),
project_config.project.deb_arch,
"",
)
def test_push_revision_prune_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
self.useFixture(fixture_setup.DeltaUploads())
snap_revision = 9
patcher = mock.patch('snapcraft.storeapi.StoreClient.push_precheck')
patcher.start()
self.addCleanup(patcher.stop)
patcher = mock.patch.object(storeapi.StoreClient, 'get_snap_history')
mock_release = patcher.start()
self.addCleanup(patcher.stop)
mock_release.return_value = [snap_revision]
mock_tracker = mock.Mock(storeapi.StatusTracker)
mock_tracker.track.return_value = {
'code': 'ready_to_release',
'processed': True,
'can_release': True,
'url': '/fake/url',
'revision': snap_revision,
}
patcher = mock.patch.object(storeapi.StoreClient, 'upload')
mock_upload = patcher.start()
self.addCleanup(patcher.stop)
mock_upload.return_value = mock_tracker
deb_arch = snapcraft.ProjectOptions().deb_arch
snap_cache = os.path.join(BaseDirectory.xdg_cache_home, 'snapcraft',
'projects', 'my-snap-name', 'snap_hashes',
deb_arch)
os.makedirs(snap_cache)
for cached_snap in self.cached_snaps:
cached_snap = cached_snap.format(deb_arch)
open(os.path.join(snap_cache, cached_snap), 'a').close()
# Create a snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
# Upload
with mock.patch('snapcraft.storeapi.StatusTracker'):
main(['push', snap_file])
real_cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(snap_file))
self.assertThat(os.path.join(snap_cache, real_cached_snap),
FileExists())
for snap in self.cached_snaps:
snap = snap.format(deb_arch)
self.assertThat(os.path.join(snap_cache, snap), Not(FileExists()))
self.assertEqual(1, len(os.listdir(snap_cache)))
def push(snap_filename, release_channels=None):
"""Push a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, push will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
if not os.path.exists(snap_filename):
raise FileNotFoundError(
'The file {!r} does not exist.'.format(snap_filename))
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml['name']
store = storeapi.StoreClient()
logger.info('Preparing to push {!r} to the store.'.format(snap_filename))
with _requires_login():
store.push_precheck(snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
arch = snap_yaml['architectures'][0]
source_snap = snap_cache.get(deb_arch=arch)
sha3_384_available = hasattr(hashlib, 'sha3_384')
if sha3_384_available and source_snap:
try:
result = _push_delta(snap_name, snap_filename, source_snap)
except StoreDeltaApplicationError as e:
logger.warning(
'Error generating delta: {}\n'
'Falling back to pushing full snap...'.format(str(e)))
result = _push_snap(snap_name, snap_filename)
except storeapi.errors.StorePushError as e:
store_error = e.error_list[0].get('message')
logger.warning(
'Unable to push delta to store: {}\n'
'Falling back to pushing full snap...'.format(store_error))
result = _push_snap(snap_name, snap_filename)
else:
result = _push_snap(snap_name, snap_filename)
# This is workaround until LP: #1599875 is solved
if 'revision' in result:
logger.info('Revision {!r} of {!r} created.'.format(
result['revision'], snap_name))
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=arch,
keep_hash=calculate_sha3_384(snap_filename))
else:
logger.info('Pushing {!r}'.format(snap_name))
if release_channels:
release(snap_name, result['revision'], release_channels)
def push(snap_filename, release_channels=None):
"""Push a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, push will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
if not os.path.exists(snap_filename):
raise FileNotFoundError(
'The file {!r} does not exist.'.format(snap_filename))
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml['name']
store = storeapi.StoreClient()
logger.info('Pushing {!r} to the store.'.format(snap_filename))
with _requires_login():
store.push_precheck(snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
arch = snap_yaml['architectures'][0]
source_snap = snap_cache.get(deb_arch=arch)
sha3_384_available = hasattr(hashlib, 'sha3_384')
if sha3_384_available and source_snap:
try:
result = _push_delta(snap_name, snap_filename, source_snap)
except StoreDeltaApplicationError as e:
logger.warning(
'Error generating delta: {}\n'
'Falling back to pushing full snap...'.format(str(e)))
result = _push_snap(snap_name, snap_filename)
except storeapi.errors.StorePushError as e:
store_error = e.error_list[0].get('message')
logger.warning(
'Unable to push delta to store: {}\n'
'Falling back to pushing full snap...'.format(store_error))
result = _push_snap(snap_name, snap_filename)
else:
result = _push_snap(snap_name, snap_filename)
# This is workaround until LP: #1599875 is solved
if 'revision' in result:
logger.info('Revision {!r} of {!r} created.'.format(
result['revision'], snap_name))
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=arch,
keep_hash=calculate_sha3_384(snap_filename))
else:
logger.info('Pushing {!r}'.format(snap_name))
if release_channels:
release(snap_name, result['revision'], release_channels)
def test_snap_cache_get_latest(self):
# Create snaps
with open(os.path.join(self.path, "snapcraft.yaml"), "w") as f:
f.write(
"""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.1'
parts:
my-part:
plugin: nil
""".format(
self.deb_arch
)
)
result = self.run_command(["snap"])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob("*0.1*.snap")[0]
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(self.path, "snapcraft.yaml"), "w") as f:
f.write(
"""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.2'
parts:
my-part:
plugin: nil
""".format(
self.deb_arch
)
)
result = self.run_command(["snap"])
self.assertThat(result.exit_code, Equals(0))
snap_file_latest = glob.glob("*0.2*.snap")[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch=self.deb_arch)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root, self.deb_arch, latest_hash
)
self.assertThat(latest_snap, Equals(expected_snap_path))
def _push_delta(snap_name, snap_filename, source_snap):
store = storeapi.StoreClient()
delta_format = 'xdelta3'
logger.info('Found cached source snap {}.'.format(source_snap))
target_snap = os.path.join(os.getcwd(), snap_filename)
try:
xdelta_generator = deltas.XDelta3Generator(source_path=source_snap,
target_path=target_snap)
delta_filename = xdelta_generator.make_delta()
except (DeltaGenerationError, DeltaGenerationTooBigError,
DeltaToolError) as e:
raise StoreDeltaApplicationError(str(e))
snap_hashes = {
'source_hash': calculate_sha3_384(source_snap),
'target_hash': calculate_sha3_384(target_snap),
'delta_hash': calculate_sha3_384(delta_filename)
}
try:
logger.info('Pushing delta {}.'.format(delta_filename))
with _requires_login():
delta_tracker = store.upload(
snap_name,
delta_filename,
delta_format=delta_format,
source_hash=snap_hashes['source_hash'],
target_hash=snap_hashes['target_hash'],
delta_hash=snap_hashes['delta_hash'])
result = delta_tracker.track()
delta_tracker.raise_for_code()
except storeapi.errors.StoreReviewError as e:
if e.code == 'processing_upload_delta_error':
raise StoreDeltaApplicationError
else:
raise
finally:
if os.path.isfile(delta_filename):
try:
os.remove(delta_filename)
except OSError:
logger.warning(
'Unable to remove delta {}.'.format(delta_filename))
return result
def push(snap_filename, release_channels=None):
"""Push a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, push will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml["name"]
built_at = snap_yaml.get("snapcraft-started-at")
store = storeapi.StoreClient()
logger.debug(
"Run push precheck and verify cached data for {!r}.".format(snap_filename)
)
with _requires_login():
store.push_precheck(snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
arch = "all"
with contextlib.suppress(KeyError):
arch = snap_yaml["architectures"][0]
source_snap = snap_cache.get(deb_arch=arch)
sha3_384_available = hasattr(hashlib, "sha3_384")
if sha3_384_available and source_snap:
try:
result = _push_delta(snap_name, snap_filename, source_snap, built_at)
except storeapi.errors.StoreDeltaApplicationError as e:
logger.warning(
"Error generating delta: {}\n"
"Falling back to pushing full snap...".format(str(e))
)
result = _push_snap(snap_name, snap_filename, built_at)
except storeapi.errors.StorePushError as e:
store_error = e.error_list[0].get("message")
logger.warning(
"Unable to push delta to store: {}\n"
"Falling back to pushing full snap...".format(store_error)
)
result = _push_snap(snap_name, snap_filename, built_at)
else:
result = _push_snap(snap_name, snap_filename, built_at)
logger.info("Revision {!r} of {!r} created.".format(result["revision"], snap_name))
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=arch, keep_hash=calculate_sha3_384(snap_filename))
if release_channels:
release(snap_name, result["revision"], release_channels)
def _push_delta(snap_name, snap_filename, source_snap):
store = storeapi.StoreClient()
delta_format = 'xdelta3'
logger.info('Found cached source snap {}.'.format(source_snap))
target_snap = os.path.join(os.getcwd(), snap_filename)
try:
xdelta_generator = deltas.XDelta3Generator(
source_path=source_snap, target_path=target_snap)
delta_filename = xdelta_generator.make_delta()
except (DeltaGenerationError, DeltaGenerationTooBigError,
DeltaToolError) as e:
raise StoreDeltaApplicationError(str(e))
snap_hashes = {'source_hash': calculate_sha3_384(source_snap),
'target_hash': calculate_sha3_384(target_snap),
'delta_hash': calculate_sha3_384(delta_filename)}
try:
logger.info('Pushing delta {}.'.format(delta_filename))
with _requires_login():
delta_tracker = store.upload(
snap_name,
delta_filename,
delta_format=delta_format,
source_hash=snap_hashes['source_hash'],
target_hash=snap_hashes['target_hash'],
delta_hash=snap_hashes['delta_hash'])
result = delta_tracker.track()
delta_tracker.raise_for_code()
except storeapi.errors.StoreReviewError as e:
if e.code == 'processing_upload_delta_error':
raise StoreDeltaApplicationError
else:
raise
finally:
if os.path.isfile(delta_filename):
try:
os.remove(delta_filename)
except OSError:
logger.warning(
'Unable to remove delta {}.'.format(delta_filename))
return result
def test_snap_cache(self):
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=self.snap_path)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root, 'amd64',
file_utils.calculate_sha3_384(self.snap_path))
self.assertThat(cached_snap_path, Equals(expected_snap_path))
self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_latest(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create snaps
with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.1'
parts:
my-part:
plugin: nil
""".format(self.deb_arch))
main(['snap'])
snap_file = glob.glob('*0.1*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.2'
parts:
my-part:
plugin: nil
""".format(self.deb_arch))
main(['snap'])
snap_file_latest = glob.glob('*0.2*.snap')[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch=self.deb_arch)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root,
self.deb_arch,
latest_hash
)
self.assertEqual(expected_snap_path, latest_snap)
def test_push_revision_cached_with_experimental_deltas(self):
# Upload
with mock.patch('snapcraft.storeapi._status_tracker.' 'StatusTracker'):
result = self.run_command(['push', self.snap_file])
self.assertThat(result.exit_code, Equals(0))
snap_cache = os.path.join(BaseDirectory.xdg_cache_home, 'snapcraft',
'projects', 'basic', 'snap_hashes', 'amd64')
cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file))
self.assertThat(cached_snap, FileExists())
def test_push_revision_prune_snap_cache(self):
snap_revision = 9
patcher = mock.patch.object(storeapi.StoreClient, "get_snap_revisions")
mock_release = patcher.start()
self.addCleanup(patcher.stop)
mock_release.return_value = [snap_revision]
mock_tracker = mock.Mock(storeapi._status_tracker.StatusTracker)
mock_tracker.track.return_value = {
"code": "ready_to_release",
"processed": True,
"can_release": True,
"url": "/fake/url",
"revision": snap_revision,
}
patcher = mock.patch.object(storeapi.StoreClient, "upload")
mock_upload = patcher.start()
self.addCleanup(patcher.stop)
mock_upload.return_value = mock_tracker
deb_arch = "amd64"
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
"snapcraft",
"projects",
"basic",
"snap_hashes",
deb_arch,
)
os.makedirs(snap_cache)
for cached_snap in self.cached_snaps:
cached_snap = cached_snap.format(deb_arch)
open(os.path.join(snap_cache, cached_snap), "a").close()
# Upload
with mock.patch("snapcraft.storeapi._status_tracker.StatusTracker"):
result = self.run_command(["push", self.snap_file])
self.assertThat(result.exit_code, Equals(0))
real_cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file))
self.assertThat(os.path.join(snap_cache, real_cached_snap),
FileExists())
for snap in self.cached_snaps:
snap = snap.format(deb_arch)
self.assertThat(os.path.join(snap_cache, snap), Not(FileExists()))
self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_push_revision_prune_snap_cache(self):
snap_revision = 9
patcher = mock.patch.object(storeapi.StoreClient, "get_snap_revisions")
mock_release = patcher.start()
self.addCleanup(patcher.stop)
mock_release.return_value = [snap_revision]
mock_tracker = mock.Mock(storeapi._status_tracker.StatusTracker)
mock_tracker.track.return_value = {
"code": "ready_to_release",
"processed": True,
"can_release": True,
"url": "/fake/url",
"revision": snap_revision,
}
patcher = mock.patch.object(storeapi.StoreClient, "upload")
mock_upload = patcher.start()
self.addCleanup(patcher.stop)
mock_upload.return_value = mock_tracker
deb_arch = "amd64"
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
"snapcraft",
"projects",
"basic",
"snap_hashes",
deb_arch,
)
os.makedirs(snap_cache)
for cached_snap in self.cached_snaps:
cached_snap = cached_snap.format(deb_arch)
open(os.path.join(snap_cache, cached_snap), "a").close()
# Upload
with mock.patch("snapcraft.storeapi._status_tracker.StatusTracker"):
result = self.run_command(["push", self.snap_file])
self.assertThat(result.exit_code, Equals(0))
real_cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file)
)
self.assertThat(os.path.join(snap_cache, real_cached_snap), FileExists())
for snap in self.cached_snaps:
snap = snap.format(deb_arch)
self.assertThat(os.path.join(snap_cache, snap), Not(FileExists()))
self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_snap_cache(self):
# cache snap
snap_cache = cache.SnapCache(project_name="cache-test")
cached_snap_path = snap_cache.cache(snap_filename=self.snap_path)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root,
"amd64",
file_utils.calculate_sha3_384(self.snap_path),
)
self.assertThat(cached_snap_path, Equals(expected_snap_path))
self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=self.snap_path)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root, 'amd64',
file_utils.calculate_sha3_384(self.snap_path))
self.assertEqual(expected_snap_path, cached_snap_path)
self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_by_hash(self):
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_cache.cache(snap_filename=self.snap_path)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(self.snap_path)
# get snap by hash
snap = snap_cache.get(deb_arch="amd64", snap_hash=snap_hash)
self.assertThat(
snap, Equals(os.path.join(snap_cache.snap_cache_root, "amd64", snap_hash))
)
def test_snap_cache_get_by_hash(self):
self.useFixture(fixture_setup.FakeTerminal())
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=self.snap_path)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(self.snap_path)
# get snap by hash
snap = snap_cache.get(deb_arch='amd64', snap_hash=snap_hash)
self.assertEqual(
os.path.join(snap_cache.snap_cache_root, 'amd64', snap_hash), snap)
def test_snap_cache_get_latest(self):
# Create snaps
meta_dir = os.path.join(self.path, "prime", "meta")
os.makedirs(meta_dir)
with open(os.path.join(meta_dir, "snap.yaml"), "w") as f:
print(
dedent("""\
name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.1'
"""),
file=f,
)
result = self.run_command(["pack", os.path.join(self.path, "prime")])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob("*0.1*.snap")[0]
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(meta_dir, "snap.yaml"), "w") as f:
print(
dedent("""\
name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.2'
"""),
file=f,
)
result = self.run_command(["pack", os.path.join(self.path, "prime")])
self.assertThat(result.exit_code, Equals(0))
snap_file_latest = glob.glob("*0.2*.snap")[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch="all")
expected_snap_path = os.path.join(snap_cache.snap_cache_root, "all",
latest_hash)
self.assertThat(latest_snap, Equals(expected_snap_path))
def test_snap_cache_get_by_hash(self):
self.useFixture(fixture_setup.FakeTerminal())
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=self.snap_path)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(self.snap_path)
# get snap by hash
snap = snap_cache.get(deb_arch='amd64', snap_hash=snap_hash)
self.assertThat(
snap, Equals(
os.path.join(snap_cache.snap_cache_root, 'amd64', snap_hash))
)
def test_push_revision_cached_with_experimental_deltas(self):
# Upload
result = self.run_command(["push", self.snap_file])
self.assertThat(result.exit_code, Equals(0))
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
"snapcraft",
"projects",
"basic",
"snap_hashes",
"amd64",
)
cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file))
self.assertThat(cached_snap, FileExists())
def test_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=self.snap_path)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root,
'amd64',
file_utils.calculate_sha3_384(self.snap_path)
)
self.assertThat(
cached_snap_path,
Equals(expected_snap_path))
self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_latest_no_architectures(self):
# Create snaps
meta_dir = os.path.join(self.path, "prime", "meta")
os.makedirs(meta_dir)
with open(os.path.join(meta_dir, "snap.yaml"), "w") as f:
f.write(
"""name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.1'
"""
)
result = self.run_command(["pack", os.path.join(self.path, "prime")])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob("*0.1*.snap")[0]
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(meta_dir, "snap.yaml"), "w") as f:
f.write(
"""name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.2'
"""
)
result = self.run_command(["pack", os.path.join(self.path, "prime")])
self.assertThat(result.exit_code, Equals(0))
snap_file_latest = glob.glob("*0.2*.snap")[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch="all")
expected_snap_path = os.path.join(
snap_cache.snap_cache_root, "all", latest_hash
)
self.assertThat(latest_snap, Equals(expected_snap_path))
def test_push_revision_cached_with_experimental_deltas(self):
# Upload
with mock.patch('snapcraft.storeapi.StatusTracker'):
result = self.run_command(['push', self.snap_file])
self.assertThat(result.exit_code, Equals(0))
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
'snapcraft',
'projects',
'basic',
'snap_hashes',
'amd64'
)
cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file))
self.assertThat(cached_snap, FileExists())
def test_push_revision_cached_with_experimental_deltas(self):
# Upload
with mock.patch("snapcraft.storeapi._status_tracker.StatusTracker"):
result = self.run_command(["push", self.snap_file])
self.assertThat(result.exit_code, Equals(0))
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
"snapcraft",
"projects",
"basic",
"snap_hashes",
"amd64",
)
cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file)
)
self.assertThat(cached_snap, FileExists())
def test_push_revision_prune_snap_cache(self):
snap_revision = 9
self.mock_tracker.track.return_value = {
"code": "ready_to_release",
"processed": True,
"can_release": True,
"url": "/fake/url",
"revision": snap_revision,
}
deb_arch = "amd64"
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
"snapcraft",
"projects",
"basic",
"snap_hashes",
deb_arch,
)
os.makedirs(snap_cache)
for cached_snap in self.cached_snaps:
cached_snap = cached_snap.format(deb_arch)
open(os.path.join(snap_cache, cached_snap), "a").close()
# Upload
result = self.run_command(["push", self.snap_file])
self.assertThat(result.exit_code, Equals(0))
real_cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(self.snap_file))
self.assertThat(os.path.join(snap_cache, real_cached_snap),
FileExists())
for snap in self.cached_snaps:
snap = snap.format(deb_arch)
self.assertThat(os.path.join(snap_cache, snap), Not(FileExists()))
self.assertThat(len(os.listdir(snap_cache)), Equals(1))
def test_snap_cache_get_latest_no_architectures(self):
# Create snaps
meta_dir = os.path.join(self.path, 'prime', 'meta')
os.makedirs(meta_dir)
with open(os.path.join(meta_dir, 'snap.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.1'
""")
result = self.run_command(['pack', os.path.join(self.path, 'prime')])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob('*0.1*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(meta_dir, 'snap.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.2'
""")
result = self.run_command(['pack', os.path.join(self.path, 'prime')])
self.assertThat(result.exit_code, Equals(0))
snap_file_latest = glob.glob('*0.2*.snap')[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch='all')
expected_snap_path = os.path.join(snap_cache.snap_cache_root, 'all',
latest_hash)
self.assertThat(latest_snap, Equals(expected_snap_path))
def test_core_setup(self, download_mock):
core_snap = self._create_core_snap()
core_snap_hash = calculate_sha3_384(core_snap)
download_mock.return_value = core_snap_hash
self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(
core_snap)
self._create_classic_confined_snapcraft_yaml()
lifecycle.execute('pull', self.project_options)
regex = ('mkdir -p {}\n'
'unsquashfs -d {} .*{}\n').format(
os.path.dirname(self.core_path), self.core_path,
core_snap_hash)
self.assertThat(
self.witness_path,
FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)))
download_mock.assert_called_once_with(
'core', 'stable', os.path.join(self.tempdir, 'core.snap'),
self.project_options.deb_arch, '')
def test_snap_cache_get_by_hash(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=snap_file)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(snap_file)
# get snap by hash
snap = snap_cache.get(deb_arch=self.deb_arch, snap_hash=snap_hash)
self.assertEqual(
os.path.join(snap_cache.snap_cache_root, self.deb_arch, snap_hash),
snap
)
def test_core_setup(self, download_mock):
core_snap = self._create_core_snap()
core_snap_hash = calculate_sha3_384(core_snap)
download_mock.return_value = core_snap_hash
self.tempdir_mock.side_effect = self._setup_tempdir_side_effect(
core_snap)
self._create_classic_confined_snapcraft_yaml()
lifecycle.execute('pull', self.project_options)
regex = (
'mkdir -p {}\n'
'unsquashfs -d {} .*{}\n').format(
os.path.dirname(self.core_path),
self.core_path,
core_snap_hash)
self.assertThat(
self.witness_path,
FileContains(matcher=MatchesRegex(regex, flags=re.DOTALL)))
download_mock.assert_called_once_with(
'core', 'stable', os.path.join(self.tempdir, 'core.snap'),
self.project_options.deb_arch, '')
def test_push_revision_cached_with_experimental_deltas(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create a snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
# Upload
with mock.patch('snapcraft.storeapi.StatusTracker'):
main(['push', snap_file])
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
'snapcraft',
'projects',
'my-snap-name',
'snap_hashes',
self.deb_arch,
)
cached_snap = os.path.join(
snap_cache, file_utils.calculate_sha3_384(snap_file))
self.assertThat(cached_snap, FileExists())
def test_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create a snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
snap_path = os.path.join(self.path, snap_file)
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=snap_file)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root,
self.deb_arch,
file_utils.calculate_sha3_384(snap_path)
)
self.assertEqual(
expected_snap_path,
cached_snap_path
)
self.assertTrue(os.path.isfile(cached_snap_path))
def _get_snap_cache_path(self, snap_filename):
snap_hash = file_utils.calculate_sha3_384(snap_filename)
arch = self._get_snap_deb_arch(snap_filename)
os.makedirs(os.path.join(self.snap_cache_root, arch), exist_ok=True)
return os.path.join(self.snap_cache_root, arch, snap_hash)
def _get_snap_cache_path(self, snap_filename):
snap_hash = file_utils.calculate_sha3_384(snap_filename)
arch = self._get_snap_deb_arch(snap_filename)
os.makedirs(os.path.join(self.snap_cache_root, arch), exist_ok=True)
return os.path.join(self.snap_cache_root, arch, snap_hash)
def test_push_revision_prune_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
snap_revision = 9
patcher = mock.patch('snapcraft.storeapi.StoreClient.push_precheck')
patcher.start()
self.addCleanup(patcher.stop)
patcher = mock.patch.object(storeapi.StoreClient, 'get_snap_revisions')
mock_release = patcher.start()
self.addCleanup(patcher.stop)
mock_release.return_value = [snap_revision]
mock_tracker = mock.Mock(storeapi.StatusTracker)
mock_tracker.track.return_value = {
'code': 'ready_to_release',
'processed': True,
'can_release': True,
'url': '/fake/url',
'revision': snap_revision,
}
patcher = mock.patch.object(storeapi.StoreClient, 'upload')
mock_upload = patcher.start()
self.addCleanup(patcher.stop)
mock_upload.return_value = mock_tracker
deb_arch = snapcraft.ProjectOptions().deb_arch
snap_cache = os.path.join(
BaseDirectory.xdg_cache_home,
'snapcraft',
'projects',
'my-snap-name',
'snap_hashes',
deb_arch
)
os.makedirs(snap_cache)
for cached_snap in self.cached_snaps:
cached_snap = cached_snap.format(deb_arch)
open(os.path.join(snap_cache, cached_snap), 'a').close()
# Create a snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
# Upload
with mock.patch('snapcraft.storeapi.StatusTracker'):
main(['push', snap_file])
real_cached_snap = os.path.join(
snap_cache,
file_utils.calculate_sha3_384(snap_file)
)
self.assertThat(os.path.join(snap_cache, real_cached_snap),
FileExists())
for snap in self.cached_snaps:
snap = snap.format(deb_arch)
self.assertThat(os.path.join(snap_cache, snap),
Not(FileExists()))
self.assertEqual(1, len(os.listdir(snap_cache)))
def upload(snap_filename, release_channels=None) -> Tuple[str, int]:
"""Upload a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, upload will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml["name"]
built_at = snap_yaml.get("snapcraft-started-at")
logger.debug("Run upload precheck and verify cached data for {!r}.".format(
snap_filename))
store_client = StoreClientCLI()
store_client.upload_precheck(snap_name=snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
try:
deb_arch = snap_yaml["architectures"][0]
except KeyError:
deb_arch = "all"
source_snap = snap_cache.get(deb_arch=deb_arch)
sha3_384_available = hasattr(hashlib, "sha3_384")
result: Optional[Dict[str, Any]] = None
if sha3_384_available and source_snap:
try:
result = _upload_delta(
store_client,
snap_name=snap_name,
snap_filename=snap_filename,
source_snap=source_snap,
built_at=built_at,
channels=release_channels,
)
except storeapi.errors.StoreDeltaApplicationError as e:
logger.warning("Error generating delta: {}\n"
"Falling back to uploading full snap...".format(
str(e)))
except storeapi.errors.StoreUploadError as upload_error:
logger.warning("Unable to upload delta to store: {}\n"
"Falling back to uploading full snap...".format(
upload_error.error_list))
if result is None:
result = _upload_snap(
store_client,
snap_name=snap_name,
snap_filename=snap_filename,
built_at=built_at,
channels=release_channels,
)
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=deb_arch,
keep_hash=calculate_sha3_384(snap_filename))
return snap_name, result["revision"]
