def push(snap_filename, release_channels=None):
"""Push a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, push will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml['name']
store = storeapi.StoreClient()
logger.info('Preparing to push {!r} to the store.'.format(snap_filename))
with _requires_login():
store.push_precheck(snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
arch = snap_yaml['architectures'][0]
source_snap = snap_cache.get(deb_arch=arch)
sha3_384_available = hasattr(hashlib, 'sha3_384')
if sha3_384_available and source_snap:
try:
result = _push_delta(snap_name, snap_filename, source_snap)
except storeapi.errors.StoreDeltaApplicationError as e:
logger.warning('Error generating delta: {}\n'
'Falling back to pushing full snap...'.format(
str(e)))
result = _push_snap(snap_name, snap_filename)
except storeapi.errors.StorePushError as e:
store_error = e.error_list[0].get('message')
logger.warning(
'Unable to push delta to store: {}\n'
'Falling back to pushing full snap...'.format(store_error))
result = _push_snap(snap_name, snap_filename)
else:
result = _push_snap(snap_name, snap_filename)
# This is workaround until LP: #1599875 is solved
if 'revision' in result:
logger.info('Revision {!r} of {!r} created.'.format(
result['revision'], snap_name))
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=arch,
keep_hash=calculate_sha3_384(snap_filename))
else:
logger.info('Pushing {!r}'.format(snap_name))
if release_channels:
release(snap_name, result['revision'], release_channels)
def push(snap_filename, release_channels=None):
"""Push a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, push will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml["name"]
store = storeapi.StoreClient()
logger.debug("Run push precheck and verify cached data for {!r}.".format(
snap_filename))
with _requires_login():
store.push_precheck(snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
arch = "all"
with contextlib.suppress(KeyError):
arch = snap_yaml["architectures"][0]
source_snap = snap_cache.get(deb_arch=arch)
sha3_384_available = hasattr(hashlib, "sha3_384")
if sha3_384_available and source_snap:
try:
result = _push_delta(snap_name, snap_filename, source_snap)
except storeapi.errors.StoreDeltaApplicationError as e:
logger.warning("Error generating delta: {}\n"
"Falling back to pushing full snap...".format(
str(e)))
result = _push_snap(snap_name, snap_filename)
except storeapi.errors.StorePushError as e:
store_error = e.error_list[0].get("message")
logger.warning(
"Unable to push delta to store: {}\n"
"Falling back to pushing full snap...".format(store_error))
result = _push_snap(snap_name, snap_filename)
else:
result = _push_snap(snap_name, snap_filename)
logger.info("Revision {!r} of {!r} created.".format(
result["revision"], snap_name))
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=arch,
keep_hash=calculate_sha3_384(snap_filename))
if release_channels:
release(snap_name, result["revision"], release_channels)
def test_snap_cache(self):
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=self.snap_path)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root, 'amd64',
file_utils.calculate_sha3_384(self.snap_path))
self.assertThat(cached_snap_path, Equals(expected_snap_path))
self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_latest(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create snaps
with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.1'
parts:
my-part:
plugin: nil
""".format(self.deb_arch))
main(['snap'])
snap_file = glob.glob('*0.1*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.2'
parts:
my-part:
plugin: nil
""".format(self.deb_arch))
main(['snap'])
snap_file_latest = glob.glob('*0.2*.snap')[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch=self.deb_arch)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root,
self.deb_arch,
latest_hash
)
self.assertEqual(expected_snap_path, latest_snap)
def test_prune_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_revision = 9
snap_file = 'my-snap-name_0.1_amd64.snap'
# create dummy snap
open(os.path.join(self.path, snap_file), 'a').close()
# cache snap
snap_cache.cache(snap_file, snap_revision)
# create other cached snap revisions
to_be_deleted_files = []
cached_snaps = [
'a-cached-snap_0.3_amd64_8.snap',
'another-cached-snap_1.0_arm64_6.snap'
]
for cached_snap in cached_snaps:
cached_snap_path = os.path.join(snap_cache.snap_cache_dir,
cached_snap)
to_be_deleted_files.append(cached_snap_path)
open(cached_snap_path, 'a').close()
real_cached_snap = _rewrite_snap_filename_with_revision(
snap_file, snap_revision)
# confirm expected snap cached
self.assertEqual(3, len(os.listdir(snap_cache.snap_cache_dir)))
self.assertTrue(
os.path.isfile(
os.path.join(snap_cache.snap_cache_dir, real_cached_snap)))
if not self.valid_revision:
with self.assertRaises(ValueError):
snap_cache.prune(keep_revision='invalid-revision')
with self.assertRaises(TypeError):
snap_cache.prune(keep_revision=None)
else:
# prune cached snaps
purned_file_list = snap_cache.prune(keep_revision=snap_revision)
# confirm other snaps are purged
self.assertEqual(set(purned_file_list), set(to_be_deleted_files))
for snap in purned_file_list:
self.assertFalse(os.path.isfile(snap))
# confirm the expected cached file still exist
self.assertEqual(1, len(os.listdir(snap_cache.snap_cache_dir)))
self.assertTrue(
os.path.isfile(
os.path.join(snap_cache.snap_cache_dir, real_cached_snap)))
def test_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=self.snap_path)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root, 'amd64',
file_utils.calculate_sha3_384(self.snap_path))
self.assertEqual(expected_snap_path, cached_snap_path)
self.assertTrue(os.path.isfile(cached_snap_path))
def test_snap_cache_get_by_hash(self):
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_cache.cache(snap_filename=self.snap_path)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(self.snap_path)
# get snap by hash
snap = snap_cache.get(deb_arch="amd64", snap_hash=snap_hash)
self.assertThat(
snap, Equals(os.path.join(snap_cache.snap_cache_root, "amd64", snap_hash))
)
def test_snap_cache_get_by_hash(self):
self.useFixture(fixture_setup.FakeTerminal())
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=self.snap_path)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(self.snap_path)
# get snap by hash
snap = snap_cache.get(deb_arch='amd64', snap_hash=snap_hash)
self.assertEqual(
os.path.join(snap_cache.snap_cache_root, 'amd64', snap_hash), snap)
def test_snap_cache_get_latest(self):
# Create snaps
meta_dir = os.path.join(self.path, "prime", "meta")
os.makedirs(meta_dir)
with open(os.path.join(meta_dir, "snap.yaml"), "w") as f:
print(
dedent("""\
name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.1'
"""),
file=f,
)
result = self.run_command(["pack", os.path.join(self.path, "prime")])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob("*0.1*.snap")[0]
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(meta_dir, "snap.yaml"), "w") as f:
print(
dedent("""\
name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.2'
"""),
file=f,
)
result = self.run_command(["pack", os.path.join(self.path, "prime")])
self.assertThat(result.exit_code, Equals(0))
snap_file_latest = glob.glob("*0.2*.snap")[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch="all")
expected_snap_path = os.path.join(snap_cache.snap_cache_root, "all",
latest_hash)
self.assertThat(latest_snap, Equals(expected_snap_path))
def test_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
snap_cache = cache.SnapCache()
snap_file = 'my-snap-name_0.1_amd64.snap'
# create dummy snap
open(os.path.join(self.path, snap_file), 'a').close()
# cache snap
cached_snap_path = snap_cache.cache(snap_file, 10)
_, expected_snap = os.path.split(cached_snap_path)
self.assertEqual('my-snap-name_0.1_amd64_10.snap', expected_snap)
self.assertTrue(os.path.isfile(cached_snap_path))
def test_prune_snap_cache(self):
self.useFixture(
fixtures.MockPatchObject(cache.SnapCache,
"_get_snap_deb_arch",
return_value="amd64"))
# Create snaps
snap_file_1 = pathlib.Path("snap_01_amd64.snap")
snap_file_2 = pathlib.Path("snap_02_amd64.snap")
for snap_file in (snap_file_1, snap_file_2):
with snap_file.open("w") as f:
# Add whatever content
print(snap_file.as_posix(), file=f)
snap_cache = cache.SnapCache(project_name="snap")
snap_file_1_path = snap_cache.cache(snap_filename=snap_file_1)
_, snap_file_1_hash = os.path.split(snap_file_1_path)
snap_file_2_path = snap_cache.cache(snap_filename=snap_file_2)
snap_file_2_dir, snap_file_2_hash = os.path.split(snap_file_2_path)
# confirm expected snap cached
self.assertThat(len(os.listdir(snap_file_2_dir)), Equals(2))
# prune
pruned_files = snap_cache.prune(deb_arch=self.deb_arch,
keep_hash=snap_file_2_hash)
self.assertThat(len(pruned_files), Equals(1))
self.assertThat(
pruned_files,
Contains(
os.path.join(snap_cache.snap_cache_root, self.deb_arch,
snap_file_1_hash)),
)
self.assertThat(
pruned_files,
Not(
Contains(
os.path.join(snap_cache.snap_cache_root,
snap_file_2_hash))),
)
def test_snap_cache_get_latest_no_architectures(self):
# Create snaps
meta_dir = os.path.join(self.path, 'prime', 'meta')
os.makedirs(meta_dir)
with open(os.path.join(meta_dir, 'snap.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.1'
""")
result = self.run_command(['pack', os.path.join(self.path, 'prime')])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob('*0.1*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=snap_file)
with open(os.path.join(meta_dir, 'snap.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
confinement: devmode
grade: devel
version: '0.2'
""")
result = self.run_command(['pack', os.path.join(self.path, 'prime')])
self.assertThat(result.exit_code, Equals(0))
snap_file_latest = glob.glob('*0.2*.snap')[0]
snap_cache.cache(snap_filename=snap_file_latest)
latest_hash = file_utils.calculate_sha3_384(snap_file_latest)
# get latest
latest_snap = snap_cache.get(deb_arch='all')
expected_snap_path = os.path.join(snap_cache.snap_cache_root, 'all',
latest_hash)
self.assertThat(latest_snap, Equals(expected_snap_path))
def test_snap_cache_get_by_hash(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_cache.cache(snap_filename=snap_file)
# get hash of snap
snap_hash = file_utils.calculate_sha3_384(snap_file)
# get snap by hash
snap = snap_cache.get(deb_arch=self.deb_arch, snap_hash=snap_hash)
self.assertEqual(
os.path.join(snap_cache.snap_cache_root, self.deb_arch, snap_hash),
snap
)
def test_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create a snap
main(['init'])
main(['snap'])
snap_file = glob.glob('*.snap')[0]
snap_path = os.path.join(self.path, snap_file)
# cache snap
snap_cache = cache.SnapCache(project_name='cache-test')
cached_snap_path = snap_cache.cache(snap_filename=snap_file)
expected_snap_path = os.path.join(
snap_cache.snap_cache_root,
self.deb_arch,
file_utils.calculate_sha3_384(snap_path)
)
self.assertEqual(
expected_snap_path,
cached_snap_path
)
self.assertTrue(os.path.isfile(cached_snap_path))
def upload(snap_filename, release_channels=None) -> Tuple[str, int]:
"""Upload a snap_filename to the store.
If a cached snap is available, a delta will be generated from
the cached snap to the new target snap and uploaded instead. In the
case of a delta processing or upload failure, upload will fall back to
uploading the full snap.
If release_channels is defined it also releases it to those channels if the
store deems the uploaded snap as ready to release.
"""
snap_yaml = _get_data_from_snap_file(snap_filename)
snap_name = snap_yaml["name"]
built_at = snap_yaml.get("snapcraft-started-at")
logger.debug("Run upload precheck and verify cached data for {!r}.".format(
snap_filename))
store_client = StoreClientCLI()
store_client.upload_precheck(snap_name=snap_name)
snap_cache = cache.SnapCache(project_name=snap_name)
try:
deb_arch = snap_yaml["architectures"][0]
except KeyError:
deb_arch = "all"
source_snap = snap_cache.get(deb_arch=deb_arch)
sha3_384_available = hasattr(hashlib, "sha3_384")
result: Optional[Dict[str, Any]] = None
if sha3_384_available and source_snap:
try:
result = _upload_delta(
store_client,
snap_name=snap_name,
snap_filename=snap_filename,
source_snap=source_snap,
built_at=built_at,
channels=release_channels,
)
except storeapi.errors.StoreDeltaApplicationError as e:
logger.warning("Error generating delta: {}\n"
"Falling back to uploading full snap...".format(
str(e)))
except storeapi.errors.StoreUploadError as upload_error:
logger.warning("Unable to upload delta to store: {}\n"
"Falling back to uploading full snap...".format(
upload_error.error_list))
if result is None:
result = _upload_snap(
store_client,
snap_name=snap_name,
snap_filename=snap_filename,
built_at=built_at,
channels=release_channels,
)
snap_cache.cache(snap_filename=snap_filename)
snap_cache.prune(deb_arch=deb_arch,
keep_hash=calculate_sha3_384(snap_filename))
return snap_name, result["revision"]
def test_prune_snap_cache(self):
self.useFixture(fixture_setup.FakeTerminal())
# Create snaps
with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.1'
parts:
my-part:
plugin: nil
""".format(self.deb_arch))
main(['snap'])
snap_file = glob.glob('*0.1_*.snap')[0]
snap_cache = cache.SnapCache(project_name='my-snap-name')
snap_file_path = snap_cache.cache(snap_filename=snap_file)
_, snap_file_hash = os.path.split(snap_file_path)
with open(os.path.join(self.path, 'snapcraft.yaml'), 'w') as f:
f.write("""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.2'
parts:
my-part:
plugin: nil
""".format(self.deb_arch))
main(['snap'])
snap_file_2 = glob.glob('*0.2*.snap')[0]
snap_file_2_path = snap_cache.cache(snap_filename=snap_file_2)
snap_file_2_dir, snap_file_2_hash = os.path.split(snap_file_2_path)
# confirm expected snap cached
self.assertEqual(2, len(os.listdir(snap_file_2_dir)))
# prune
pruned_files = snap_cache.prune(deb_arch=self.deb_arch,
keep_hash=snap_file_2_hash)
self.assertEqual(1, len(pruned_files))
self.assertIn(
os.path.join(
snap_cache.snap_cache_root,
self.deb_arch,
snap_file_hash
),
pruned_files
)
self.assertNotIn(
os.path.join(snap_cache.snap_cache_root, snap_file_2_hash),
pruned_files
)
def test_prune_snap_cache(self):
# Create snaps
with open(os.path.join(self.path, "snapcraft.yaml"), "w") as f:
f.write(
"""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.1'
parts:
my-part:
plugin: nil
""".format(
self.deb_arch
)
)
result = self.run_command(["snap"])
self.assertThat(result.exit_code, Equals(0))
snap_file = glob.glob("*0.1_*.snap")[0]
snap_cache = cache.SnapCache(project_name="my-snap-name")
snap_file_path = snap_cache.cache(snap_filename=snap_file)
_, snap_file_hash = os.path.split(snap_file_path)
with open(os.path.join(self.path, "snapcraft.yaml"), "w") as f:
f.write(
"""name: my-snap-name
summary: test cached snap
description: test cached snap
architectures: ['{}']
confinement: devmode
grade: devel
version: '0.2'
parts:
my-part:
plugin: nil
""".format(
self.deb_arch
)
)
result = self.run_command(["snap"])
self.assertThat(result.exit_code, Equals(0))
snap_file_2 = glob.glob("*0.2*.snap")[0]
snap_file_2_path = snap_cache.cache(snap_filename=snap_file_2)
snap_file_2_dir, snap_file_2_hash = os.path.split(snap_file_2_path)
# confirm expected snap cached
self.assertThat(len(os.listdir(snap_file_2_dir)), Equals(2))
# prune
pruned_files = snap_cache.prune(
deb_arch=self.deb_arch, keep_hash=snap_file_2_hash
)
self.assertThat(len(pruned_files), Equals(1))
self.assertIn(
os.path.join(snap_cache.snap_cache_root, self.deb_arch, snap_file_hash),
pruned_files,
)
self.assertNotIn(
os.path.join(snap_cache.snap_cache_root, snap_file_2_hash), pruned_files
)
