def test_redirect(self):
test_tree = tree.Tree(self.config)
# Create some channels and aliases
test_tree.create_channel("parent")
test_tree.create_channel_redirect("redirect", "parent")
# Test standard failure cases
self.assertRaises(KeyError, test_tree.create_channel_redirect,
"redirect", "parent")
self.assertRaises(KeyError, test_tree.create_channel_redirect,
"redirect1", "parent1")
self.assertRaises(KeyError, test_tree.sync_redirects, "parent1")
# Publish a basic image
test_tree.create_device("parent", "device")
# # First file
first = os.path.join(self.config.publish_path, "parent/device/full")
open(first, "w+").close()
gpg.sign_file(self.config, "image-signing", first)
# # Second file
second = os.path.join(self.config.publish_path,
"parent/device/version-1234.tar.xz")
tools.generate_version_tarball(self.config, "parent", "test", "1234",
second.replace(".xz", ""))
tools.xz_compress(second.replace(".xz", ""))
os.remove(second.replace(".xz", ""))
gpg.sign_file(self.config, "image-signing", second)
with open(second.replace(".tar.xz", ".json"), "w+") as fd:
metadata = {}
metadata['channel.ini'] = {}
metadata['channel.ini']['version_detail'] = "test"
fd.write(json.dumps(metadata))
gpg.sign_file(self.config, "image-signing",
second.replace(".tar.xz", ".json"))
# # Adding the entry
device = test_tree.get_device("parent", "device")
device.create_image(
"full", 1234, "abc",
["parent/device/full", "parent/device/version-1234.tar.xz"])
device.set_phased_percentage(1234, 50)
# # Sync the redirects
test_tree.sync_redirects("parent")
# # Get the target
target = test_tree.get_device("redirect", "device")
# Confirm the fs layout
self.assertTrue(not os.path.exists(
os.path.join(self.config.publish_path, "redirect")))
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "parent", "device")))
self.assertEqual(device.list_images(), target.list_images())
def sync_alias(self, channel_name):
"""
Update a channel with data from its parent.
"""
with channels_json(self.config, self.indexpath) as channels:
if channel_name not in channels:
raise KeyError("Couldn't find channel: %s" % channel_name)
if "alias" not in channels[channel_name] or \
channels[channel_name]['alias'] == channel_name:
raise TypeError("Not a channel alias")
target_name = channels[channel_name]['alias']
if target_name not in channels:
raise KeyError("Couldn't find target channel: %s" %
target_name)
# Start by looking for added/removed devices
devices = set(channels[channel_name]['devices'].keys())
target_devices = set(channels[target_name]['devices'].keys())
# # Remove any removed device
for device in devices - target_devices:
self.remove_device(channel_name, device)
# # Add any missing device
for device in target_devices - devices:
self.create_device(channel_name, device)
# Iterate through all the devices to import builds
for device_name in target_devices:
device = self.get_device(channel_name, device_name)
target_device = self.get_device(target_name, device_name)
# Extract all the current builds
device_images = {(image['version'], image.get("base", None),
image['type'])
for image in device.list_images()}
target_images = {(image['version'], image.get("base", None),
image['type'])
for image in target_device.list_images()}
# Remove any removed image
for image in device_images - target_images:
device.remove_image(image[2], image[0], base=image[1])
# Create the path if it doesn't exist
if not os.path.exists(device.path):
os.makedirs(device.path)
# Add any missing image
with index_json(self.config, device.indexpath, True) as index:
for image in sorted(target_images - device_images):
orig = [entry for entry in target_device.list_images()
if entry['type'] == image[2] and
entry['version'] == image[0] and
entry.get("base", None) == image[1]]
entry = copy.deepcopy(orig[0])
# Remove the current version tarball
version_detail = None
version_index = len(entry['files'])
for fentry in entry['files']:
if fentry['path'].endswith("version-%s.tar.xz" %
entry['version']):
version_path = "%s/%s" % (
self.config.publish_path, fentry['path'])
if os.path.exists(
version_path.replace(".tar.xz",
".json")):
with open(
version_path.replace(
".tar.xz", ".json")) as fd:
metadata = json.loads(fd.read())
if "channel.ini" in metadata:
version_detail = \
metadata['channel.ini'].get(
"version_detail", None)
version_index = fentry['order']
entry['files'].remove(fentry)
break
# Generate a new one
path = os.path.join(device.path,
"version-%s.tar.xz" %
entry['version'])
abspath, relpath = tools.expand_path(path,
device.pub_path)
if not os.path.exists(abspath):
tools.generate_version_tarball(
self.config, channel_name, device_name,
str(entry['version']),
abspath.replace(".xz", ""),
version_detail=version_detail,
channel_target=target_name)
tools.xz_compress(abspath.replace(".xz", ""))
os.remove(abspath.replace(".xz", ""))
gpg.sign_file(self.config, "image-signing",
abspath)
with open(abspath, "rb") as fd:
checksum = sha256(fd.read()).hexdigest()
# Generate the new file entry
version = {}
version['order'] = version_index
version['path'] = "/%s" % "/".join(
relpath.split(os.sep))
version['signature'] = "/%s.asc" % "/".join(
relpath.split(os.sep))
version['checksum'] = checksum
version['size'] = int(os.stat(abspath).st_size)
# And add it
entry['files'].append(version)
index['images'].append(entry)
# Sync phased-percentage
versions = sorted({entry[0] for entry in target_images})
if versions:
device.set_phased_percentage(
versions[-1],
target_device.get_phased_percentage(versions[-1]))
return True
def test_rename(self):
test_tree = tree.Tree(self.config)
# Create some channels and aliases
test_tree.create_channel("old")
test_tree.create_channel("existing")
# Test standard failure cases
self.assertRaises(KeyError, test_tree.rename_channel, "old1",
"test/new")
self.assertRaises(KeyError, test_tree.rename_channel, "old",
"existing")
# Publish a basic image
test_tree.create_device("old", "device")
# # First file
first = os.path.join(self.config.publish_path, "old/device/full")
open(first, "w+").close()
gpg.sign_file(self.config, "image-signing", first)
# # Second file
second = os.path.join(self.config.publish_path,
"old/device/version-1234.tar.xz")
tools.generate_version_tarball(self.config, "old", "test", "1234",
second.replace(".xz", ""))
tools.xz_compress(second.replace(".xz", ""))
os.remove(second.replace(".xz", ""))
gpg.sign_file(self.config, "image-signing", second)
with open(second.replace(".tar.xz", ".json"), "w+") as fd:
metadata = {}
metadata['channel.ini'] = {}
metadata['channel.ini']['version_detail'] = "test"
fd.write(json.dumps(metadata))
gpg.sign_file(self.config, "image-signing",
second.replace(".tar.xz", ".json"))
# # Adding the entry
device = test_tree.get_device("old", "device")
device.create_image(
"full", 1234, "abc",
["old/device/full", "old/device/version-1234.tar.xz"])
device.set_phased_percentage(1234, 50)
# Rename
os.makedirs(os.path.join(self.config.publish_path, "test/new"))
self.assertRaises(Exception, test_tree.rename_channel, "old",
"test/new")
os.rmdir(os.path.join(self.config.publish_path, "test/new"))
os.rmdir(os.path.join(self.config.publish_path, "test"))
self.assertTrue(test_tree.rename_channel("old", "test/new"))
self.assertEqual(
test_tree.list_channels()['test/new'],
{'devices': {
'device': {
'index': '/test/new/device/index.json'
}
}})
def test_channels(self):
# Test getting a tree instance
test_tree = tree.Tree(self.config)
self.assertEqual(test_tree.list_channels(), {})
# Test publishing a keyring
keyring = gpg.Keyring(self.config, "image-signing")
keyring.set_metadata("image-signing")
keyring.import_keys(
os.path.join(self.config.gpg_key_path, "image-signing"))
self.assertRaises(Exception, test_tree.publish_keyring,
"image-signing")
keyring_tar = keyring.generate_tarball()
tools.xz_compress(keyring_tar)
self.assertRaises(Exception, test_tree.publish_keyring,
"image-signing")
gpg.sign_file(self.config, "image-master", "%s.xz" % keyring_tar)
test_tree.publish_keyring("image-signing")
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "gpg", "image-signing"
".tar.xz")))
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "gpg", "image-signing"
".tar.xz.asc")))
# Test invalid tree path
self.assertRaises(Exception, tree.Tree, self.config,
os.path.join(self.temp_directory, "invalid"))
# Test channel creation
test_tree.create_channel("first")
test_tree.create_channel("second")
self.assertRaises(KeyError, test_tree.create_channel, "second")
# Test channel removal
test_tree.remove_channel("first")
test_tree.remove_channel("second")
self.assertRaises(KeyError, test_tree.remove_channel, "second")
# Test invalid json
with open(test_tree.indexpath, "w+") as fd:
fd.write("[]")
self.assertRaises(TypeError, test_tree.list_channels)
with open(test_tree.indexpath, "w+") as fd:
fd.write("{'a': 'a'}")
self.assertRaises(ValueError, test_tree.list_channels)
os.remove(test_tree.indexpath)
# Test hidding a channel
test_tree.create_channel("testing")
self.assertEqual(test_tree.list_channels(),
{'testing': {
'devices': {}
}})
test_tree.hide_channel("testing")
self.assertEqual(test_tree.list_channels(),
{'testing': {
'devices': {},
'hidden': True
}})
test_tree.show_channel("testing")
self.assertEqual(test_tree.list_channels(),
{'testing': {
'devices': {}
}})
self.assertRaises(KeyError, test_tree.hide_channel, "invalid")
self.assertRaises(KeyError, test_tree.show_channel, "invalid")
test_tree.remove_channel("testing")
# Test device creation
test_tree.create_channel("testing")
test_tree.create_device("testing", "test")
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "testing", "test",
"index.json")))
self.assertEqual(
test_tree.list_channels(), {
'testing': {
'devices': {
'test': {
'index': '/testing/test/'
'index.json'
}
}
}
})
self.assertRaises(KeyError, test_tree.create_device, "invalid", "test")
self.assertRaises(KeyError, test_tree.create_device, "testing", "test")
# Test listing devices
self.assertRaises(KeyError, test_tree.list_devices, "invalid")
self.assertEqual(test_tree.list_devices("testing"), ["test"])
# Test the index generation
os.mkdir(os.path.join(self.config.publish_path, "testing", "empty"))
self.assertRaises(Exception, test_tree.generate_index)
test_tree.generate_index("I know what I'm doing")
self.assertEqual(
test_tree.list_channels(), {
'testing': {
'devices': {
'test': {
'index': '/testing/test/'
'index.json'
}
}
}
})
device_keyring = os.path.join(self.config.publish_path, "testing",
"test", "device.tar.xz")
open(device_keyring, "w+").close()
test_tree.generate_index("I know what I'm doing")
self.assertEqual(
test_tree.list_channels(), {
'testing': {
'devices': {
'test': {
'index': '/testing/test/'
'index.json'
}
}
}
})
gpg.sign_file(self.config, "image-signing", device_keyring)
test_tree.generate_index("I know what I'm doing")
self.assertEqual(
test_tree.list_channels(), {
'testing': {
'devices': {
'test': {
'index': '/testing/test/index.json',
'keyring': {
'path':
'/testing/test/'
'device.tar.xz',
'signature':
'/testing/test/'
'device.tar'
'.xz.asc'
}
}
}
}
})
# Test grabbing a device entry
self.assertRaises(KeyError, test_tree.get_device, "invalid", "test")
self.assertRaises(KeyError, test_tree.get_device, "testing", "invalid")
self.assertTrue(
isinstance(test_tree.get_device("testing", "test"), tree.Device))
# Test device removal
test_tree.create_device("testing", "to-remove")
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "testing", "to-remove",
"index.json")))
self.assertRaises(KeyError, test_tree.remove_device, "invalid", "test")
self.assertRaises(KeyError, test_tree.remove_device, "testing",
"invalid")
test_tree.remove_device("testing", "to-remove")
self.assertFalse(
os.path.exists(
os.path.join(self.config.publish_path, "testing", "to-remove",
"index.json")))
self.assertEqual(
test_tree.list_channels(), {
'testing': {
'devices': {
'test': {
'index': '/testing/test/index.json',
'keyring': {
'path': '/testing/test/device'
'.tar.xz',
'signature': '/testing/test/devi'
'ce.tar.xz.asc'
}
}
}
}
})
# Test setting the device keyring
self.assertRaises(KeyError, test_tree.set_device_keyring, "invalid",
"test", "invalid")
self.assertRaises(KeyError, test_tree.set_device_keyring, "testing",
"invalid", "invalid")
test_tree.set_device_keyring("testing", "test",
"testing/test/device.tar.xz")
self.assertRaises(Exception, test_tree.set_device_keyring, "testing",
"test", "invalid")
unsigned_path = os.path.join(self.config.publish_path, "unsigned")
open(unsigned_path, "w+").close()
self.assertRaises(Exception, test_tree.set_device_keyring, "testing",
"test", "unsigned")
os.remove(unsigned_path)
def test_per_device_redirect(self):
""" """
test_tree = tree.Tree(self.config)
# Create some channels
test_tree.create_channel("parent")
test_tree.create_channel("redirect")
# Create the required devices
test_tree.create_device("parent", "device")
test_tree.create_device("parent", "other")
test_tree.create_device("redirect", "other")
# Test standard failure cases
self.assertRaises(KeyError,
test_tree.create_per_device_channel_redirect,
"device", "redirect1", "parent")
self.assertRaises(KeyError,
test_tree.create_per_device_channel_redirect,
"other", "redirect", "parent")
self.assertRaises(KeyError,
test_tree.create_per_device_channel_redirect,
"device", "redirect", "parent1")
self.assertRaises(KeyError,
test_tree.create_per_device_channel_redirect,
"device2", "redirect", "parent")
# Create the device channel redirect
test_tree.create_per_device_channel_redirect("device", "redirect",
"parent")
# Publish an image
# # First file
first = os.path.join(self.config.publish_path, "parent/device/full")
open(first, "w+").close()
gpg.sign_file(self.config, "image-signing", first)
# # Second file
second = os.path.join(self.config.publish_path,
"parent/device/version-1234.tar.xz")
tools.generate_version_tarball(self.config, "parent", "test", "1234",
second.replace(".xz", ""))
tools.xz_compress(second.replace(".xz", ""))
os.remove(second.replace(".xz", ""))
gpg.sign_file(self.config, "image-signing", second)
with open(second.replace(".tar.xz", ".json"), "w+") as fd:
metadata = {}
metadata['channel.ini'] = {}
metadata['channel.ini']['version_detail'] = "test"
fd.write(json.dumps(metadata))
gpg.sign_file(self.config, "image-signing",
second.replace(".tar.xz", ".json"))
# # Adding the entry
device = test_tree.get_device("parent", "device")
device.create_image(
"full", 1234, "abc",
["parent/device/full", "parent/device/version-1234.tar.xz"])
device.set_phased_percentage(1234, 50)
# # Get the target
target = test_tree.get_device("redirect", "device")
# Confirm the fs layout
self.assertTrue(
os.path.exists(os.path.join(self.config.publish_path, "redirect")))
self.assertFalse(
os.path.exists(
os.path.join(self.config.publish_path, "redirect", "device")))
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "parent", "device")))
self.assertEqual(device.list_images(), target.list_images())
# Check the channels index
channels = test_tree.list_channels()
self.assertIn("other", channels['redirect']['devices'])
self.assertEqual(channels['redirect']['devices']['other']['index'],
"/redirect/other/index.json")
self.assertIn("device", channels['redirect']['devices'])
self.assertEqual(channels['redirect']['devices']['device']['index'],
"/parent/device/index.json")
self.assertIn("redirect", channels['redirect']['devices']['device'])
# Try removing a per-channel redirect
self.assertTrue(test_tree.remove_device("redirect", "device"))
# Confirm files are not removed
self.assertTrue(
os.path.exists(
os.path.join(self.config.publish_path, "parent", "device",
"index.json")))
def test_alias(self):
test_tree = tree.Tree(self.config)
# Create some channels and aliases
test_tree.create_channel("parent")
test_tree.create_channel_alias("alias", "parent")
test_tree.create_channel_redirect("redirect", "alias")
# Test standard failure cases
self.assertRaises(KeyError, test_tree.create_channel_alias, "alias",
"parent")
self.assertRaises(KeyError, test_tree.create_channel_alias, "alias1",
"parent1")
self.assertRaises(KeyError, test_tree.change_channel_alias, "alias1",
"parent")
self.assertRaises(KeyError, test_tree.change_channel_alias, "alias",
"parent1")
self.assertRaises(KeyError, test_tree.change_channel_alias, "parent",
"parent")
self.assertRaises(KeyError, test_tree.change_channel_alias, "redirect",
"parent")
self.assertRaises(KeyError, test_tree.sync_aliases, "missing")
self.assertRaises(KeyError, test_tree.sync_alias, "missing")
self.assertRaises(TypeError, test_tree.sync_alias, "parent")
test_tree.remove_channel("parent")
self.assertRaises(KeyError, test_tree.sync_alias, "alias")
test_tree.create_channel("parent")
# Publish a basic image
test_tree.create_device("parent", "device")
test_tree.create_device("parent", "device2")
test_tree.create_device("parent", "device3")
test_tree.create_device("alias", "device1")
# # First file
first = os.path.join(self.config.publish_path, "parent/device/full")
open(first, "w+").close()
gpg.sign_file(self.config, "image-signing", first)
# # Second file
second = os.path.join(self.config.publish_path,
"parent/device/version-1234.tar.xz")
tools.generate_version_tarball(self.config, "parent", "test", "1234",
second.replace(".xz", ""))
tools.xz_compress(second.replace(".xz", ""))
os.remove(second.replace(".xz", ""))
gpg.sign_file(self.config, "image-signing", second)
with open(second.replace(".tar.xz", ".json"), "w+") as fd:
metadata = {}
metadata['channel.ini'] = {}
metadata['channel.ini']['version_detail'] = "test"
fd.write(json.dumps(metadata))
gpg.sign_file(self.config, "image-signing",
second.replace(".tar.xz", ".json"))
# # Adding the entry
device = test_tree.get_device("parent", "device")
device.create_image(
"full", 1234, "abc",
["parent/device/full", "parent/device/version-1234.tar.xz"])
device.set_phased_percentage(1234, 50)
# # Adding a fake entry to the alias channel
device = test_tree.get_device("alias", "device")
device.create_image(
"full", 1235, "abc",
["parent/device/full", "parent/device/version-1234.tar.xz"])
# Sync the alises
device3 = test_tree.get_device("alias", "device3")
shutil.rmtree(device3.path)
test_tree.sync_aliases("parent")
alias_device = test_tree.get_device("alias", "device")
self.assertEqual(alias_device.get_phased_percentage(1234), 50)
test_tree.create_channel("new_parent")
test_tree.change_channel_alias("alias", "new_parent")
test_tree.remove_channel("alias")
test_tree.remove_channel("new_parent")
test_tree.remove_channel("parent")
def test_generate_delta(self):
# Source tarball
source_path = os.path.join(self.temp_directory, "source.tar")
source_path_xz = "%s.xz" % source_path
source_tar = tarfile.open(source_path, "w")
source_tar.close()
tools.xz_compress(source_path)
os.remove(source_path)
# Source json
with open(os.path.join(self.temp_directory, "source.json"),
"w+") as fd:
source_json = {}
source_json['a'] = 1
fd.write(json.dumps(source_json))
# Destination tarball
destination_path = os.path.join(self.temp_directory, "destination.tar")
destination_path_xz = "%s.xz" % destination_path
destination_tar = tarfile.open(destination_path, "w")
destination_tar.close()
tools.xz_compress(destination_path)
os.remove(destination_path)
# Destination json
with open(os.path.join(self.temp_directory, "destination.json"),
"w+") as fd:
source_json = {}
source_json['b'] = 2
fd.write(json.dumps(source_json))
# Check that version tarballs are just returned
open(os.path.join(self.temp_directory,
"version-1.tar.xz"), "w+").close()
open(os.path.join(self.temp_directory,
"version-2.tar.xz"), "w+").close()
self.assertEqual(
generators.generate_delta(
self.config,
os.path.join(self.temp_directory, "version-1.tar.xz"),
os.path.join(self.temp_directory, "version-2.tar.xz")),
os.path.join(self.temp_directory, "version-2.tar.xz"))
# Check that keyring tarballs are just returned
open(os.path.join(self.temp_directory,
"keyring-1.tar.xz"), "w+").close()
self.assertEqual(
generators.generate_delta(
self.config,
os.path.join(self.temp_directory, "keyring-1.tar.xz"),
os.path.join(self.temp_directory, "keyring-1.tar.xz")),
os.path.join(self.temp_directory, "keyring-1.tar.xz"))
# Generate the diff
self.assertEqual(
generators.generate_delta(self.config, source_path_xz,
destination_path_xz),
os.path.join(self.config.publish_path, "pool",
"destination.delta-source.tar.xz"))
# Check that we get cached entries
generators.generate_delta(self.config, source_path_xz,
destination_path_xz)