def test__clears_component_error_when_successful(self):
register_persistent_error(COMPONENT.REGION_IMAGE_IMPORT,
factory.make_string())
[factory.make_BootSource(keyring_data=b'1234') for _ in range(3)]
download_image_descriptions = self.patch(
download_descriptions_module, 'download_image_descriptions')
# Make all of the downloads successful.
download_image_descriptions.return_value = BootImageMapping()
cache_boot_sources()
self.assertIsNone(get_persistent_error(COMPONENT.REGION_IMAGE_IMPORT))
def test_insert_item_validates(self):
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict)
item, _ = self.make_item(os='ubuntu')
self.patch(download_descriptions,
'products_exdata').return_value = item
dumper.insert_item({'os': 'ubuntu'}, sentinel.src, sentinel.target, (
factory.make_name('product_name'),
factory.make_name('product_version'),
), sentinel.contentsource)
self.assertItemsEqual([], list(boot_images_dict.mapping))
def test_adds_title_to_extra(self):
source = factory.make_BootSource(keyring_data=b"1234")
os = factory.make_name("os")
release = factory.make_name("release")
os_title = factory.make_name("os_title")
release_title = factory.make_name("release_title")
image_spec = make_image_spec(os=os, release=release)
image_mapping = BootImageMapping()
image_mapping.setdefault(image_spec, {
"os_title": os_title,
"release_title": release_title
})
mock_download = self.patch(bootsources,
"download_all_image_descriptions")
mock_download.return_value = image_mapping
cache_boot_sources()
cached = BootSourceCache.objects.filter(boot_source=source).first()
self.assertDictEqual({"title": "%s %s" % (os_title, release_title)},
cached.extra)
def test__adds_title_to_extra(self):
source = factory.make_BootSource(keyring_data=b'1234')
os = factory.make_name('os')
release = factory.make_name('release')
os_title = factory.make_name('os_title')
release_title = factory.make_name('release_title')
image_spec = make_image_spec(os=os, release=release)
image_mapping = BootImageMapping()
image_mapping.setdefault(image_spec, {
'os_title': os_title,
'release_title': release_title,
})
mock_download = self.patch(bootsources,
'download_all_image_descriptions')
mock_download.return_value = image_mapping
cache_boot_sources()
cached = BootSourceCache.objects.filter(boot_source=source).first()
self.assertDictEqual({'title': '%s %s' % (os_title, release_title)},
cached.extra)
def test__adds_release_to_cache(self):
source = factory.make_BootSource(keyring_data=b'1234')
os = factory.make_name('os')
release = factory.make_name('release')
release_codename = factory.make_name('codename')
release_title = factory.make_name('title')
support_eol = factory.make_date().strftime("%Y-%m-%d")
image_spec = make_image_spec(os=os, release=release)
image_mapping = BootImageMapping()
image_mapping.setdefault(
image_spec, {
"release_codename": release_codename,
"release_title": release_title,
"support_eol": support_eol,
})
bootsources._update_cache(source.to_dict_without_selections(),
image_mapping)
cached = BootSourceCache.objects.filter(boot_source=source).first()
self.assertEqual(release_codename, cached.release_codename)
self.assertEqual(release_title, cached.release_title)
self.assertEqual(support_eol, cached.support_eol.strftime("%Y-%m-%d"))
def download_all_image_descriptions(sources,
user_agent=None,
validate_products=True):
"""Download image metadata for all sources in `config`."""
boot = BootImageMapping()
for source in sources:
repo_boot = download_image_descriptions(
source['url'],
keyring=source.get('keyring', None),
user_agent=user_agent,
validate_products=validate_products)
boot_merge(boot, repo_boot, source['selections'])
return boot
def test__adds_release_codename_title_and_support_eol(self):
source = factory.make_BootSource(keyring_data=b'1234')
os = factory.make_name('os')
release = factory.make_name('release')
release_codename = factory.make_name('codename')
release_title = factory.make_name('title')
support_eol = factory.make_date().strftime("%Y-%m-%d")
image_spec = make_image_spec(os=os, release=release)
image_mapping = BootImageMapping()
image_mapping.setdefault(image_spec, {
"release_codename": release_codename,
"release_title": release_title,
"support_eol": support_eol,
})
mock_download = self.patch(
bootsources, 'download_all_image_descriptions')
mock_download.return_value = image_mapping
cache_boot_sources()
cached = BootSourceCache.objects.filter(boot_source=source).first()
self.assertEqual(release_codename, cached.release_codename)
self.assertEqual(release_title, cached.release_title)
self.assertEqual(support_eol, cached.support_eol.strftime("%Y-%m-%d"))
def test_sync_does_propagate_ioerror(self):
io_error = factory.make_exception_type(bases=(IOError, ))
mock_sync = self.patch(download_descriptions.BasicMirrorWriter, "sync")
mock_sync.side_effect = io_error()
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict)
with FakeLogger("maas.import-images", level=logging.INFO) as maaslog:
self.assertRaises(io_error, dumper.sync, sentinel.reader,
sentinel.path)
self.assertDocTestMatches("...error...syncing boot images...",
maaslog.output)
def test_dump_json_combines_similar_entries(self):
image = make_image_spec()
other_release = factory.make_name('other-release')
resource1 = factory.make_name('resource')
resource2 = factory.make_name('other-resource')
image_dict = BootImageMapping()
set_resource(image_dict, image, resource1)
set_resource(
image_dict, image._replace(release=other_release), resource2)
self.assertEqual(
{
image.os: {
image.arch: {
image.subarch: {
image.kflavor: {
image.release: {image.label: resource1},
other_release: {image.label: resource2},
},
},
},
},
},
json.loads(image_dict.dump_json()))
def test_obeys_filters(self):
filters = [{
"os": factory.make_name("os"),
"arches": [factory.make_name("other-arch")],
"subarches": [factory.make_name("other-subarch")],
"release": factory.make_name("other-release"),
"label": [factory.make_name("other-label")],
}]
total_resources = BootImageMapping()
resources_from_repo = set_resource()
download_descriptions.boot_merge(total_resources,
resources_from_repo,
filters=filters)
self.assertEqual({}, total_resources.mapping)
def test_insert_item_sets_generic_to_release_item_for_hwe_version(self):
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict)
os = "ubuntu"
release = "xenial"
arch = "amd64"
label = "release"
hwep_subarch = "hwe-16.04"
hwep_subarches = ["generic", "hwe-16.04", "hwe-16.10"]
hwes_subarch = "hwe-16.10"
hwes_subarches = ["generic", "hwe-16.04", "hwe-16.10"]
hwep_item, compat_item = self.make_item(
os=os,
release=release,
arch=arch,
subarch=hwep_subarch,
subarches=hwep_subarches,
label=label,
)
hwes_item, _ = self.make_item(
os=os,
release=release,
arch=arch,
subarch=hwes_subarch,
subarches=hwes_subarches,
label=label,
)
self.patch(download_descriptions, "products_exdata").side_effect = [
hwep_item,
hwes_item,
]
for _ in range(2):
dumper.insert_item(
{"os": "ubuntu"},
sentinel.src,
sentinel.target,
(
"com.ubuntu.maas.daily:v3:boot:12.04:amd64:hwe-p",
factory.make_name("product_version"),
),
sentinel.contentsource,
)
image_spec = make_image_spec(os=os,
release=release,
arch=arch,
subarch="generic",
label=label)
self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
def test_obeys_filters(self):
filters = [
{
'os': factory.make_name('os'),
'arches': [factory.make_name('other-arch')],
'subarches': [factory.make_name('other-subarch')],
'release': factory.make_name('other-release'),
'label': [factory.make_name('other-label')],
},
]
total_resources = BootImageMapping()
resources_from_repo = set_resource()
download_descriptions.boot_merge(total_resources,
resources_from_repo,
filters=filters)
self.assertEqual({}, total_resources.mapping)
def test_insert_item_adds_item_per_subarch(self):
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict)
subarches = [factory.make_name('subarch') for _ in range(3)]
item, _ = self.make_item(subarch=subarches.pop(), subarches=subarches)
self.patch(download_descriptions,
'products_exdata').return_value = item
dumper.insert_item(sentinel.data, sentinel.src, sentinel.target,
(factory.make_name('product_name'),
factory.make_name('product_version')),
sentinel.contentsource)
image_specs = [
make_image_spec(os=item['os'],
release=item['release'],
arch=item['arch'],
subarch=subarch,
label=item['label']) for subarch in subarches
]
self.assertItemsEqual(image_specs, list(boot_images_dict.mapping))
def test_insert_item_doesnt_validate_when_instructed(self):
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict, validate_products=False)
item, _ = self.make_item(os='ubuntu')
self.patch(download_descriptions,
'products_exdata').return_value = item
dumper.insert_item({'os': 'ubuntu'}, sentinel.src, sentinel.target, (
factory.make_name('product_name'),
factory.make_name('product_version'),
), sentinel.contentsource)
image_specs = [
make_image_spec(os=item['os'],
release=item['release'],
arch=item['arch'],
subarch=subarch,
label=item['label'])
for subarch in item['subarches'].split(',')
]
self.assertItemsEqual(image_specs, list(boot_images_dict.mapping))
def test_concatenates_similar_resources(self):
image1 = make_image_spec()
image2 = make_image_spec()
resource = make_boot_resource()
boot_dict = BootImageMapping()
# Create two images in boot_dict, both containing the same resource.
for image in [image1, image2]:
set_resource(
boot_dict=boot_dict, resource=resource.copy(), image_spec=image
)
products_mapping = map_products(boot_dict)
key = (
resource["content_id"],
resource["product_name"],
resource["version_name"],
)
self.assertEqual([key], list(products_mapping.mapping))
self.assertItemsEqual(
[image1.subarch, image2.subarch], products_mapping.get(resource)
)
def test__consistent_query_count(self):
source = factory.make_BootSource(keyring_data=b'1234')
image_mapping = BootImageMapping()
for _ in range(random.randint(20, 50)):
self.make_release(image_mapping)
# Add all the items to the cache, always 5.
queries, _ = count_queries(bootsources._update_cache,
source.to_dict_without_selections(),
image_mapping)
self.assertEquals(5, queries)
# Now that they all already exist, it should only be 4 queries.
queries, _ = count_queries(bootsources._update_cache,
source.to_dict_without_selections(),
image_mapping)
self.assertEquals(4, queries)
# Do it again just to be sure.
queries, _ = count_queries(bootsources._update_cache,
source.to_dict_without_selections(),
image_mapping)
self.assertEquals(4, queries)
def test__catches_connection_errors_and_sets_component_error(self):
sources = [
factory.make_BootSource(keyring_data=b'1234') for _ in range(3)]
download_image_descriptions = self.patch(
download_descriptions_module, 'download_image_descriptions')
error_text_one = factory.make_name("<error1>")
error_text_two = factory.make_name("<error2>")
# Make two of the downloads fail.
download_image_descriptions.side_effect = [
ConnectionError(error_text_one),
BootImageMapping(),
IOError(error_text_two),
]
cache_boot_sources()
base_error = "Failed to import images from boot source {url}: {err}"
error_part_one = base_error.format(
url=sources[0].url, err=html.escape(error_text_one))
error_part_two = base_error.format(
url=sources[2].url, err=html.escape(error_text_two))
expected_error = error_part_one + '<br>' + error_part_two
actual_error = get_persistent_error(COMPONENT.REGION_IMAGE_IMPORT)
self.assertEqual(expected_error, actual_error)
def test_insert_item_sets_compat_item_specific_to_subarch(self):
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict)
subarches = [factory.make_name("subarch") for _ in range(5)]
compat_subarch = subarches.pop()
item, _ = self.make_item(subarch=subarches.pop(), subarches=subarches)
second_item, compat_item = self.make_item(
os=item["os"],
release=item["release"],
arch=item["arch"],
subarch=compat_subarch,
subarches=[compat_subarch],
label=item["label"],
)
self.patch(download_descriptions, "products_exdata").side_effect = [
item,
second_item,
]
for _ in range(2):
dumper.insert_item(
sentinel.data,
sentinel.src,
sentinel.target,
(
factory.make_name("product_name"),
factory.make_name("product_version"),
),
sentinel.contentsource,
)
image_spec = make_image_spec(
os=item["os"],
release=item["release"],
arch=item["arch"],
subarch=compat_subarch,
label=item["label"],
)
self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
def download_image_descriptions(path,
keyring=None,
user_agent=None,
validate_products=True):
"""Download image metadata from upstream Simplestreams repo.
:param path: The path to a Simplestreams repo.
:param keyring: Optional keyring for verifying the repo's signatures.
:param user_agent: Optional user agent string for downloading the image
descriptions.
:return: A `BootImageMapping` describing available boot resources.
"""
maaslog.info("Downloading image descriptions from %s", path)
mirror, rpath = path_from_mirror_url(path, None)
policy = get_signing_policy(rpath, keyring)
if user_agent is None:
# If user_agent is NOT set, we know we are downloading descriptions
# from the Region controller *by* the Rack controller.
maaslog.info("Rack downloading image descriptions from '%s'.", path)
reader = UrlMirrorReader(mirror, policy=policy)
else:
# Since user_agent is set, we know we are downloading descriptions
# from the Images repository *by* the Region.
maaslog.info("Region downloading image descriptions from '%s'.", path)
try:
reader = UrlMirrorReader(mirror,
policy=policy,
user_agent=user_agent)
except TypeError:
# UrlMirrorReader doesn't support the user_agent argument.
# simplestream >=bzr429 is required for this feature.
reader = UrlMirrorReader(mirror, policy=policy)
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict, validate_products=validate_products)
dumper.sync(reader, rpath)
return boot_images_dict
def test_insert_item_sets_generic_to_release_item_for_hwe_version(self):
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict)
os = 'ubuntu'
release = 'xenial'
arch = 'amd64'
label = 'release'
hwep_subarch = 'hwe-16.04'
hwep_subarches = ['generic', 'hwe-16.04', 'hwe-16.10']
hwes_subarch = 'hwe-16.10'
hwes_subarches = ['generic', 'hwe-16.04', 'hwe-16.10']
hwep_item, compat_item = self.make_item(os=os,
release=release,
arch=arch,
subarch=hwep_subarch,
subarches=hwep_subarches,
label=label)
hwes_item, _ = self.make_item(os=os,
release=release,
arch=arch,
subarch=hwes_subarch,
subarches=hwes_subarches,
label=label)
self.patch(download_descriptions,
'products_exdata').side_effect = [hwep_item, hwes_item]
for _ in range(2):
dumper.insert_item(
{'os': 'ubuntu'}, sentinel.src, sentinel.target, (
'com.ubuntu.maas.daily:v3:boot:12.04:amd64:hwe-p',
factory.make_name('product_version'),
), sentinel.contentsource)
image_spec = make_image_spec(os=os,
release=release,
arch=arch,
subarch='generic',
label=label)
self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
def test_is_empty_returns_False_if_not_empty(self):
mapping = BootImageMapping()
mapping.setdefault(make_image_spec(), factory.make_name("resource"))
self.assertFalse(mapping.is_empty())
def test_load_json_returns_empty_mapping_for_invalid_json(self):
bad_json = ""
mapping = BootImageMapping.load_json(bad_json)
self.assertEqual({}, mapping.mapping)
def test_initially_empty(self):
self.assertItemsEqual([], BootImageMapping().items())
def test_is_empty_returns_True_if_empty(self):
self.assertTrue(BootImageMapping().is_empty())
def test_maps_empty_dict_to_empty_dict(self):
empty_boot_image_dict = BootImageMapping()
self.assertEqual({}, map_products(empty_boot_image_dict).mapping)
def test_extract_ephemeral_image_params_with_metadata(self):
path, osystem, arch, subarch, release, label = self._make_path()
# Patch OperatingSystemRegistry to return a fixed list of
# values.
purpose1 = factory.make_name("purpose")
purpose2 = factory.make_name("purpose")
xi_purpose = "ephemeral"
xi_path = factory.make_name("xi_path")
xi_type = factory.make_name("xi_type")
purposes = [purpose1, purpose2, xi_purpose]
self._patch_osystem_registry(purposes,
xinstall_params=(xi_path, xi_type))
# Create some maas.meta content.
image = ImageSpec(os=osystem,
arch=arch,
subarch=subarch,
kflavor='generic',
release=release,
label=label)
image_resource = dict(subarches=factory.make_name("subarches"))
mapping = BootImageMapping()
mapping.setdefault(image, image_resource)
maas_meta = mapping.dump_json()
params = extract_image_params(path, maas_meta)
self.assertItemsEqual([
{
"osystem": osystem,
"architecture": arch,
"subarchitecture": subarch,
"release": release,
"label": label,
"purpose": purpose1,
"xinstall_path": '',
"xinstall_type": '',
"supported_subarches": image_resource["subarches"],
},
{
"osystem": osystem,
"architecture": arch,
"subarchitecture": subarch,
"release": release,
"label": label,
"purpose": purpose2,
"xinstall_path": '',
"xinstall_type": '',
"supported_subarches": image_resource["subarches"],
},
{
"osystem": osystem,
"architecture": arch,
"subarchitecture": subarch,
"release": release,
"label": label,
"purpose": xi_purpose,
"xinstall_path": xi_path,
"xinstall_type": xi_type,
"supported_subarches": image_resource["subarches"],
},
], params)
def test_load_json_result_of_old_data_uses_ubuntu_as_os(self):
test_meta_file_content = make_maas_meta_without_os()
mapping = BootImageMapping.load_json(test_meta_file_content)
os = {image.os for image, _ in mapping.items()}.pop()
self.assertEqual("ubuntu", os)
def test_dump_json_represents_empty_dict_as_empty_object(self):
self.assertEqual("{}", BootImageMapping().dump_json())
def make_boot_image_mapping(image_specs=None):
mapping = BootImageMapping()
if image_specs is not None:
for image_spec in image_specs:
mapping.setdefault(image_spec, {})
return mapping
def test_setdefault_sets_unset_item(self):
image_dict = BootImageMapping()
image = make_image_spec()
resource = factory.make_name("resource")
image_dict.setdefault(image, resource)
self.assertItemsEqual([(image, resource)], image_dict.items())
