def test__clears_component_error_when_successful(self): register_persistent_error(COMPONENT.REGION_IMAGE_IMPORT, factory.make_string()) [factory.make_BootSource(keyring_data=b'1234') for _ in range(3)] download_image_descriptions = self.patch( download_descriptions_module, 'download_image_descriptions') # Make all of the downloads successful. download_image_descriptions.return_value = BootImageMapping() cache_boot_sources() self.assertIsNone(get_persistent_error(COMPONENT.REGION_IMAGE_IMPORT))
def test_insert_item_validates(self): boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict) item, _ = self.make_item(os='ubuntu') self.patch(download_descriptions, 'products_exdata').return_value = item dumper.insert_item({'os': 'ubuntu'}, sentinel.src, sentinel.target, ( factory.make_name('product_name'), factory.make_name('product_version'), ), sentinel.contentsource) self.assertItemsEqual([], list(boot_images_dict.mapping))
def test_adds_title_to_extra(self): source = factory.make_BootSource(keyring_data=b"1234") os = factory.make_name("os") release = factory.make_name("release") os_title = factory.make_name("os_title") release_title = factory.make_name("release_title") image_spec = make_image_spec(os=os, release=release) image_mapping = BootImageMapping() image_mapping.setdefault(image_spec, { "os_title": os_title, "release_title": release_title }) mock_download = self.patch(bootsources, "download_all_image_descriptions") mock_download.return_value = image_mapping cache_boot_sources() cached = BootSourceCache.objects.filter(boot_source=source).first() self.assertDictEqual({"title": "%s %s" % (os_title, release_title)}, cached.extra)
def test__adds_title_to_extra(self): source = factory.make_BootSource(keyring_data=b'1234') os = factory.make_name('os') release = factory.make_name('release') os_title = factory.make_name('os_title') release_title = factory.make_name('release_title') image_spec = make_image_spec(os=os, release=release) image_mapping = BootImageMapping() image_mapping.setdefault(image_spec, { 'os_title': os_title, 'release_title': release_title, }) mock_download = self.patch(bootsources, 'download_all_image_descriptions') mock_download.return_value = image_mapping cache_boot_sources() cached = BootSourceCache.objects.filter(boot_source=source).first() self.assertDictEqual({'title': '%s %s' % (os_title, release_title)}, cached.extra)
def test__adds_release_to_cache(self): source = factory.make_BootSource(keyring_data=b'1234') os = factory.make_name('os') release = factory.make_name('release') release_codename = factory.make_name('codename') release_title = factory.make_name('title') support_eol = factory.make_date().strftime("%Y-%m-%d") image_spec = make_image_spec(os=os, release=release) image_mapping = BootImageMapping() image_mapping.setdefault( image_spec, { "release_codename": release_codename, "release_title": release_title, "support_eol": support_eol, }) bootsources._update_cache(source.to_dict_without_selections(), image_mapping) cached = BootSourceCache.objects.filter(boot_source=source).first() self.assertEqual(release_codename, cached.release_codename) self.assertEqual(release_title, cached.release_title) self.assertEqual(support_eol, cached.support_eol.strftime("%Y-%m-%d"))
def download_all_image_descriptions(sources, user_agent=None, validate_products=True): """Download image metadata for all sources in `config`.""" boot = BootImageMapping() for source in sources: repo_boot = download_image_descriptions( source['url'], keyring=source.get('keyring', None), user_agent=user_agent, validate_products=validate_products) boot_merge(boot, repo_boot, source['selections']) return boot
def test__adds_release_codename_title_and_support_eol(self): source = factory.make_BootSource(keyring_data=b'1234') os = factory.make_name('os') release = factory.make_name('release') release_codename = factory.make_name('codename') release_title = factory.make_name('title') support_eol = factory.make_date().strftime("%Y-%m-%d") image_spec = make_image_spec(os=os, release=release) image_mapping = BootImageMapping() image_mapping.setdefault(image_spec, { "release_codename": release_codename, "release_title": release_title, "support_eol": support_eol, }) mock_download = self.patch( bootsources, 'download_all_image_descriptions') mock_download.return_value = image_mapping cache_boot_sources() cached = BootSourceCache.objects.filter(boot_source=source).first() self.assertEqual(release_codename, cached.release_codename) self.assertEqual(release_title, cached.release_title) self.assertEqual(support_eol, cached.support_eol.strftime("%Y-%m-%d"))
def test_sync_does_propagate_ioerror(self): io_error = factory.make_exception_type(bases=(IOError, )) mock_sync = self.patch(download_descriptions.BasicMirrorWriter, "sync") mock_sync.side_effect = io_error() boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict) with FakeLogger("maas.import-images", level=logging.INFO) as maaslog: self.assertRaises(io_error, dumper.sync, sentinel.reader, sentinel.path) self.assertDocTestMatches("...error...syncing boot images...", maaslog.output)
def test_dump_json_combines_similar_entries(self): image = make_image_spec() other_release = factory.make_name('other-release') resource1 = factory.make_name('resource') resource2 = factory.make_name('other-resource') image_dict = BootImageMapping() set_resource(image_dict, image, resource1) set_resource( image_dict, image._replace(release=other_release), resource2) self.assertEqual( { image.os: { image.arch: { image.subarch: { image.kflavor: { image.release: {image.label: resource1}, other_release: {image.label: resource2}, }, }, }, }, }, json.loads(image_dict.dump_json()))
def test_obeys_filters(self): filters = [{ "os": factory.make_name("os"), "arches": [factory.make_name("other-arch")], "subarches": [factory.make_name("other-subarch")], "release": factory.make_name("other-release"), "label": [factory.make_name("other-label")], }] total_resources = BootImageMapping() resources_from_repo = set_resource() download_descriptions.boot_merge(total_resources, resources_from_repo, filters=filters) self.assertEqual({}, total_resources.mapping)
def test_insert_item_sets_generic_to_release_item_for_hwe_version(self): boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict) os = "ubuntu" release = "xenial" arch = "amd64" label = "release" hwep_subarch = "hwe-16.04" hwep_subarches = ["generic", "hwe-16.04", "hwe-16.10"] hwes_subarch = "hwe-16.10" hwes_subarches = ["generic", "hwe-16.04", "hwe-16.10"] hwep_item, compat_item = self.make_item( os=os, release=release, arch=arch, subarch=hwep_subarch, subarches=hwep_subarches, label=label, ) hwes_item, _ = self.make_item( os=os, release=release, arch=arch, subarch=hwes_subarch, subarches=hwes_subarches, label=label, ) self.patch(download_descriptions, "products_exdata").side_effect = [ hwep_item, hwes_item, ] for _ in range(2): dumper.insert_item( {"os": "ubuntu"}, sentinel.src, sentinel.target, ( "com.ubuntu.maas.daily:v3:boot:12.04:amd64:hwe-p", factory.make_name("product_version"), ), sentinel.contentsource, ) image_spec = make_image_spec(os=os, release=release, arch=arch, subarch="generic", label=label) self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
def test_obeys_filters(self): filters = [ { 'os': factory.make_name('os'), 'arches': [factory.make_name('other-arch')], 'subarches': [factory.make_name('other-subarch')], 'release': factory.make_name('other-release'), 'label': [factory.make_name('other-label')], }, ] total_resources = BootImageMapping() resources_from_repo = set_resource() download_descriptions.boot_merge(total_resources, resources_from_repo, filters=filters) self.assertEqual({}, total_resources.mapping)
def test_insert_item_adds_item_per_subarch(self): boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict) subarches = [factory.make_name('subarch') for _ in range(3)] item, _ = self.make_item(subarch=subarches.pop(), subarches=subarches) self.patch(download_descriptions, 'products_exdata').return_value = item dumper.insert_item(sentinel.data, sentinel.src, sentinel.target, (factory.make_name('product_name'), factory.make_name('product_version')), sentinel.contentsource) image_specs = [ make_image_spec(os=item['os'], release=item['release'], arch=item['arch'], subarch=subarch, label=item['label']) for subarch in subarches ] self.assertItemsEqual(image_specs, list(boot_images_dict.mapping))
def test_insert_item_doesnt_validate_when_instructed(self): boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict, validate_products=False) item, _ = self.make_item(os='ubuntu') self.patch(download_descriptions, 'products_exdata').return_value = item dumper.insert_item({'os': 'ubuntu'}, sentinel.src, sentinel.target, ( factory.make_name('product_name'), factory.make_name('product_version'), ), sentinel.contentsource) image_specs = [ make_image_spec(os=item['os'], release=item['release'], arch=item['arch'], subarch=subarch, label=item['label']) for subarch in item['subarches'].split(',') ] self.assertItemsEqual(image_specs, list(boot_images_dict.mapping))
def test_concatenates_similar_resources(self): image1 = make_image_spec() image2 = make_image_spec() resource = make_boot_resource() boot_dict = BootImageMapping() # Create two images in boot_dict, both containing the same resource. for image in [image1, image2]: set_resource( boot_dict=boot_dict, resource=resource.copy(), image_spec=image ) products_mapping = map_products(boot_dict) key = ( resource["content_id"], resource["product_name"], resource["version_name"], ) self.assertEqual([key], list(products_mapping.mapping)) self.assertItemsEqual( [image1.subarch, image2.subarch], products_mapping.get(resource) )
def test__consistent_query_count(self): source = factory.make_BootSource(keyring_data=b'1234') image_mapping = BootImageMapping() for _ in range(random.randint(20, 50)): self.make_release(image_mapping) # Add all the items to the cache, always 5. queries, _ = count_queries(bootsources._update_cache, source.to_dict_without_selections(), image_mapping) self.assertEquals(5, queries) # Now that they all already exist, it should only be 4 queries. queries, _ = count_queries(bootsources._update_cache, source.to_dict_without_selections(), image_mapping) self.assertEquals(4, queries) # Do it again just to be sure. queries, _ = count_queries(bootsources._update_cache, source.to_dict_without_selections(), image_mapping) self.assertEquals(4, queries)
def test__catches_connection_errors_and_sets_component_error(self): sources = [ factory.make_BootSource(keyring_data=b'1234') for _ in range(3)] download_image_descriptions = self.patch( download_descriptions_module, 'download_image_descriptions') error_text_one = factory.make_name("<error1>") error_text_two = factory.make_name("<error2>") # Make two of the downloads fail. download_image_descriptions.side_effect = [ ConnectionError(error_text_one), BootImageMapping(), IOError(error_text_two), ] cache_boot_sources() base_error = "Failed to import images from boot source {url}: {err}" error_part_one = base_error.format( url=sources[0].url, err=html.escape(error_text_one)) error_part_two = base_error.format( url=sources[2].url, err=html.escape(error_text_two)) expected_error = error_part_one + '<br>' + error_part_two actual_error = get_persistent_error(COMPONENT.REGION_IMAGE_IMPORT) self.assertEqual(expected_error, actual_error)
def test_insert_item_sets_compat_item_specific_to_subarch(self): boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict) subarches = [factory.make_name("subarch") for _ in range(5)] compat_subarch = subarches.pop() item, _ = self.make_item(subarch=subarches.pop(), subarches=subarches) second_item, compat_item = self.make_item( os=item["os"], release=item["release"], arch=item["arch"], subarch=compat_subarch, subarches=[compat_subarch], label=item["label"], ) self.patch(download_descriptions, "products_exdata").side_effect = [ item, second_item, ] for _ in range(2): dumper.insert_item( sentinel.data, sentinel.src, sentinel.target, ( factory.make_name("product_name"), factory.make_name("product_version"), ), sentinel.contentsource, ) image_spec = make_image_spec( os=item["os"], release=item["release"], arch=item["arch"], subarch=compat_subarch, label=item["label"], ) self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
def download_image_descriptions(path, keyring=None, user_agent=None, validate_products=True): """Download image metadata from upstream Simplestreams repo. :param path: The path to a Simplestreams repo. :param keyring: Optional keyring for verifying the repo's signatures. :param user_agent: Optional user agent string for downloading the image descriptions. :return: A `BootImageMapping` describing available boot resources. """ maaslog.info("Downloading image descriptions from %s", path) mirror, rpath = path_from_mirror_url(path, None) policy = get_signing_policy(rpath, keyring) if user_agent is None: # If user_agent is NOT set, we know we are downloading descriptions # from the Region controller *by* the Rack controller. maaslog.info("Rack downloading image descriptions from '%s'.", path) reader = UrlMirrorReader(mirror, policy=policy) else: # Since user_agent is set, we know we are downloading descriptions # from the Images repository *by* the Region. maaslog.info("Region downloading image descriptions from '%s'.", path) try: reader = UrlMirrorReader(mirror, policy=policy, user_agent=user_agent) except TypeError: # UrlMirrorReader doesn't support the user_agent argument. # simplestream >=bzr429 is required for this feature. reader = UrlMirrorReader(mirror, policy=policy) boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict, validate_products=validate_products) dumper.sync(reader, rpath) return boot_images_dict
def test_insert_item_sets_generic_to_release_item_for_hwe_version(self): boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict) os = 'ubuntu' release = 'xenial' arch = 'amd64' label = 'release' hwep_subarch = 'hwe-16.04' hwep_subarches = ['generic', 'hwe-16.04', 'hwe-16.10'] hwes_subarch = 'hwe-16.10' hwes_subarches = ['generic', 'hwe-16.04', 'hwe-16.10'] hwep_item, compat_item = self.make_item(os=os, release=release, arch=arch, subarch=hwep_subarch, subarches=hwep_subarches, label=label) hwes_item, _ = self.make_item(os=os, release=release, arch=arch, subarch=hwes_subarch, subarches=hwes_subarches, label=label) self.patch(download_descriptions, 'products_exdata').side_effect = [hwep_item, hwes_item] for _ in range(2): dumper.insert_item( {'os': 'ubuntu'}, sentinel.src, sentinel.target, ( 'com.ubuntu.maas.daily:v3:boot:12.04:amd64:hwe-p', factory.make_name('product_version'), ), sentinel.contentsource) image_spec = make_image_spec(os=os, release=release, arch=arch, subarch='generic', label=label) self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
def test_is_empty_returns_False_if_not_empty(self): mapping = BootImageMapping() mapping.setdefault(make_image_spec(), factory.make_name("resource")) self.assertFalse(mapping.is_empty())
def test_load_json_returns_empty_mapping_for_invalid_json(self): bad_json = "" mapping = BootImageMapping.load_json(bad_json) self.assertEqual({}, mapping.mapping)
def test_initially_empty(self): self.assertItemsEqual([], BootImageMapping().items())
def test_is_empty_returns_True_if_empty(self): self.assertTrue(BootImageMapping().is_empty())
def test_maps_empty_dict_to_empty_dict(self): empty_boot_image_dict = BootImageMapping() self.assertEqual({}, map_products(empty_boot_image_dict).mapping)
def test_extract_ephemeral_image_params_with_metadata(self): path, osystem, arch, subarch, release, label = self._make_path() # Patch OperatingSystemRegistry to return a fixed list of # values. purpose1 = factory.make_name("purpose") purpose2 = factory.make_name("purpose") xi_purpose = "ephemeral" xi_path = factory.make_name("xi_path") xi_type = factory.make_name("xi_type") purposes = [purpose1, purpose2, xi_purpose] self._patch_osystem_registry(purposes, xinstall_params=(xi_path, xi_type)) # Create some maas.meta content. image = ImageSpec(os=osystem, arch=arch, subarch=subarch, kflavor='generic', release=release, label=label) image_resource = dict(subarches=factory.make_name("subarches")) mapping = BootImageMapping() mapping.setdefault(image, image_resource) maas_meta = mapping.dump_json() params = extract_image_params(path, maas_meta) self.assertItemsEqual([ { "osystem": osystem, "architecture": arch, "subarchitecture": subarch, "release": release, "label": label, "purpose": purpose1, "xinstall_path": '', "xinstall_type": '', "supported_subarches": image_resource["subarches"], }, { "osystem": osystem, "architecture": arch, "subarchitecture": subarch, "release": release, "label": label, "purpose": purpose2, "xinstall_path": '', "xinstall_type": '', "supported_subarches": image_resource["subarches"], }, { "osystem": osystem, "architecture": arch, "subarchitecture": subarch, "release": release, "label": label, "purpose": xi_purpose, "xinstall_path": xi_path, "xinstall_type": xi_type, "supported_subarches": image_resource["subarches"], }, ], params)
def test_load_json_result_of_old_data_uses_ubuntu_as_os(self): test_meta_file_content = make_maas_meta_without_os() mapping = BootImageMapping.load_json(test_meta_file_content) os = {image.os for image, _ in mapping.items()}.pop() self.assertEqual("ubuntu", os)
def test_dump_json_represents_empty_dict_as_empty_object(self): self.assertEqual("{}", BootImageMapping().dump_json())
def make_boot_image_mapping(image_specs=None): mapping = BootImageMapping() if image_specs is not None: for image_spec in image_specs: mapping.setdefault(image_spec, {}) return mapping
def test_setdefault_sets_unset_item(self): image_dict = BootImageMapping() image = make_image_spec() resource = factory.make_name("resource") image_dict.setdefault(image, resource) self.assertItemsEqual([(image, resource)], image_dict.items())