def make_osystem_with_releases(testcase, osystem_name=None, releases=None): """Generate an arbitrary operating system. :param osystem_name: The operating system name. Useful in cases where we need to test that not supplying an os works correctly. :param releases: The list of releases name. Useful in cases where we need to test that not supplying a release works correctly. """ if osystem_name is None: osystem_name = factory.make_name('os') if releases is None: releases = [factory.make_name('release') for _ in range(3)] rpc_releases = [make_rpc_release(release) for release in releases] if osystem_name not in OperatingSystemRegistry: OperatingSystemRegistry.register_item(osystem_name, CustomOS()) testcase.addCleanup(OperatingSystemRegistry.unregister_item, osystem_name) # Make sure the commissioning Ubuntu release and all created releases # are available to all architectures. architectures = [ node.architecture for node in Node.objects.distinct('architecture') ] if len(architectures) == 0: architectures.append('%s/generic' % factory.make_name('arch')) for arch in architectures: factory.make_default_ubuntu_release_bootable(arch.split('/')[0]) for release in releases: factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.UPLOADED, name=('%s/%s' % (osystem_name, release)), architecture=arch) return make_rpc_osystem(osystem_name, releases=rpc_releases)
def test_creates_boot_resoures_with_uploaded_rtype(self): os = factory.make_name("os") series = factory.make_name("series") OperatingSystemRegistry.register_item(os, CustomOS()) self.addCleanup(OperatingSystemRegistry.unregister_item, os) name = "%s/%s" % (os, series) architecture = make_usable_architecture(self) upload_type, filetype = self.pick_filetype() size = random.randint(1024, 2048) content = factory.make_string(size).encode("utf-8") upload_name = factory.make_name("filename") uploaded_file = SimpleUploadedFile(content=content, name=upload_name) data = { "name": name, "architecture": architecture, "filetype": upload_type, } form = BootResourceForm(data=data, files={"content": uploaded_file}) self.assertTrue(form.is_valid(), form._errors) form.save() resource = BootResource.objects.get( rtype=BOOT_RESOURCE_TYPE.UPLOADED, name=name, architecture=architecture, ) resource_set = resource.sets.first() rfile = resource_set.files.first() self.assertTrue(filetype, rfile.filetype) self.assertTrue(filetype, rfile.filename) self.assertTrue(size, rfile.largefile.total_size) with rfile.largefile.content.open("rb") as stream: written_content = stream.read() self.assertEqual(content, written_content)
def make_osystem_requiring_license_key(testcase, osystem=None, release=None): if osystem is None: osystem = factory.make_name("osystem") if release is None: release = random.choice(REQUIRE_LICENSE_KEY) distro_series = "%s/%s" % (osystem, release) drv = WindowsOS() drv.title = osystem OperatingSystemRegistry.register_item(osystem, drv) factory.make_BootResource( name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED, extra={"title": drv.get_release_title(release)}, ) testcase.addCleanup(OperatingSystemRegistry.unregister_item, osystem) return { "name": osystem, "title": osystem, "default_release": release, "default_commissioning_release": None, "releases": [{ "name": distro_series, "title": drv.get_release_title(release), "requires_license_key": True, "can_commission": False, }], }
def test_adds_boot_resource_set_to_existing_uploaded_boot_resource(self): os = factory.make_name('os') series = factory.make_name('series') OperatingSystemRegistry.register_item(os, CustomOS()) self.addCleanup(OperatingSystemRegistry.unregister_item, os) name = '%s/%s' % (os, series) architecture = make_usable_architecture(self) resource = factory.make_usable_boot_resource( rtype=BOOT_RESOURCE_TYPE.UPLOADED, name=name, architecture=architecture) upload_type, filetype = self.pick_filetype() size = random.randint(1024, 2048) content = factory.make_string(size).encode('utf-8') upload_name = factory.make_name('filename') uploaded_file = SimpleUploadedFile(content=content, name=upload_name) data = { 'name': name, 'architecture': architecture, 'filetype': upload_type, 'keep_old': True, } form = BootResourceForm(data=data, files={'content': uploaded_file}) self.assertTrue(form.is_valid(), form._errors) form.save() resource = reload_object(resource) resource_set = resource.sets.order_by('id').last() rfile = resource_set.files.first() self.assertTrue(filetype, rfile.filetype) self.assertTrue(filetype, rfile.filename) self.assertTrue(size, rfile.largefile.total_size) with rfile.largefile.content.open('rb') as stream: written_content = stream.read() self.assertEqual(content, written_content) self.assertEqual(resource.rtype, BOOT_RESOURCE_TYPE.UPLOADED)
def make_os_with_license_key(self, osystem=None, osystem_title=None, release=None): """Makes a fake operating system that has a release that requires a license key.""" if osystem is None: osystem = factory.make_name("osystem") if osystem_title is None: osystem_title = osystem + "_title" if release is None: release = random.choice(REQUIRE_LICENSE_KEY) distro_series = "%s/%s" % (osystem, release) drv = WindowsOS() drv.title = osystem_title OperatingSystemRegistry.register_item(osystem, drv) factory.make_BootResource( name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED, extra={"title": drv.get_release_title(release)}, ) self.addCleanup(OperatingSystemRegistry.unregister_item, osystem) return ( { "name": osystem, "title": osystem_title }, { "name": release, "title": drv.get_release_title(release) }, )
def test_removes_old_bootresourcefiles(self): # Regression test for LP:1660418 os = factory.make_name('os') series = factory.make_name('series') OperatingSystemRegistry.register_item(os, CustomOS()) self.addCleanup(OperatingSystemRegistry.unregister_item, os) name = '%s/%s' % (os, series) architecture = make_usable_architecture(self) resource = factory.make_usable_boot_resource( rtype=BOOT_RESOURCE_TYPE.UPLOADED, name=name, architecture=architecture) upload_type, filetype = self.pick_filetype() size = random.randint(1024, 2048) content = factory.make_string(size).encode('utf-8') upload_name = factory.make_name('filename') uploaded_file = SimpleUploadedFile(content=content, name=upload_name) data = { 'name': name, 'architecture': architecture, 'filetype': upload_type, } form = BootResourceForm(data=data, files={'content': uploaded_file}) self.assertTrue(form.is_valid(), form._errors) form.save() self.assertEqual( 1, BootResourceFile.objects.filter( resource_set__resource=resource).count())
def make_osystem_requiring_license_key(testcase, osystem=None, release=None): if osystem is None: osystem = factory.make_name('osystem') if release is None: release = random.choice(REQUIRE_LICENSE_KEY) distro_series = '%s/%s' % (osystem, release) drv = WindowsOS() drv.title = osystem OperatingSystemRegistry.register_item(osystem, drv) factory.make_BootResource( name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED, extra={'title': drv.get_release_title(release)}) testcase.addCleanup( OperatingSystemRegistry.unregister_item, osystem) return { 'name': osystem, 'title': osystem, 'default_release': release, 'default_commissioning_release': None, 'releases': [{ 'name': distro_series, 'title': drv.get_release_title(release), 'requires_license_key': True, 'can_commission': False, }] }
def make_os(testcase): osystem = factory.make_name('osystem') release = random.choice(REQUIRE_LICENSE_KEY) distro_series = '%s/%s' % (osystem, release) drv = WindowsOS() drv.title = osystem OperatingSystemRegistry.register_item(osystem, drv) factory.make_BootResource( name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED, extra={'title': drv.get_release_title(release)}) testcase.addCleanup( OperatingSystemRegistry.unregister_item, osystem) return osystem, release
def list_all_usable_osystems(releases=None): """Return all operating systems that can be used for nodes.""" if releases is None: releases = list_all_usable_releases() osystems = [] for os_name, releases in releases.items(): osystem = OperatingSystemRegistry.get_item(os_name) if osystem: default_commissioning_release = ( osystem.get_default_commissioning_release() ) default_release = osystem.get_default_release() title = osystem.title else: default_commissioning_release = "" default_release = "" title = os_name osystems.append( { "name": os_name, "title": title, "default_commissioning_release": default_commissioning_release, "default_release": default_release, "releases": releases, } ) return sorted(osystems, key=itemgetter("title"))
def test_prevents_reversed_osystem_from_driver(self): reserved_name = factory.make_name('name') OperatingSystemRegistry.register_item(reserved_name, CustomOS()) upload_type, filetype = self.pick_filetype() size = random.randint(1024, 2048) content = factory.make_string(size).encode('utf-8') upload_name = factory.make_name('filename') uploaded_file = SimpleUploadedFile(content=content, name=upload_name) data = { 'name': reserved_name, 'title': factory.make_name('title'), 'architecture': make_usable_architecture(self), 'filetype': upload_type, } form = BootResourceForm(data=data, files={'content': uploaded_file}) self.assertFalse(form.is_valid())
def test_settings_shows_license_keys_if_OS_supporting_keys(self): self.client.login(user=factory.make_admin()) osystem = factory.make_name('osystem') release = random.choice(REQUIRE_LICENSE_KEY) distro_series = '%s/%s' % (osystem, release) drv = WindowsOS() drv.title = osystem OperatingSystemRegistry.register_item(osystem, drv) factory.make_BootResource( name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED, extra={'title': drv.get_release_title(release)}) self.addCleanup(OperatingSystemRegistry.unregister_item, osystem) response = self.client.get(reverse('settings_general')) doc = fromstring(response.content) license_keys = doc.cssselect('a[href="/MAAS/settings/license-keys/"]') self.assertEqual(1, len(license_keys), "Didn't show the license key section.")
def extract_image_params(path, maas_meta): """Represent a list of TFTP path elements as a list of boot-image dicts. :param path: Tuple or list that consists of a full [osystem, architecture, subarchitecture, release] that identify a kind of boot for which we may need an image. :param maas_meta: Contents of the maas.meta file. This may be an empty string. :return: A list of dicts, each of which may also include additional items of meta-data that are not elements in the path, such as "subarches". """ if path[0] == "bootloader": osystem, release, arch = path subarch = "generic" label = "*" else: osystem, arch, subarch, release, label = path osystem_obj = OperatingSystemRegistry.get_item(osystem, default=None) if osystem_obj is None: return [] purposes = osystem_obj.get_boot_image_purposes(arch, subarch, release, label) # Expand the path into a list of dicts, one for each boot purpose. params = [] for purpose in purposes: image = dict( osystem=osystem, architecture=arch, subarchitecture=subarch, release=release, label=label, purpose=purpose, ) if (purpose == BOOT_IMAGE_PURPOSE.XINSTALL or purpose == BOOT_IMAGE_PURPOSE.EPHEMERAL): xinstall_path, xinstall_type = osystem_obj.get_xinstall_parameters( arch, subarch, release, label) image["xinstall_path"] = xinstall_path image["xinstall_type"] = xinstall_type else: image["xinstall_path"] = "" image["xinstall_type"] = "" params.append(image) # Merge in the meta-data. for image_dict in params: metadata = extract_metadata(maas_meta, image_dict) image_dict.update(metadata) return params
def make_osystem(testcase, osystem, purpose=None, releases=None): """Makes the operating system class and registers it.""" if osystem not in OperatingSystemRegistry: fake = FakeOS(osystem, purpose, releases) OperatingSystemRegistry.register_item(fake.name, fake) testcase.addCleanup(OperatingSystemRegistry.unregister_item, osystem) return fake else: obj = OperatingSystemRegistry[osystem] old_func = obj.get_boot_image_purposes testcase.patch(obj, "get_boot_image_purposes").return_value = purpose def reset_func(obj, old_func): obj.get_boot_image_purposes = old_func testcase.addCleanup(reset_func, obj, old_func) return obj
def list_all_usable_releases(): """Return all releases for all operating systems that can be used.""" distro_series = {} seen_releases = set() for br in BootResource.objects.filter(bootloader_type=None): # An OS can have multiple boot resource for one release. e.g Ubuntu # Bionic has ga-18.04 and ga-18.04-lowlatency. This list should only # contain one entry per OS. if br.name in seen_releases: continue seen_releases.add(br.name) if "/" in br.name: os_name, release = br.name.split("/") else: os_name = "custom" release = br.name osystem = OperatingSystemRegistry.get_item(os_name) if osystem is not None: title = osystem.get_release_title(release) if title is None: title = release can_commission = ( release in osystem.get_supported_commissioning_releases()) requires_license_key = osystem.requires_license_key(release) else: title = br.name can_commission = requires_license_key = False if br.rtype == BOOT_RESOURCE_TYPE.UPLOADED: # User may set the title of an uploaded resource. if "title" in br.extra: title = br.extra["title"] else: title = release if os_name not in distro_series: distro_series[os_name] = [] distro_series[os_name].append({ "name": release, "title": title, "can_commission": can_commission, "requires_license_key": requires_license_key, }) for osystem, releases in distro_series.items(): distro_series[osystem] = sorted(releases, key=itemgetter("title")) return OrderedDict(sorted(distro_series.items()))
def get_boot_image(self, params, client, remote_ip): """Get the boot image for the params on this rack controller. Calls `MarkNodeFailed` for the machine if its a known machine. """ is_ephemeral = False try: osystem_obj = OperatingSystemRegistry.get_item(params['osystem'], default=None) purposes = osystem_obj \ .get_boot_image_purposes(params["arch"], params["subarch"], params.get("release", ""), params.get("label", "")) if "ephemeral" in purposes: is_ephemeral = True except: pass system_id = params.pop("system_id", None) if params["purpose"] == "local" and not is_ephemeral: # Local purpose doesn't use a boot image so jsut set the label # to "local", but this value will no be used. params["label"] = "local" return params else: if params["purpose"] == "local" and is_ephemeral: params["purpose"] = "ephemeral" boot_image = get_boot_image(params) if boot_image is None: # No matching boot image. description = "Missing boot image %s/%s/%s/%s." % ( params['osystem'], params["arch"], params["subarch"], params["release"]) # Call MarkNodeFailed if this was a known machine. if system_id is not None: d = client(MarkNodeFailed, system_id=system_id, error_description=description) d.addErrback( log.err, "Failed to mark machine failed: %s" % description) else: maaslog.error( "Enlistment failed to boot %s; missing required boot " "image %s/%s/%s/%s." % (remote_ip, params['osystem'], params["arch"], params["subarch"], params["release"])) params["label"] = "no-such-image" else: params["label"] = boot_image["label"] return params
def clean(self): """Validate the model. Checks that the name is in a valid format, for its type. """ if self.rtype == BOOT_RESOURCE_TYPE.UPLOADED: if "/" in self.name: os_name = self.name.split("/")[0] osystem = OperatingSystemRegistry.get_item(os_name) if osystem is None: raise ValidationError( "%s boot resource cannot contain a '/' in it's name " "unless it starts with a supported operating system." % (self.display_rtype)) elif self.rtype in RTYPE_REQUIRING_OS_SERIES_NAME: if "/" not in self.name: raise ValidationError( "%s boot resource must contain a '/' in it's name." % (self.display_rtype))
def get_boot_image(self, params, client, remote_ip): """Get the boot image for the params on this rack controller. Calls `MarkNodeFailed` for the machine if its a known machine. """ is_ephemeral = False try: osystem_obj = OperatingSystemRegistry.get_item( params["osystem"], default=None ) purposes = osystem_obj.get_boot_image_purposes( params["arch"], params["subarch"], params.get("release", ""), params.get("label", ""), ) if "ephemeral" in purposes: is_ephemeral = True except Exception: pass # Check to see if the we are PXE booting a device. if params["purpose"] == "local-device": mac = network.find_mac_via_arp(remote_ip) log.info( "Device %s with MAC address %s is PXE booting; " "instructing the device to boot locally." % (params["hostname"], mac) ) # Set purpose back to local now that we have the message logged. params["purpose"] = "local" system_id = params.pop("system_id", None) if params["purpose"] == "local" and not is_ephemeral: # Local purpose doesn't use a boot image so just set the label # to "local". params["label"] = "local" return params else: if params["purpose"] == "local" and is_ephemeral: params["purpose"] = "ephemeral" boot_image = get_boot_image(params) if boot_image is None: # No matching boot image. description = "Missing boot image %s/%s/%s/%s." % ( params["osystem"], params["arch"], params["subarch"], params["release"], ) # Call MarkNodeFailed if this was a known machine. if system_id is not None: d = client( MarkNodeFailed, system_id=system_id, error_description=description, ) d.addErrback( log.err, "Failed to mark machine failed: %s" % description, ) else: maaslog.error( "Enlistment failed to boot %s; missing required boot " "image %s/%s/%s/%s." % ( remote_ip, params["osystem"], params["arch"], params["subarch"], params["release"], ) ) params["label"] = "no-such-image" else: params["label"] = boot_image["label"] return params
def validate_hwe_kernel( hwe_kernel, min_hwe_kernel, architecture, osystem, distro_series, commissioning_osystem=undefined, commissioning_distro_series=undefined, ): """Validates that hwe_kernel works on the selected os/release/arch. Checks that the current hwe_kernel is avalible for the selected os/release/architecture combination, and that the selected hwe_kernel is >= min_hwe_kernel. If no hwe_kernel is selected one will be chosen. """ def validate_kernel_str(kstr): return kstr.startswith("hwe-") or kstr.startswith("ga-") if not all((osystem, architecture, distro_series)): return hwe_kernel # If we are dealing with an ephemeral image, just return the hwe_kernel # as-is, i.e. just stick with generic. osystem_obj = OperatingSystemRegistry.get_item(osystem, default=None) if osystem_obj is not None: arch, subarch = architecture.split("/") purposes = osystem_obj.get_boot_image_purposes(arch, subarch, distro_series, "*") if "ephemeral" in purposes: return hwe_kernel # If we're not deploying Ubuntu we are just setting the kernel to be used # during deployment if osystem != "ubuntu": osystem = commissioning_osystem if osystem is undefined: osystem = Config.objects.get_config("commissioning_osystem") distro_series = commissioning_distro_series if distro_series is undefined: distro_series = Config.objects.get_config( "commissioning_distro_series") arch, subarch = architecture.split("/") if subarch != "generic" and ( (hwe_kernel and validate_kernel_str(hwe_kernel)) or (min_hwe_kernel and validate_kernel_str(min_hwe_kernel))): raise ValidationError( "Subarchitecture(%s) must be generic when setting hwe_kernel." % subarch) os_release = osystem + "/" + distro_series if hwe_kernel and validate_kernel_str(hwe_kernel): usable_kernels = BootResource.objects.get_usable_hwe_kernels( os_release, arch) if hwe_kernel not in usable_kernels: raise ValidationError("%s is not available for %s on %s." % (hwe_kernel, os_release, architecture)) if not release_a_newer_than_b(hwe_kernel, distro_series): raise ValidationError("%s is too old to use on %s." % (hwe_kernel, os_release)) if (min_hwe_kernel and validate_kernel_str(min_hwe_kernel)) and ( not release_a_newer_than_b(hwe_kernel, min_hwe_kernel)): raise ValidationError( "hwe_kernel(%s) is older than min_hwe_kernel(%s)." % (hwe_kernel, min_hwe_kernel)) return hwe_kernel elif min_hwe_kernel and validate_kernel_str(min_hwe_kernel): # Determine what kflavor is being used by check against a list of # known kflavors. valid_kflavors = { br.kflavor for br in BootResource.objects.exclude(kflavor=None) } kflavor = "generic" for kernel_part in min_hwe_kernel.split("-"): if kernel_part in valid_kflavors: kflavor = kernel_part break usable_kernels = BootResource.objects.get_usable_hwe_kernels( os_release, arch, kflavor) for i in usable_kernels: if release_a_newer_than_b( i, min_hwe_kernel) and release_a_newer_than_b( i, distro_series): return i raise ValidationError( "%s has no kernels available which meet min_hwe_kernel(%s)." % (distro_series, min_hwe_kernel)) for kernel in BootResource.objects.get_usable_hwe_kernels( os_release, arch, "generic"): if release_a_newer_than_b(kernel, distro_series): return kernel raise ValidationError("%s has no kernels available." % distro_series)
def test_operating_system_registry(self): self.assertItemsEqual([], OperatingSystemRegistry) OperatingSystemRegistry.register_item("resource", sentinel.resource) self.assertIn(sentinel.resource, (item for name, item in OperatingSystemRegistry))