def make_osystem_with_releases(testcase, osystem_name=None, releases=None):
"""Generate an arbitrary operating system.
:param osystem_name: The operating system name. Useful in cases where
we need to test that not supplying an os works correctly.
:param releases: The list of releases name. Useful in cases where
we need to test that not supplying a release works correctly.
"""
if osystem_name is None:
osystem_name = factory.make_name('os')
if releases is None:
releases = [factory.make_name('release') for _ in range(3)]
rpc_releases = [make_rpc_release(release) for release in releases]
if osystem_name not in OperatingSystemRegistry:
OperatingSystemRegistry.register_item(osystem_name, CustomOS())
testcase.addCleanup(OperatingSystemRegistry.unregister_item,
osystem_name)
# Make sure the commissioning Ubuntu release and all created releases
# are available to all architectures.
architectures = [
node.architecture for node in Node.objects.distinct('architecture')
]
if len(architectures) == 0:
architectures.append('%s/generic' % factory.make_name('arch'))
for arch in architectures:
factory.make_default_ubuntu_release_bootable(arch.split('/')[0])
for release in releases:
factory.make_BootResource(rtype=BOOT_RESOURCE_TYPE.UPLOADED,
name=('%s/%s' % (osystem_name, release)),
architecture=arch)
return make_rpc_osystem(osystem_name, releases=rpc_releases)
def test_creates_boot_resoures_with_uploaded_rtype(self):
os = factory.make_name("os")
series = factory.make_name("series")
OperatingSystemRegistry.register_item(os, CustomOS())
self.addCleanup(OperatingSystemRegistry.unregister_item, os)
name = "%s/%s" % (os, series)
architecture = make_usable_architecture(self)
upload_type, filetype = self.pick_filetype()
size = random.randint(1024, 2048)
content = factory.make_string(size).encode("utf-8")
upload_name = factory.make_name("filename")
uploaded_file = SimpleUploadedFile(content=content, name=upload_name)
data = {
"name": name,
"architecture": architecture,
"filetype": upload_type,
}
form = BootResourceForm(data=data, files={"content": uploaded_file})
self.assertTrue(form.is_valid(), form._errors)
form.save()
resource = BootResource.objects.get(
rtype=BOOT_RESOURCE_TYPE.UPLOADED,
name=name,
architecture=architecture,
)
resource_set = resource.sets.first()
rfile = resource_set.files.first()
self.assertTrue(filetype, rfile.filetype)
self.assertTrue(filetype, rfile.filename)
self.assertTrue(size, rfile.largefile.total_size)
with rfile.largefile.content.open("rb") as stream:
written_content = stream.read()
self.assertEqual(content, written_content)
def make_osystem_requiring_license_key(testcase, osystem=None, release=None):
if osystem is None:
osystem = factory.make_name("osystem")
if release is None:
release = random.choice(REQUIRE_LICENSE_KEY)
distro_series = "%s/%s" % (osystem, release)
drv = WindowsOS()
drv.title = osystem
OperatingSystemRegistry.register_item(osystem, drv)
factory.make_BootResource(
name=distro_series,
rtype=BOOT_RESOURCE_TYPE.UPLOADED,
extra={"title": drv.get_release_title(release)},
)
testcase.addCleanup(OperatingSystemRegistry.unregister_item, osystem)
return {
"name":
osystem,
"title":
osystem,
"default_release":
release,
"default_commissioning_release":
None,
"releases": [{
"name": distro_series,
"title": drv.get_release_title(release),
"requires_license_key": True,
"can_commission": False,
}],
}
def test_adds_boot_resource_set_to_existing_uploaded_boot_resource(self):
os = factory.make_name('os')
series = factory.make_name('series')
OperatingSystemRegistry.register_item(os, CustomOS())
self.addCleanup(OperatingSystemRegistry.unregister_item, os)
name = '%s/%s' % (os, series)
architecture = make_usable_architecture(self)
resource = factory.make_usable_boot_resource(
rtype=BOOT_RESOURCE_TYPE.UPLOADED,
name=name,
architecture=architecture)
upload_type, filetype = self.pick_filetype()
size = random.randint(1024, 2048)
content = factory.make_string(size).encode('utf-8')
upload_name = factory.make_name('filename')
uploaded_file = SimpleUploadedFile(content=content, name=upload_name)
data = {
'name': name,
'architecture': architecture,
'filetype': upload_type,
'keep_old': True,
}
form = BootResourceForm(data=data, files={'content': uploaded_file})
self.assertTrue(form.is_valid(), form._errors)
form.save()
resource = reload_object(resource)
resource_set = resource.sets.order_by('id').last()
rfile = resource_set.files.first()
self.assertTrue(filetype, rfile.filetype)
self.assertTrue(filetype, rfile.filename)
self.assertTrue(size, rfile.largefile.total_size)
with rfile.largefile.content.open('rb') as stream:
written_content = stream.read()
self.assertEqual(content, written_content)
self.assertEqual(resource.rtype, BOOT_RESOURCE_TYPE.UPLOADED)
def make_os_with_license_key(self,
osystem=None,
osystem_title=None,
release=None):
"""Makes a fake operating system that has a release that requires a
license key."""
if osystem is None:
osystem = factory.make_name("osystem")
if osystem_title is None:
osystem_title = osystem + "_title"
if release is None:
release = random.choice(REQUIRE_LICENSE_KEY)
distro_series = "%s/%s" % (osystem, release)
drv = WindowsOS()
drv.title = osystem_title
OperatingSystemRegistry.register_item(osystem, drv)
factory.make_BootResource(
name=distro_series,
rtype=BOOT_RESOURCE_TYPE.UPLOADED,
extra={"title": drv.get_release_title(release)},
)
self.addCleanup(OperatingSystemRegistry.unregister_item, osystem)
return (
{
"name": osystem,
"title": osystem_title
},
{
"name": release,
"title": drv.get_release_title(release)
},
)
def test_removes_old_bootresourcefiles(self):
# Regression test for LP:1660418
os = factory.make_name('os')
series = factory.make_name('series')
OperatingSystemRegistry.register_item(os, CustomOS())
self.addCleanup(OperatingSystemRegistry.unregister_item, os)
name = '%s/%s' % (os, series)
architecture = make_usable_architecture(self)
resource = factory.make_usable_boot_resource(
rtype=BOOT_RESOURCE_TYPE.UPLOADED,
name=name,
architecture=architecture)
upload_type, filetype = self.pick_filetype()
size = random.randint(1024, 2048)
content = factory.make_string(size).encode('utf-8')
upload_name = factory.make_name('filename')
uploaded_file = SimpleUploadedFile(content=content, name=upload_name)
data = {
'name': name,
'architecture': architecture,
'filetype': upload_type,
}
form = BootResourceForm(data=data, files={'content': uploaded_file})
self.assertTrue(form.is_valid(), form._errors)
form.save()
self.assertEqual(
1,
BootResourceFile.objects.filter(
resource_set__resource=resource).count())
def make_osystem_requiring_license_key(testcase, osystem=None, release=None):
if osystem is None:
osystem = factory.make_name('osystem')
if release is None:
release = random.choice(REQUIRE_LICENSE_KEY)
distro_series = '%s/%s' % (osystem, release)
drv = WindowsOS()
drv.title = osystem
OperatingSystemRegistry.register_item(osystem, drv)
factory.make_BootResource(
name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED,
extra={'title': drv.get_release_title(release)})
testcase.addCleanup(
OperatingSystemRegistry.unregister_item, osystem)
return {
'name': osystem,
'title': osystem,
'default_release': release,
'default_commissioning_release': None,
'releases': [{
'name': distro_series,
'title': drv.get_release_title(release),
'requires_license_key': True,
'can_commission': False,
}]
}
def make_os(testcase):
osystem = factory.make_name('osystem')
release = random.choice(REQUIRE_LICENSE_KEY)
distro_series = '%s/%s' % (osystem, release)
drv = WindowsOS()
drv.title = osystem
OperatingSystemRegistry.register_item(osystem, drv)
factory.make_BootResource(
name=distro_series, rtype=BOOT_RESOURCE_TYPE.UPLOADED,
extra={'title': drv.get_release_title(release)})
testcase.addCleanup(
OperatingSystemRegistry.unregister_item, osystem)
return osystem, release
def list_all_usable_osystems(releases=None):
"""Return all operating systems that can be used for nodes."""
if releases is None:
releases = list_all_usable_releases()
osystems = []
for os_name, releases in releases.items():
osystem = OperatingSystemRegistry.get_item(os_name)
if osystem:
default_commissioning_release = (
osystem.get_default_commissioning_release()
)
default_release = osystem.get_default_release()
title = osystem.title
else:
default_commissioning_release = ""
default_release = ""
title = os_name
osystems.append(
{
"name": os_name,
"title": title,
"default_commissioning_release": default_commissioning_release,
"default_release": default_release,
"releases": releases,
}
)
return sorted(osystems, key=itemgetter("title"))
def test_prevents_reversed_osystem_from_driver(self):
reserved_name = factory.make_name('name')
OperatingSystemRegistry.register_item(reserved_name, CustomOS())
upload_type, filetype = self.pick_filetype()
size = random.randint(1024, 2048)
content = factory.make_string(size).encode('utf-8')
upload_name = factory.make_name('filename')
uploaded_file = SimpleUploadedFile(content=content, name=upload_name)
data = {
'name': reserved_name,
'title': factory.make_name('title'),
'architecture': make_usable_architecture(self),
'filetype': upload_type,
}
form = BootResourceForm(data=data, files={'content': uploaded_file})
self.assertFalse(form.is_valid())
def test_settings_shows_license_keys_if_OS_supporting_keys(self):
self.client.login(user=factory.make_admin())
osystem = factory.make_name('osystem')
release = random.choice(REQUIRE_LICENSE_KEY)
distro_series = '%s/%s' % (osystem, release)
drv = WindowsOS()
drv.title = osystem
OperatingSystemRegistry.register_item(osystem, drv)
factory.make_BootResource(
name=distro_series,
rtype=BOOT_RESOURCE_TYPE.UPLOADED,
extra={'title': drv.get_release_title(release)})
self.addCleanup(OperatingSystemRegistry.unregister_item, osystem)
response = self.client.get(reverse('settings_general'))
doc = fromstring(response.content)
license_keys = doc.cssselect('a[href="/MAAS/settings/license-keys/"]')
self.assertEqual(1, len(license_keys),
"Didn't show the license key section.")
def extract_image_params(path, maas_meta):
"""Represent a list of TFTP path elements as a list of boot-image dicts.
:param path: Tuple or list that consists of a full [osystem, architecture,
subarchitecture, release] that identify a kind of boot for which we
may need an image.
:param maas_meta: Contents of the maas.meta file. This may be an
empty string.
:return: A list of dicts, each of which may also include additional
items of meta-data that are not elements in the path, such as
"subarches".
"""
if path[0] == "bootloader":
osystem, release, arch = path
subarch = "generic"
label = "*"
else:
osystem, arch, subarch, release, label = path
osystem_obj = OperatingSystemRegistry.get_item(osystem, default=None)
if osystem_obj is None:
return []
purposes = osystem_obj.get_boot_image_purposes(arch, subarch, release,
label)
# Expand the path into a list of dicts, one for each boot purpose.
params = []
for purpose in purposes:
image = dict(
osystem=osystem,
architecture=arch,
subarchitecture=subarch,
release=release,
label=label,
purpose=purpose,
)
if (purpose == BOOT_IMAGE_PURPOSE.XINSTALL
or purpose == BOOT_IMAGE_PURPOSE.EPHEMERAL):
xinstall_path, xinstall_type = osystem_obj.get_xinstall_parameters(
arch, subarch, release, label)
image["xinstall_path"] = xinstall_path
image["xinstall_type"] = xinstall_type
else:
image["xinstall_path"] = ""
image["xinstall_type"] = ""
params.append(image)
# Merge in the meta-data.
for image_dict in params:
metadata = extract_metadata(maas_meta, image_dict)
image_dict.update(metadata)
return params
def make_osystem(testcase, osystem, purpose=None, releases=None):
"""Makes the operating system class and registers it."""
if osystem not in OperatingSystemRegistry:
fake = FakeOS(osystem, purpose, releases)
OperatingSystemRegistry.register_item(fake.name, fake)
testcase.addCleanup(OperatingSystemRegistry.unregister_item, osystem)
return fake
else:
obj = OperatingSystemRegistry[osystem]
old_func = obj.get_boot_image_purposes
testcase.patch(obj, "get_boot_image_purposes").return_value = purpose
def reset_func(obj, old_func):
obj.get_boot_image_purposes = old_func
testcase.addCleanup(reset_func, obj, old_func)
return obj
def list_all_usable_releases():
"""Return all releases for all operating systems that can be used."""
distro_series = {}
seen_releases = set()
for br in BootResource.objects.filter(bootloader_type=None):
# An OS can have multiple boot resource for one release. e.g Ubuntu
# Bionic has ga-18.04 and ga-18.04-lowlatency. This list should only
# contain one entry per OS.
if br.name in seen_releases:
continue
seen_releases.add(br.name)
if "/" in br.name:
os_name, release = br.name.split("/")
else:
os_name = "custom"
release = br.name
osystem = OperatingSystemRegistry.get_item(os_name)
if osystem is not None:
title = osystem.get_release_title(release)
if title is None:
title = release
can_commission = (
release in osystem.get_supported_commissioning_releases())
requires_license_key = osystem.requires_license_key(release)
else:
title = br.name
can_commission = requires_license_key = False
if br.rtype == BOOT_RESOURCE_TYPE.UPLOADED:
# User may set the title of an uploaded resource.
if "title" in br.extra:
title = br.extra["title"]
else:
title = release
if os_name not in distro_series:
distro_series[os_name] = []
distro_series[os_name].append({
"name":
release,
"title":
title,
"can_commission":
can_commission,
"requires_license_key":
requires_license_key,
})
for osystem, releases in distro_series.items():
distro_series[osystem] = sorted(releases, key=itemgetter("title"))
return OrderedDict(sorted(distro_series.items()))
def get_boot_image(self, params, client, remote_ip):
"""Get the boot image for the params on this rack controller.
Calls `MarkNodeFailed` for the machine if its a known machine.
"""
is_ephemeral = False
try:
osystem_obj = OperatingSystemRegistry.get_item(params['osystem'],
default=None)
purposes = osystem_obj \
.get_boot_image_purposes(params["arch"], params["subarch"],
params.get("release", ""),
params.get("label", ""))
if "ephemeral" in purposes:
is_ephemeral = True
except:
pass
system_id = params.pop("system_id", None)
if params["purpose"] == "local" and not is_ephemeral:
# Local purpose doesn't use a boot image so jsut set the label
# to "local", but this value will no be used.
params["label"] = "local"
return params
else:
if params["purpose"] == "local" and is_ephemeral:
params["purpose"] = "ephemeral"
boot_image = get_boot_image(params)
if boot_image is None:
# No matching boot image.
description = "Missing boot image %s/%s/%s/%s." % (
params['osystem'], params["arch"], params["subarch"],
params["release"])
# Call MarkNodeFailed if this was a known machine.
if system_id is not None:
d = client(MarkNodeFailed,
system_id=system_id,
error_description=description)
d.addErrback(
log.err,
"Failed to mark machine failed: %s" % description)
else:
maaslog.error(
"Enlistment failed to boot %s; missing required boot "
"image %s/%s/%s/%s." %
(remote_ip, params['osystem'], params["arch"],
params["subarch"], params["release"]))
params["label"] = "no-such-image"
else:
params["label"] = boot_image["label"]
return params
def clean(self):
"""Validate the model.
Checks that the name is in a valid format, for its type.
"""
if self.rtype == BOOT_RESOURCE_TYPE.UPLOADED:
if "/" in self.name:
os_name = self.name.split("/")[0]
osystem = OperatingSystemRegistry.get_item(os_name)
if osystem is None:
raise ValidationError(
"%s boot resource cannot contain a '/' in it's name "
"unless it starts with a supported operating system." %
(self.display_rtype))
elif self.rtype in RTYPE_REQUIRING_OS_SERIES_NAME:
if "/" not in self.name:
raise ValidationError(
"%s boot resource must contain a '/' in it's name." %
(self.display_rtype))
def get_boot_image(self, params, client, remote_ip):
"""Get the boot image for the params on this rack controller.
Calls `MarkNodeFailed` for the machine if its a known machine.
"""
is_ephemeral = False
try:
osystem_obj = OperatingSystemRegistry.get_item(
params["osystem"], default=None
)
purposes = osystem_obj.get_boot_image_purposes(
params["arch"],
params["subarch"],
params.get("release", ""),
params.get("label", ""),
)
if "ephemeral" in purposes:
is_ephemeral = True
except Exception:
pass
# Check to see if the we are PXE booting a device.
if params["purpose"] == "local-device":
mac = network.find_mac_via_arp(remote_ip)
log.info(
"Device %s with MAC address %s is PXE booting; "
"instructing the device to boot locally."
% (params["hostname"], mac)
)
# Set purpose back to local now that we have the message logged.
params["purpose"] = "local"
system_id = params.pop("system_id", None)
if params["purpose"] == "local" and not is_ephemeral:
# Local purpose doesn't use a boot image so just set the label
# to "local".
params["label"] = "local"
return params
else:
if params["purpose"] == "local" and is_ephemeral:
params["purpose"] = "ephemeral"
boot_image = get_boot_image(params)
if boot_image is None:
# No matching boot image.
description = "Missing boot image %s/%s/%s/%s." % (
params["osystem"],
params["arch"],
params["subarch"],
params["release"],
)
# Call MarkNodeFailed if this was a known machine.
if system_id is not None:
d = client(
MarkNodeFailed,
system_id=system_id,
error_description=description,
)
d.addErrback(
log.err,
"Failed to mark machine failed: %s" % description,
)
else:
maaslog.error(
"Enlistment failed to boot %s; missing required boot "
"image %s/%s/%s/%s."
% (
remote_ip,
params["osystem"],
params["arch"],
params["subarch"],
params["release"],
)
)
params["label"] = "no-such-image"
else:
params["label"] = boot_image["label"]
return params
def validate_hwe_kernel(
hwe_kernel,
min_hwe_kernel,
architecture,
osystem,
distro_series,
commissioning_osystem=undefined,
commissioning_distro_series=undefined,
):
"""Validates that hwe_kernel works on the selected os/release/arch.
Checks that the current hwe_kernel is avalible for the selected
os/release/architecture combination, and that the selected hwe_kernel is >=
min_hwe_kernel. If no hwe_kernel is selected one will be chosen.
"""
def validate_kernel_str(kstr):
return kstr.startswith("hwe-") or kstr.startswith("ga-")
if not all((osystem, architecture, distro_series)):
return hwe_kernel
# If we are dealing with an ephemeral image, just return the hwe_kernel
# as-is, i.e. just stick with generic.
osystem_obj = OperatingSystemRegistry.get_item(osystem, default=None)
if osystem_obj is not None:
arch, subarch = architecture.split("/")
purposes = osystem_obj.get_boot_image_purposes(arch, subarch,
distro_series, "*")
if "ephemeral" in purposes:
return hwe_kernel
# If we're not deploying Ubuntu we are just setting the kernel to be used
# during deployment
if osystem != "ubuntu":
osystem = commissioning_osystem
if osystem is undefined:
osystem = Config.objects.get_config("commissioning_osystem")
distro_series = commissioning_distro_series
if distro_series is undefined:
distro_series = Config.objects.get_config(
"commissioning_distro_series")
arch, subarch = architecture.split("/")
if subarch != "generic" and (
(hwe_kernel and validate_kernel_str(hwe_kernel)) or
(min_hwe_kernel and validate_kernel_str(min_hwe_kernel))):
raise ValidationError(
"Subarchitecture(%s) must be generic when setting hwe_kernel." %
subarch)
os_release = osystem + "/" + distro_series
if hwe_kernel and validate_kernel_str(hwe_kernel):
usable_kernels = BootResource.objects.get_usable_hwe_kernels(
os_release, arch)
if hwe_kernel not in usable_kernels:
raise ValidationError("%s is not available for %s on %s." %
(hwe_kernel, os_release, architecture))
if not release_a_newer_than_b(hwe_kernel, distro_series):
raise ValidationError("%s is too old to use on %s." %
(hwe_kernel, os_release))
if (min_hwe_kernel and validate_kernel_str(min_hwe_kernel)) and (
not release_a_newer_than_b(hwe_kernel, min_hwe_kernel)):
raise ValidationError(
"hwe_kernel(%s) is older than min_hwe_kernel(%s)." %
(hwe_kernel, min_hwe_kernel))
return hwe_kernel
elif min_hwe_kernel and validate_kernel_str(min_hwe_kernel):
# Determine what kflavor is being used by check against a list of
# known kflavors.
valid_kflavors = {
br.kflavor
for br in BootResource.objects.exclude(kflavor=None)
}
kflavor = "generic"
for kernel_part in min_hwe_kernel.split("-"):
if kernel_part in valid_kflavors:
kflavor = kernel_part
break
usable_kernels = BootResource.objects.get_usable_hwe_kernels(
os_release, arch, kflavor)
for i in usable_kernels:
if release_a_newer_than_b(
i, min_hwe_kernel) and release_a_newer_than_b(
i, distro_series):
return i
raise ValidationError(
"%s has no kernels available which meet min_hwe_kernel(%s)." %
(distro_series, min_hwe_kernel))
for kernel in BootResource.objects.get_usable_hwe_kernels(
os_release, arch, "generic"):
if release_a_newer_than_b(kernel, distro_series):
return kernel
raise ValidationError("%s has no kernels available." % distro_series)
def test_operating_system_registry(self):
self.assertItemsEqual([], OperatingSystemRegistry)
OperatingSystemRegistry.register_item("resource", sentinel.resource)
self.assertIn(sentinel.resource,
(item for name, item in OperatingSystemRegistry))
