def generate_omapi_key():
"""Generate a HMAC-MD5 key by calling out to the dnssec-keygen tool.
:return: The shared key suitable for OMAPI access.
:type: string
"""
# dnssec-keygen writes out files to a specified directory, so we
# need to make a temp directory for that.
# This relies on the temporary directory being accessible only to its
# owner.
temp_prefix = "%s." % os.path.basename(__file__)
with tempdir(prefix=temp_prefix) as tmpdir:
key = run_repeated_keygen(tmpdir)
return key
def test_bootloader_only_allows_one_subarch(self):
with tempdir() as snapshot_path:
tag, links = self.make_files(snapshot_path)
osystem = factory.make_name('osystem')
arch = factory.make_name('arch')
release = factory.make_name('release')
label = factory.make_name('label')
subarches = [factory.make_name('subarch') for _ in range(3)]
bootloader_type = factory.make_name('bootloader-type')
self.assertRaises(AssertionError,
download_resources.link_resources, snapshot_path,
links, osystem, arch, release, label, subarches,
bootloader_type)
def test_link_bootloader_copies_previous_downloaded_files(self):
method = FakeBootMethod()
with tempdir() as tmp:
new_dir = os.path.join(tmp, "new")
current_dir = os.path.join(tmp, "current")
os.makedirs(new_dir)
os.makedirs(current_dir)
for bootloader_file in method.bootloader_files:
factory.make_file(current_dir, bootloader_file)
method.link_bootloader(new_dir)
for bootloader_file in method.bootloader_files:
bootloader_file_path = os.path.join(new_dir, bootloader_file)
self.assertTrue(os.path.isfile(bootloader_file_path))
def test_link_bootloader_links_simplestream_bootloader_files(self):
method = FakeBootMethod()
with tempdir() as tmp:
stream_path = os.path.join(tmp, 'bootloader',
method.bios_boot_method,
method.bootloader_arches[0])
os.makedirs(stream_path)
for bootloader_file in method.bootloader_files:
factory.make_file(stream_path, bootloader_file)
method.link_bootloader(tmp)
for bootloader_file in method.bootloader_files:
bootloader_file_path = os.path.join(tmp, bootloader_file)
self.assertTrue(os.path.islink(bootloader_file_path))
def test_link_bootloader_logs_missing_previous_downloaded_files(self):
method = FakeBootMethod()
mock_maaslog = self.patch(maaslog, "error")
mock_try_send_rack_event = self.patch(boot, "try_send_rack_event")
with tempdir() as tmp:
new_dir = os.path.join(tmp, "new")
current_dir = os.path.join(tmp, "current")
os.makedirs(new_dir)
os.makedirs(current_dir)
for bootloader_file in method.bootloader_files[1:]:
factory.make_file(current_dir, bootloader_file)
method.link_bootloader(new_dir)
self.assertThat(mock_maaslog, MockCalledOnce())
self.assertThat(mock_try_send_rack_event, MockCalledOnce())
def test_link_bootloader_logs_missing_simplestream_file(self):
method = FakeBootMethod()
mock_maaslog = self.patch(maaslog, 'error')
mock_try_send_rack_event = self.patch(boot, 'try_send_rack_event')
with tempdir() as tmp:
stream_path = os.path.join(tmp, 'bootloader',
method.bios_boot_method,
method.bootloader_arches[0])
os.makedirs(stream_path)
for bootloader_file in method.bootloader_files[1:]:
factory.make_file(stream_path, bootloader_file)
method.link_bootloader(tmp)
self.assertThat(mock_maaslog, MockCalledOnce())
self.assertThat(mock_try_send_rack_event, MockCalledOnce())
def test_link_bootloader_creates_grub_cfg(self):
method = UEFIAMD64BootMethod()
with tempdir() as tmp:
stream_path = os.path.join(tmp, 'bootloader',
method.bios_boot_method,
method.bootloader_arches[0])
os.makedirs(stream_path)
for bootloader_file in method.bootloader_files:
factory.make_file(stream_path, bootloader_file)
method.link_bootloader(tmp)
for bootloader_file in method.bootloader_files:
bootloader_file_path = os.path.join(tmp, bootloader_file)
self.assertTrue(os.path.islink(bootloader_file_path))
grub_file_path = os.path.join(tmp, 'grub', 'grub.cfg')
self.assertTrue(grub_file_path, FileContains(CONFIG_FILE))
def test_link_simplestream_bootloaders_creates_lpxelinux_and_links(self):
method = PXEBootMethod()
with tempdir() as tmp:
stream_path = os.path.join(
tmp,
"bootloader",
method.bios_boot_method,
method.bootloader_arches[0],
)
os.makedirs(stream_path)
for bootloader_file in method.bootloader_files:
factory.make_file(stream_path, bootloader_file)
method.link_bootloader(tmp)
self.assertTrue(os.path.exists(os.path.join(tmp, "lpxelinux.0")))
self.assertTrue(os.path.islink(os.path.join(tmp, "pxelinux.0")))
def test_link_simplestream_bootloaders_creates_syslinux_link(self):
method = PXEBootMethod()
with tempdir() as tmp:
stream_path = os.path.join(tmp, 'bootloader',
method.bios_boot_method,
method.bootloader_arches[0])
os.makedirs(stream_path)
for bootloader_file in method.bootloader_files:
factory.make_file(stream_path, bootloader_file)
method.link_bootloader(tmp)
for bootloader_file in method.bootloader_files:
bootloader_file_path = os.path.join(tmp, bootloader_file)
self.assertTrue(os.path.islink(bootloader_file_path))
syslinux_link = os.path.join(tmp, 'syslinux')
self.assertTrue(os.path.islink(syslinux_link))
self.assertEquals(stream_path, os.path.realpath(syslinux_link))
def test_link_bootloader_links_bootloaders_found_elsewhere_on_fs(self):
method = FakeBootMethod()
with tempdir() as tmp:
bootresources_dir = os.path.join(tmp, 'boot-resources')
new_dir = os.path.join(bootresources_dir, 'new')
current_dir = os.path.join(bootresources_dir, 'current')
os.makedirs(new_dir)
os.makedirs(current_dir)
for bootloader_file in method.bootloader_files:
factory.make_file(tmp, bootloader_file)
atomic_symlink(os.path.join(tmp, bootloader_file),
os.path.join(current_dir, bootloader_file))
method.link_bootloader(new_dir)
for bootloader_file in method.bootloader_files:
bootloader_file_path = os.path.join(new_dir, bootloader_file)
self.assertTrue(os.path.islink(bootloader_file_path))
def test_extracts_files(self):
with tempdir() as cache_dir:
store = FileStore(cache_dir)
tar_xz, files = self.make_tar_xz(cache_dir)
sha256, size = self.get_file_info(tar_xz)
checksums = {'sha256': sha256}
with open(tar_xz, 'rb') as f:
content_source = ChecksummingContentSource(f, checksums, size)
cached_files = download_resources.extract_archive_tar(
store, os.path.basename(tar_xz), sha256, checksums, size,
content_source)
for f, info in files.items():
cached_file = os.path.join(cache_dir,
'%s-%s' % (f, sha256))
expected_cached_file = (cached_file, f)
self.assertIn(expected_cached_file, cached_files)
self.assertTrue(os.path.exists(cached_file))
self.assertEqual(info, self.get_file_info(cached_file))
def make_tar_xz(self, path):
tar_xz_path = os.path.join(path,
factory.make_name('archive') + '.tar.xz')
files = {}
with tarfile.open(tar_xz_path, 'w:xz') as tar:
with tempdir() as tmp:
for _ in range(3):
f = factory.make_file(tmp)
tar_path = os.path.basename(f)
tar.add(f, tar_path)
files[tar_path] = self.get_file_info(f)
subdir = os.path.join(tmp, 'subdir')
os.makedirs(subdir)
for _ in range(3):
f = factory.make_file(subdir)
tar_path = f[len(tmp) + 1:]
tar.add(f, tar_path)
files[tar_path] = self.get_file_info(f)
return tar_xz_path, files
def test_links_bootloader(self):
with tempdir() as snapshot_path:
tag, links = self.make_files(snapshot_path)
osystem = factory.make_name('osystem')
arch = factory.make_name('arch')
release = factory.make_name('release')
label = factory.make_name('label')
subarches = [factory.make_name('subarch')]
bootloader_type = factory.make_name('bootloader-type')
download_resources.link_resources(snapshot_path, links, osystem,
arch, release, label, subarches,
bootloader_type)
for cached_file, logical_name in links:
cached_file_path = os.path.join(snapshot_path, cached_file)
logical_name_path = os.path.join(snapshot_path, 'bootloader',
bootloader_type, arch)
self.assertTrue(os.path.exists(cached_file_path))
self.assertTrue(os.path.exists(logical_name_path))
def test_links_resources(self):
with tempdir() as snapshot_path:
tag, links = self.make_files(snapshot_path)
osystem = factory.make_name('osystem')
arch = factory.make_name('arch')
release = factory.make_name('release')
label = factory.make_name('label')
subarches = [factory.make_name('subarch') for _ in range(3)]
download_resources.link_resources(snapshot_path, links, osystem,
arch, release, label, subarches,
None)
for subarch in subarches:
for cached_file, logical_name in links:
cached_file_path = os.path.join(snapshot_path, cached_file)
logical_name_path = os.path.join(snapshot_path, osystem,
arch, subarch, release,
label, logical_name)
self.assertTrue(os.path.exists(cached_file_path))
self.assertTrue(os.path.exists(logical_name_path))
def test_returns_files_from_cache(self):
with tempdir() as cache_dir:
store = FileStore(cache_dir)
tar_xz, files = self.make_tar_xz(cache_dir)
sha256, size = self.get_file_info(tar_xz)
checksums = {'sha256': sha256}
with open(tar_xz, 'rb') as f:
content_source = ChecksummingContentSource(f, checksums, size)
download_resources.extract_archive_tar(
store, os.path.basename(tar_xz), sha256, checksums, size,
content_source)
mocked_tar = self.patch(download_resources.tarfile, 'open')
cached_files = download_resources.extract_archive_tar(
store, os.path.basename(tar_xz), sha256, checksums, size,
content_source)
self.assertThat(mocked_tar, MockNotCalled())
for f, info in files.items():
cached_file = os.path.join(cache_dir,
'%s-%s' % (f, sha256))
expected_cached_file = (cached_file, f)
self.assertIn(expected_cached_file, cached_files)
def test_link_bootloader_copies_previously_downloaded_files(self):
method = PXEBootMethod()
with tempdir() as tmp:
new_dir = os.path.join(tmp, 'new')
current_dir = os.path.join(tmp, 'current')
os.makedirs(new_dir)
os.makedirs(current_dir)
factory.make_file(current_dir, method.bootloader_files[0])
for bootloader_file in method.bootloader_files[1:]:
factory.make_file(current_dir, bootloader_file)
real_syslinux_dir = os.path.join(tmp, 'syslinux')
os.makedirs(real_syslinux_dir)
atomic_symlink(real_syslinux_dir,
os.path.join(current_dir, 'syslinux'))
method.link_bootloader(new_dir)
for bootloader_file in method.bootloader_files:
bootloader_file_path = os.path.join(new_dir, bootloader_file)
self.assertTrue(os.path.isfile(bootloader_file_path))
syslinux_link = os.path.join(new_dir, 'syslinux')
self.assertTrue(os.path.islink(syslinux_link))
self.assertEquals(real_syslinux_dir,
os.path.realpath(syslinux_link))
def test_cleans_up_on_successful_exit(self):
with tempdir() as directory:
file_path = factory.make_file(directory)
self.assertThat(directory, Not(DirExists()))
self.assertThat(file_path, Not(FileExists()))
def cache_boot_sources():
"""Cache all image information in boot sources.
Called from *outside* of a transaction this will:
1. Retrieve information about all boot sources from the database. The
transaction is committed before proceeding.
2. The boot sources are consulted (i.e. there's network IO now) and image
descriptions downloaded.
3. Update the boot source cache with the fetched information. If the boot
source has been modified or deleted during #2 then the results are
discarded.
This approach does not require an exclusive lock.
"""
# Nomenclature herein: `bootsource` is an ORM record for BootSource;
# `source` is one of those converted to a dict. The former ought not to be
# used outside of a transactional context.
@transactional
def get_sources():
return list(
bootsource.to_dict_without_selections()
for bootsource in BootSource.objects.all()
# TODO: Only where there are no corresponding BootSourceCache
# records or the BootSource's updated timestamp is later than any
# of the BootSourceCache records' timestamps.
)
@transactional
def update_cache(source, descriptions):
try:
bootsource = BootSource.objects.get(url=source["url"])
except BootSource.DoesNotExist:
# The record was deleted while we were fetching the description.
maaslog.debug(
"Image descriptions at %s are no longer needed; discarding.",
source["url"])
else:
if bootsource.compare_dict_without_selections(source):
# Only delete from the cache once we have the descriptions.
BootSourceCache.objects.filter(boot_source=bootsource).delete()
if not descriptions.is_empty():
for spec, item in descriptions.mapping.items():
title = get_product_title(item)
if title is None:
extra = {}
else:
extra = {'title': title}
BootSourceCache.objects.create(
boot_source=bootsource, os=spec.os,
arch=spec.arch, subarch=spec.subarch,
kflavor=spec.kflavor,
release=spec.release, label=spec.label,
release_codename=item.get('release_codename'),
release_title=item.get('release_title'),
support_eol=item.get('support_eol'),
bootloader_type=item.get('bootloader-type'),
extra=extra,
)
maaslog.debug(
"Image descriptions for %s have been updated.",
source["url"])
else:
maaslog.debug(
"Image descriptions for %s are outdated; discarding.",
source["url"])
@transactional
def check_commissioning_series_selected():
commissioning_osystem = Config.objects.get_config(
name='commissioning_osystem')
commissioning_series = Config.objects.get_config(
name='commissioning_distro_series')
qs = BootSourceSelection.objects.filter(
os=commissioning_osystem, release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident='commissioning_series_unselected').exists():
Notification.objects.create_error_for_users(
'%s %s is configured as the commissioning release but it '
'is not selected for download!' % (
commissioning_osystem, commissioning_series),
ident='commissioning_series_unselected')
qs = BootSourceCache.objects.filter(
os=commissioning_osystem, release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident='commissioning_series_unavailable').exists():
Notification.objects.create_error_for_users(
'%s %s is configured as the commissioning release but it '
'is unavailable in the configured streams!' % (
commissioning_osystem, commissioning_series),
ident='commissioning_series_unavailable')
# FIXME: This modifies the environment of the entire process, which is Not
# Cool. We should integrate with simplestreams in a more Pythonic manner.
yield deferToDatabase(set_simplestreams_env)
errors = []
sources = yield deferToDatabase(get_sources)
for source in sources:
with tempdir("keyrings") as keyrings_path:
[source] = write_all_keyrings(keyrings_path, [source])
try:
descriptions = download_all_image_descriptions(
[source],
user_agent=get_maas_version_user_agent())
except (IOError, ConnectionError) as error:
errors.append(
"Failed to import images from boot source "
"%s: %s" % (source["url"], error))
else:
yield deferToDatabase(update_cache, source, descriptions)
yield deferToDatabase(check_commissioning_series_selected)
maaslog.info("Updated boot sources cache.")
component = COMPONENT.REGION_IMAGE_IMPORT
if len(errors) > 0:
yield deferToDatabase(
register_persistent_error, component,
"<br>".join(map(html.escape, errors)))
else:
yield deferToDatabase(
discard_persistent_error, component)
def import_images(sources):
"""Import images. Callable from the command line.
:param config: An iterable of dicts representing the sources from
which boot images will be downloaded.
"""
if len(sources) == 0:
msg = "Can't import: region did not provide a source."
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
return False
msg = "Starting rack boot image import"
maaslog.info(msg)
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
with ClusterConfiguration.open() as config:
storage = FilePath(config.tftp_root).parent().path
with tempdir("keyrings") as keyrings_path:
# XXX: Band-aid to ensure that the keyring_data is bytes. Future task:
# try to figure out why this sometimes happens.
for source in sources:
if "keyring_data" in source and not isinstance(
source["keyring_data"], bytes):
source["keyring_data"] = source["keyring_data"].encode("utf-8")
# We download the keyrings now because we need them for both
# download_all_image_descriptions() and
# download_all_boot_resources() later.
sources = write_all_keyrings(keyrings_path, sources)
# The region produces a SimpleStream which is similar, but not
# identical to the actual SimpleStream. These differences cause
# validation to fail. So grab everything from the region and trust it
# did proper filtering before the rack.
image_descriptions = download_all_image_descriptions(
sources, validate_products=False)
if image_descriptions.is_empty():
msg = ("Finished importing boot images, the region does not have "
"any boot images available.")
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
return False
meta_file_content = image_descriptions.dump_json()
if meta_contains(storage, meta_file_content):
maaslog.info("Finished importing boot images, the region does not "
"have any new images.")
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
maaslog.info(msg)
return False
product_mapping = map_products(image_descriptions)
try:
snapshot_path = download_all_boot_resources(
sources, storage, product_mapping)
except Exception as e:
try_send_rack_event(
EVENT_TYPES.RACK_IMPORT_ERROR,
"Unable to import boot images: %s" % e,
)
maaslog.error(
"Unable to import boot images; cleaning up failed snapshot "
"and cache.")
# Cleanup snapshots and cache since download failed.
cleanup_snapshots_and_cache(storage)
raise
maaslog.info("Writing boot image metadata.")
write_snapshot_metadata(snapshot_path, meta_file_content)
maaslog.info("Linking boot images snapshot %s" % snapshot_path)
link_bootloaders(snapshot_path)
# If we got here, all went well. This is now truly the "current" snapshot.
update_current_symlink(storage, snapshot_path)
# Now cleanup the old snapshots and cache.
maaslog.info("Cleaning up old snapshots and cache.")
cleanup_snapshots_and_cache(storage)
# Import is now finished.
msg = "Finished importing boot images."
maaslog.info(msg)
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
return True
def test_restricts_access(self):
with tempdir() as directory:
mode = os.stat(directory).st_mode
self.assertEqual(
stat.S_IMODE(mode), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
)
def cache_boot_sources():
"""Cache all image information in boot sources.
Called from *outside* of a transaction this will:
1. Retrieve information about all boot sources from the database. The
transaction is committed before proceeding.
2. The boot sources are consulted (i.e. there's network IO now) and image
descriptions downloaded.
3. Update the boot source cache with the fetched information. If the boot
source has been modified or deleted during #2 then the results are
discarded.
This approach does not require an exclusive lock.
"""
# Nomenclature herein: `bootsource` is an ORM record for BootSource;
# `source` is one of those converted to a dict. The former ought not to be
# used outside of a transactional context.
@transactional
def get_sources():
return list(
bootsource.to_dict_without_selections()
for bootsource in BootSource.objects.all()
# TODO: Only where there are no corresponding BootSourceCache
# records or the BootSource's updated timestamp is later than any
# of the BootSourceCache records' timestamps.
)
@transactional
def check_commissioning_series_selected():
commissioning_osystem = Config.objects.get_config(
name="commissioning_osystem")
commissioning_series = Config.objects.get_config(
name="commissioning_distro_series")
qs = BootSourceSelection.objects.filter(os=commissioning_osystem,
release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident="commissioning_series_unselected").exists():
Notification.objects.create_error_for_users(
"%s %s is configured as the commissioning release but it "
"is not selected for download!" %
(commissioning_osystem, commissioning_series),
ident="commissioning_series_unselected",
)
qs = BootSourceCache.objects.filter(os=commissioning_osystem,
release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident="commissioning_series_unavailable").exists():
Notification.objects.create_error_for_users(
"%s %s is configured as the commissioning release but it "
"is unavailable in the configured streams!" %
(commissioning_osystem, commissioning_series),
ident="commissioning_series_unavailable",
)
@transactional
def get_proxy():
enabled = Config.objects.get_config("enable_http_proxy")
proxy = Config.objects.get_config("http_proxy")
if enabled and proxy:
return proxy
return False
# FIXME: This modifies the environment of the entire process, which is Not
# Cool. We should integrate with simplestreams in a more Pythonic manner.
yield deferToDatabase(set_simplestreams_env)
errors = []
sources = yield deferToDatabase(get_sources)
for source in sources:
with tempdir("keyrings") as keyrings_path:
[source] = write_all_keyrings(keyrings_path, [source])
try:
user_agent = yield deferToDatabase(get_maas_user_agent)
descriptions = download_all_image_descriptions(
[source], user_agent=user_agent)
except (IOError, ConnectionError) as error:
msg = "Failed to import images from " "%s: %s" % (
source["url"],
error,
)
errors.append(msg)
maaslog.error(msg)
except sutil.SignatureMissingException as error:
# Raise an error to the UI.
proxy = yield deferToDatabase(get_proxy)
if not proxy:
msg = ("Failed to import images from %s (%s). Verify "
"network connectivity and try again." %
(source["url"], error))
else:
msg = ("Failed to import images from %s (%s). Verify "
"network connectivity via your external "
"proxy (%s) and try again." %
(source["url"], error, proxy))
errors.append(msg)
else:
yield deferToDatabase(_update_cache, source, descriptions)
yield deferToDatabase(check_commissioning_series_selected)
component = COMPONENT.REGION_IMAGE_IMPORT
if len(errors) > 0:
maaslog.error("Unable to update boot sources cache.")
yield deferToDatabase(
register_persistent_error,
component,
"<br>".join(map(html.escape, errors)),
)
else:
maaslog.info("Updated boot sources cache.")
yield deferToDatabase(discard_persistent_error, component)
def test_uses_suffix(self):
suffix = factory.make_string(3)
with tempdir(suffix=suffix) as directory:
pass
self.assertThat(os.path.basename(directory), EndsWith(suffix))
def test_uses_prefix(self):
prefix = factory.make_string(3)
with tempdir(prefix=prefix) as directory:
pass
self.assertThat(os.path.basename(directory), StartsWith(prefix))
def test_yields_unicode(self):
with tempdir() as directory:
pass
self.assertIsInstance(directory, str)
def test_tolerates_disappearing_dir(self):
with tempdir() as directory:
rmtree(directory)
self.assertThat(directory, Not(DirExists()))
def test_creates_unique_directory(self):
with tempdir() as dir1, tempdir() as dir2:
pass
self.assertNotEqual(dir1, dir2)
