def write_all_keyrings(directory, sources):
"""For a given set of `sources`, write the keyrings to disk.
:param directory: A directory where the key files should be written. Use
a dedicated temporary directory for this, and clean it up when done.
:param sources: An iterable of the sources whose keyrings need to be
written.
:return: The sources iterable, with each source whose keyring has
been written now having a "keyring" value set, pointing to the file
on disk.
"""
for source in sources:
source_url = source.get("url")
keyring_file = source.get("keyring")
keyring_data = source.get("keyring_data")
if keyring_file is not None and keyring_data is not None:
maaslog.warning(
"Both a keyring file and keyring data were specified; "
"ignoring the keyring file."
)
if keyring_data is not None:
keyring_file = os.path.join(
directory, calculate_keyring_name(source_url)
)
write_keyring(keyring_file, keyring_data)
source["keyring"] = keyring_file
return sources
def update_targets_conf(snapshot):
"""Runs tgt-admin to update the new targets from "maas.tgt"."""
# Ensure that tgt is running before tgt-admin is used.
service_monitor.ensureService("tgt").wait(30)
# Update the tgt config.
targets_conf = os.path.join(snapshot, 'maas.tgt')
# The targets_conf may not exist in the event the BootSource is broken
# and images havn't been imported yet. This fixes LP:1655721
if not os.path.exists(targets_conf):
return
try:
call_and_check(
sudo([
get_path('/usr/sbin/tgt-admin'),
'--conf',
targets_conf,
'--update',
'ALL',
]))
except ExternalProcessError as e:
msg = "Unable to update TGT config: %s" % e
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
def insert_item(self, data, src, target, pedigree, contentsource):
"""Overridable from `BasicMirrorWriter`."""
item = products_exdata(src, pedigree)
if self.validate_products and not validate_product(item, pedigree[0]):
maaslog.warning("Ignoring unsupported product %s" % pedigree[0])
return
os = get_os_from_product(item)
arch = item["arch"]
subarches = item.get("subarches", "generic")
if item.get("bootloader-type") is None:
release = item["release"]
kflavor = item.get("kflavor", "generic")
else:
release = item["bootloader-type"]
kflavor = "bootloader"
label = item["label"]
base_image = ImageSpec(os, arch, None, kflavor, release, label)
compact_item = clean_up_repo_item(item)
if os == "ubuntu-core":
# For Ubuntu Core we only want one entry per release/arch/gadget
gadget = item.get("gadget_snap", "generic")
kflavor = item.get("kernel_snap", "generic")
release = "%s-%s" % (release, gadget)
self.boot_images_dict.setdefault(
base_image._replace(
subarch="generic", kflavor=kflavor, release=release
),
compact_item,
)
else:
for subarch in subarches.split(","):
self.boot_images_dict.setdefault(
base_image._replace(subarch=subarch), compact_item
)
# HWE resources need to map to a specfic resource, and not just to
# any of the supported subarchitectures for that resource.
subarch = item.get("subarch", "generic")
self.boot_images_dict.set(
base_image._replace(subarch=subarch), compact_item
)
if os == "ubuntu" and item.get("version") is not None:
# HWE resources with generic, should map to the HWE that ships
# with that release. Starting with Xenial kernels changed from
# using the naming format hwe-<letter> to ga-<version>. Look
# for both.
hwe_archs = ["ga-%s" % item["version"], "hwe-%s" % release[0]]
if subarch in hwe_archs and "generic" in subarches:
self.boot_images_dict.set(
base_image._replace(subarch="generic"), compact_item
)
def sync(self, reader, path):
try:
super(RepoDumper, self).sync(reader, path)
except IOError:
maaslog.warning(
"I/O error while syncing boot images. If this problem "
"persists, verify network connectivity and disk usage.")
# This MUST NOT suppress the I/O error because callers use
# self.boot_images_dict as the "return" value. Suppressing
# exceptions here gives the caller no reason to doubt that
# boot_images_dict is not utter garbage and so pass it up the
# stack where it is then acted upon, to empty out BootSourceCache
# for example. True story.
raise
def sync(self, reader, path):
try:
super().sync(reader, path)
except IOError:
maaslog.warning(
"I/O error while syncing boot images. If this problem "
"persists, verify network connectivity and disk usage.")
# This MUST NOT suppress the I/O error because callers use
# self.boot_images_dict as the "return" value. Suppressing
# exceptions here gives the caller no reason to doubt that
# boot_images_dict is not utter garbage and so pass it up the
# stack where it is then acted upon, to empty out BootSourceCache
# for example. True story.
raise
except sutil.SignatureMissingException as error:
# Handle this error here so we can log for both the region and rack
# have been unable to use simplestreams.
maaslog.error(
"Failed to download image descriptions with Simplestreams "
"(%s). Verify network connectivity." % error)
raise
def import_images(sources):
"""Import images. Callable from the command line.
:param config: An iterable of dicts representing the sources from
which boot images will be downloaded.
"""
if len(sources) == 0:
msg = "Can't import: region did not provide a source."
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
return False
msg = "Starting rack boot image import"
maaslog.info(msg)
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
with ClusterConfiguration.open() as config:
storage = FilePath(config.tftp_root).parent().path
with tempdir("keyrings") as keyrings_path:
# XXX: Band-aid to ensure that the keyring_data is bytes. Future task:
# try to figure out why this sometimes happens.
for source in sources:
if "keyring_data" in source and not isinstance(
source["keyring_data"], bytes):
source["keyring_data"] = source["keyring_data"].encode("utf-8")
# We download the keyrings now because we need them for both
# download_all_image_descriptions() and
# download_all_boot_resources() later.
sources = write_all_keyrings(keyrings_path, sources)
# The region produces a SimpleStream which is similar, but not
# identical to the actual SimpleStream. These differences cause
# validation to fail. So grab everything from the region and trust it
# did proper filtering before the rack.
image_descriptions = download_all_image_descriptions(
sources, validate_products=False)
if image_descriptions.is_empty():
msg = ("Finished importing boot images, the region does not have "
"any boot images available.")
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
return False
meta_file_content = image_descriptions.dump_json()
if meta_contains(storage, meta_file_content):
maaslog.info("Finished importing boot images, the region does not "
"have any new images.")
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
maaslog.info(msg)
return False
product_mapping = map_products(image_descriptions)
try:
snapshot_path = download_all_boot_resources(
sources, storage, product_mapping)
except Exception as e:
try_send_rack_event(
EVENT_TYPES.RACK_IMPORT_ERROR,
"Unable to import boot images: %s" % e,
)
maaslog.error(
"Unable to import boot images; cleaning up failed snapshot "
"and cache.")
# Cleanup snapshots and cache since download failed.
cleanup_snapshots_and_cache(storage)
raise
maaslog.info("Writing boot image metadata.")
write_snapshot_metadata(snapshot_path, meta_file_content)
maaslog.info("Linking boot images snapshot %s" % snapshot_path)
link_bootloaders(snapshot_path)
# If we got here, all went well. This is now truly the "current" snapshot.
update_current_symlink(storage, snapshot_path)
# Now cleanup the old snapshots and cache.
maaslog.info("Cleaning up old snapshots and cache.")
cleanup_snapshots_and_cache(storage)
# Import is now finished.
msg = "Finished importing boot images."
maaslog.info(msg)
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
return True
