def fetch(self, params):
"""Fetch the releases and the arches from the provided source."""
# Must be administrator.
assert self.user.is_superuser, "Permission denied."
# Build a source, but its not saved into the database.
boot_source = self.get_bootsource(params, from_db=False)
try:
# Validate the boot source fields without committing it.
boot_source.clean_fields()
except ValidationError as error:
raise HandlerValidationError(error)
source = boot_source.to_dict_without_selections()
# FIXME: This modifies the environment of the entire process, which is
# Not Cool. We should integrate with simplestreams in a more
# Pythonic manner.
set_simplestreams_env()
with tempdir("keyrings") as keyrings_path:
[source] = write_all_keyrings(keyrings_path, [source])
try:
descriptions = download_all_image_descriptions(
[source], user_agent=get_maas_user_agent())
except Exception as error:
raise HandlerError(str(error))
items = list(descriptions.items())
err_msg = "Mirror provides no Ubuntu images."
if len(items) == 0:
raise HandlerError(err_msg)
releases = {}
arches = {}
for image_spec, product_info in items:
# Only care about Ubuntu images.
if image_spec.os != "ubuntu":
continue
releases[image_spec.release] = {
"name":
image_spec.release,
"title":
product_info.get(
"release_title",
format_ubuntu_distro_series(image_spec.release),
),
"checked":
False,
"deleted":
False,
}
arches[image_spec.arch] = {
"name": image_spec.arch,
"title": image_spec.arch,
"checked": False,
"deleted": False,
}
if len(releases) == 0 or len(arches) == 0:
raise HandlerError(err_msg)
return json.dumps({
"releases": list(releases.values()),
"arches": list(arches.values()),
})
def test_writes_keyring_data(self):
fake_write_keyring = self.patch(keyrings, "write_keyring")
sources = [{
"url": "http://%s" % self.getUniqueString(),
"keyring_data": factory.make_bytes(),
} for _ in range(5)]
keyring_path = self.make_dir()
keyrings.write_all_keyrings(keyring_path, sources)
expected_calls = (mock.call(
os.path.join(
keyring_path,
keyrings.calculate_keyring_name(source["url"]),
),
source["keyring_data"],
) for source in sources)
self.assertThat(fake_write_keyring, MockCallsMatch(*expected_calls))
def test_returns_sources(self):
self.patch(keyrings, "write_keyring")
sources = [{
"url": "http://%s" % self.getUniqueString(),
"keyring_data": factory.make_bytes(),
} for _ in range(5)]
keyring_path = self.make_dir()
expected_values = [
os.path.join(keyring_path,
keyrings.calculate_keyring_name(source["url"]))
for source in sources
]
returned_sources = keyrings.write_all_keyrings(keyring_path, sources)
actual_values = [source.get("keyring") for source in returned_sources]
self.assertEqual(expected_values, actual_values)
def test_ignores_existing_keyrings(self):
self.patch(keyrings, 'write_keyring')
fake_maaslog = self.patch(keyrings, 'maaslog')
source = {
'url': self.getUniqueString(),
'keyring': self.getUniqueString(),
'keyring_data': self.getUniqueString(),
}
keyring_path = self.make_dir()
[returned_source] = keyrings.write_all_keyrings(keyring_path, [source])
expected_keyring = os.path.join(
keyring_path, keyrings.calculate_keyring_name(source['url']))
self.assertEqual(expected_keyring, returned_source.get('keyring'))
self.assertThat(
fake_maaslog.warning,
MockCalledWith(
"Both a keyring file and keyring data were specified; "
"ignoring the keyring file."))
def cache_boot_sources():
"""Cache all image information in boot sources.
Called from *outside* of a transaction this will:
1. Retrieve information about all boot sources from the database. The
transaction is committed before proceeding.
2. The boot sources are consulted (i.e. there's network IO now) and image
descriptions downloaded.
3. Update the boot source cache with the fetched information. If the boot
source has been modified or deleted during #2 then the results are
discarded.
This approach does not require an exclusive lock.
"""
# Nomenclature herein: `bootsource` is an ORM record for BootSource;
# `source` is one of those converted to a dict. The former ought not to be
# used outside of a transactional context.
@transactional
def get_sources():
return list(
bootsource.to_dict_without_selections()
for bootsource in BootSource.objects.all()
# TODO: Only where there are no corresponding BootSourceCache
# records or the BootSource's updated timestamp is later than any
# of the BootSourceCache records' timestamps.
)
@transactional
def check_commissioning_series_selected():
commissioning_osystem = Config.objects.get_config(
name="commissioning_osystem")
commissioning_series = Config.objects.get_config(
name="commissioning_distro_series")
qs = BootSourceSelection.objects.filter(os=commissioning_osystem,
release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident="commissioning_series_unselected").exists():
Notification.objects.create_error_for_users(
"%s %s is configured as the commissioning release but it "
"is not selected for download!" %
(commissioning_osystem, commissioning_series),
ident="commissioning_series_unselected",
)
qs = BootSourceCache.objects.filter(os=commissioning_osystem,
release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident="commissioning_series_unavailable").exists():
Notification.objects.create_error_for_users(
"%s %s is configured as the commissioning release but it "
"is unavailable in the configured streams!" %
(commissioning_osystem, commissioning_series),
ident="commissioning_series_unavailable",
)
@transactional
def get_proxy():
enabled = Config.objects.get_config("enable_http_proxy")
proxy = Config.objects.get_config("http_proxy")
if enabled and proxy:
return proxy
return False
# FIXME: This modifies the environment of the entire process, which is Not
# Cool. We should integrate with simplestreams in a more Pythonic manner.
yield deferToDatabase(set_simplestreams_env)
errors = []
sources = yield deferToDatabase(get_sources)
for source in sources:
with tempdir("keyrings") as keyrings_path:
[source] = write_all_keyrings(keyrings_path, [source])
try:
user_agent = yield deferToDatabase(get_maas_user_agent)
descriptions = download_all_image_descriptions(
[source], user_agent=user_agent)
except (IOError, ConnectionError) as error:
msg = "Failed to import images from " "%s: %s" % (
source["url"],
error,
)
errors.append(msg)
maaslog.error(msg)
except sutil.SignatureMissingException as error:
# Raise an error to the UI.
proxy = yield deferToDatabase(get_proxy)
if not proxy:
msg = ("Failed to import images from %s (%s). Verify "
"network connectivity and try again." %
(source["url"], error))
else:
msg = ("Failed to import images from %s (%s). Verify "
"network connectivity via your external "
"proxy (%s) and try again." %
(source["url"], error, proxy))
errors.append(msg)
else:
yield deferToDatabase(_update_cache, source, descriptions)
yield deferToDatabase(check_commissioning_series_selected)
component = COMPONENT.REGION_IMAGE_IMPORT
if len(errors) > 0:
maaslog.error("Unable to update boot sources cache.")
yield deferToDatabase(
register_persistent_error,
component,
"<br>".join(map(html.escape, errors)),
)
else:
maaslog.info("Updated boot sources cache.")
yield deferToDatabase(discard_persistent_error, component)
def import_images(sources):
"""Import images. Callable from the command line.
:param config: An iterable of dicts representing the sources from
which boot images will be downloaded.
"""
if len(sources) == 0:
msg = "Can't import: region did not provide a source."
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
return False
msg = "Starting rack boot image import"
maaslog.info(msg)
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
with ClusterConfiguration.open() as config:
storage = FilePath(config.tftp_root).parent().path
with tempdir("keyrings") as keyrings_path:
# XXX: Band-aid to ensure that the keyring_data is bytes. Future task:
# try to figure out why this sometimes happens.
for source in sources:
if "keyring_data" in source and not isinstance(
source["keyring_data"], bytes):
source["keyring_data"] = source["keyring_data"].encode("utf-8")
# We download the keyrings now because we need them for both
# download_all_image_descriptions() and
# download_all_boot_resources() later.
sources = write_all_keyrings(keyrings_path, sources)
# The region produces a SimpleStream which is similar, but not
# identical to the actual SimpleStream. These differences cause
# validation to fail. So grab everything from the region and trust it
# did proper filtering before the rack.
image_descriptions = download_all_image_descriptions(
sources, validate_products=False)
if image_descriptions.is_empty():
msg = ("Finished importing boot images, the region does not have "
"any boot images available.")
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
maaslog.warning(msg)
return False
meta_file_content = image_descriptions.dump_json()
if meta_contains(storage, meta_file_content):
maaslog.info("Finished importing boot images, the region does not "
"have any new images.")
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
maaslog.info(msg)
return False
product_mapping = map_products(image_descriptions)
try:
snapshot_path = download_all_boot_resources(
sources, storage, product_mapping)
except Exception as e:
try_send_rack_event(
EVENT_TYPES.RACK_IMPORT_ERROR,
"Unable to import boot images: %s" % e,
)
maaslog.error(
"Unable to import boot images; cleaning up failed snapshot "
"and cache.")
# Cleanup snapshots and cache since download failed.
cleanup_snapshots_and_cache(storage)
raise
maaslog.info("Writing boot image metadata.")
write_snapshot_metadata(snapshot_path, meta_file_content)
maaslog.info("Linking boot images snapshot %s" % snapshot_path)
link_bootloaders(snapshot_path)
# If we got here, all went well. This is now truly the "current" snapshot.
update_current_symlink(storage, snapshot_path)
# Now cleanup the old snapshots and cache.
maaslog.info("Cleaning up old snapshots and cache.")
cleanup_snapshots_and_cache(storage)
# Import is now finished.
msg = "Finished importing boot images."
maaslog.info(msg)
try_send_rack_event(EVENT_TYPES.RACK_IMPORT_INFO, msg)
return True
def cache_boot_sources():
"""Cache all image information in boot sources.
Called from *outside* of a transaction this will:
1. Retrieve information about all boot sources from the database. The
transaction is committed before proceeding.
2. The boot sources are consulted (i.e. there's network IO now) and image
descriptions downloaded.
3. Update the boot source cache with the fetched information. If the boot
source has been modified or deleted during #2 then the results are
discarded.
This approach does not require an exclusive lock.
"""
# Nomenclature herein: `bootsource` is an ORM record for BootSource;
# `source` is one of those converted to a dict. The former ought not to be
# used outside of a transactional context.
@transactional
def get_sources():
return list(
bootsource.to_dict_without_selections()
for bootsource in BootSource.objects.all()
# TODO: Only where there are no corresponding BootSourceCache
# records or the BootSource's updated timestamp is later than any
# of the BootSourceCache records' timestamps.
)
@transactional
def update_cache(source, descriptions):
try:
bootsource = BootSource.objects.get(url=source["url"])
except BootSource.DoesNotExist:
# The record was deleted while we were fetching the description.
maaslog.debug(
"Image descriptions at %s are no longer needed; discarding.",
source["url"])
else:
if bootsource.compare_dict_without_selections(source):
# Only delete from the cache once we have the descriptions.
BootSourceCache.objects.filter(boot_source=bootsource).delete()
if not descriptions.is_empty():
for spec, item in descriptions.mapping.items():
title = get_product_title(item)
if title is None:
extra = {}
else:
extra = {'title': title}
BootSourceCache.objects.create(
boot_source=bootsource, os=spec.os,
arch=spec.arch, subarch=spec.subarch,
kflavor=spec.kflavor,
release=spec.release, label=spec.label,
release_codename=item.get('release_codename'),
release_title=item.get('release_title'),
support_eol=item.get('support_eol'),
bootloader_type=item.get('bootloader-type'),
extra=extra,
)
maaslog.debug(
"Image descriptions for %s have been updated.",
source["url"])
else:
maaslog.debug(
"Image descriptions for %s are outdated; discarding.",
source["url"])
@transactional
def check_commissioning_series_selected():
commissioning_osystem = Config.objects.get_config(
name='commissioning_osystem')
commissioning_series = Config.objects.get_config(
name='commissioning_distro_series')
qs = BootSourceSelection.objects.filter(
os=commissioning_osystem, release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident='commissioning_series_unselected').exists():
Notification.objects.create_error_for_users(
'%s %s is configured as the commissioning release but it '
'is not selected for download!' % (
commissioning_osystem, commissioning_series),
ident='commissioning_series_unselected')
qs = BootSourceCache.objects.filter(
os=commissioning_osystem, release=commissioning_series)
if not qs.exists():
if not Notification.objects.filter(
ident='commissioning_series_unavailable').exists():
Notification.objects.create_error_for_users(
'%s %s is configured as the commissioning release but it '
'is unavailable in the configured streams!' % (
commissioning_osystem, commissioning_series),
ident='commissioning_series_unavailable')
# FIXME: This modifies the environment of the entire process, which is Not
# Cool. We should integrate with simplestreams in a more Pythonic manner.
yield deferToDatabase(set_simplestreams_env)
errors = []
sources = yield deferToDatabase(get_sources)
for source in sources:
with tempdir("keyrings") as keyrings_path:
[source] = write_all_keyrings(keyrings_path, [source])
try:
descriptions = download_all_image_descriptions(
[source],
user_agent=get_maas_version_user_agent())
except (IOError, ConnectionError) as error:
errors.append(
"Failed to import images from boot source "
"%s: %s" % (source["url"], error))
else:
yield deferToDatabase(update_cache, source, descriptions)
yield deferToDatabase(check_commissioning_series_selected)
maaslog.info("Updated boot sources cache.")
component = COMPONENT.REGION_IMAGE_IMPORT
if len(errors) > 0:
yield deferToDatabase(
register_persistent_error, component,
"<br>".join(map(html.escape, errors)))
else:
yield deferToDatabase(
discard_persistent_error, component)
