def test_source_user_agent_no_trailing_slash(self):
"""
When user_agent is set, it is passed to the ContentSource even
if there is no trailing slash.
"""
reader = UrlMirrorReader("/prefix",
mirrors=["a/", "b/"],
user_agent="test agent")
cs = reader.source("some/path")
# A factory function is set instead of the URL_READER, and
# it constructs a URL_READER with user_agent passed in.
url_reader = cs.url_reader
self.assertNotEqual(URL_READER, url_reader)
# Override the default URL_READER to track arguments being passed.
simplestreams.mirrors.cs.URL_READER = fake_url_reader
result = url_reader("a", "b", something="c")
# It passes all the same arguments, with "user_agent" added in.
self.assertEqual(
{
"user_agent": "test agent",
"something": "c",
"ARGS": ("a", "b")
}, result)
# Restore default UrlReader.
simplestreams.mirrors.cs.URL_READER = URL_READER
def download_image_descriptions(
path, keyring=None, user_agent=None, validate_products=True):
"""Download image metadata from upstream Simplestreams repo.
:param path: The path to a Simplestreams repo.
:param keyring: Optional keyring for verifying the repo's signatures.
:param user_agent: Optional user agent string for downloading the image
descriptions.
:return: A `BootImageMapping` describing available boot resources.
"""
maaslog.info("Downloading image descriptions from %s", path)
mirror, rpath = path_from_mirror_url(path, None)
policy = get_signing_policy(rpath, keyring)
if user_agent is None:
reader = UrlMirrorReader(mirror, policy=policy)
else:
try:
reader = UrlMirrorReader(
mirror, policy=policy, user_agent=user_agent)
except TypeError:
# UrlMirrorReader doesn't support the user_agent argument.
# simplestream >=bzr429 is required for this feature.
reader = UrlMirrorReader(mirror, policy=policy)
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict, validate_products=validate_products)
dumper.sync(reader, rpath)
return boot_images_dict
def test_source_no_trailing_slash(self):
"""Even if prefix lacks a trailing slash, it behaves the same."""
reader = UrlMirrorReader("/prefix/",
mirrors=["a/", "b/"],
user_agent=None)
cs = reader.source("some/path")
self.assertEqual("/prefix/some/path", cs.url)
self.assertEqual(["a/some/path", "b/some/path"], cs.mirrors)
self.assertEqual(URL_READER, cs.url_reader)
def test_source(self):
"""source() method returns a ContentSource."""
# Verify source() returns a content source constructed using the
# appropriate path and mirrors.
reader = UrlMirrorReader("/prefix/",
mirrors=["a/", "b/"],
user_agent=None)
cs = reader.source("some/path")
# Resulting ContentSource is passed an URL as a concatenation of
# the prefix and the path.
self.assertEqual("/prefix/some/path", cs.url)
# Mirror URLs have path appended.
self.assertEqual(["a/some/path", "b/some/path"], cs.mirrors)
# Default URL_READER is returned.
self.assertEqual(URL_READER, cs.url_reader)
def do_sync(charm_conf, status_exchange):
# NOTE(beisner): the user_agent variable was an unused assignment (lint).
# It may be worth re-visiting its usage, intent and benefit with the
# UrlMirrorReader call below at some point. Leaving it disabled for now,
# and not assigning it since it is not currently utilized.
# user_agent = charm_conf.get("user_agent")
for mirror_info in charm_conf['mirror_list']:
mirror_url, initial_path = path_from_mirror_url(
mirror_info['url'], mirror_info['path'])
log.info("configuring sync for url {}".format(mirror_info))
smirror = UrlMirrorReader(mirror_url, policy=policy)
if charm_conf['use_swift']:
store = SwiftObjectStore(SWIFT_DATA_DIR)
else:
# Use the local apache server to serve product streams
store = FileStore(prefix=APACHE_DATA_DIR)
content_id = charm_conf['content_id_template'].format(
region=charm_conf['region'])
config = {
'max_items': mirror_info['max'],
'modify_hook': charm_conf['modify_hook_scripts'],
'keep_items': True,
'content_id': content_id,
'cloud_name': charm_conf['cloud_name'],
'item_filters': mirror_info['item_filters'],
'hypervisor_mapping': charm_conf.get('hypervisor_mapping', False)
}
mirror_args = dict(config=config,
objectstore=store,
name_prefix=charm_conf['name_prefix'])
mirror_args['custom_properties'] = charm_conf.get(
'custom_properties', False)
if SIMPLESTREAMS_HAS_PROGRESS:
log.info("Calling DryRun mirror to get item list")
drmirror = glance.ItemInfoDryRunMirror(config=config,
objectstore=store)
drmirror.sync(smirror, path=initial_path)
p = StatusMessageProgressAggregator(drmirror.items,
status_exchange.send_message)
mirror_args['progress_callback'] = p.progress_callback
else:
log.info("Detected simplestreams version without progress"
" update support. Only limited feedback available.")
tmirror = GlanceMirrorWithCustomProperties(**mirror_args)
log.info("calling GlanceMirror.sync")
tmirror.sync(smirror, path=initial_path)
def test_source_user_agent(self):
"""Default user_agent is set and passed to the ContentSource."""
reader = UrlMirrorReader("/prefix/", mirrors=["a/", "b/"])
cs = reader.source("some/path")
# A factory function is set instead of the URL_READER, and
# it constructs a URL_READER with user_agent passed in.
url_reader = cs.url_reader
self.assertNotEqual(URL_READER, url_reader)
# Override the default URL_READER to track arguments being passed.
simplestreams.mirrors.cs.URL_READER = fake_url_reader
result = url_reader("a", "b", something="c")
# It passes all the same arguments, with "user_agent" added in.
self.assertEqual(
{
"user_agent": "python-simplestreams/0.1",
"something": "c",
"ARGS": ("a", "b")
}, result)
# Restore default UrlReader.
simplestreams.mirrors.cs.URL_READER = URL_READER
def do_sync(charm_conf, status_exchange):
for mirror_info in charm_conf['mirror_list']:
mirror_url, initial_path = path_from_mirror_url(
mirror_info['url'], mirror_info['path'])
log.info("configuring sync for url {}".format(mirror_info))
smirror = UrlMirrorReader(mirror_url, policy=policy)
if charm_conf['use_swift']:
store = SwiftObjectStore(SWIFT_DATA_DIR)
else:
store = None
content_id = charm_conf['content_id_template'].format(
region=charm_conf['region'])
config = {
'max_items': mirror_info['max'],
'modify_hook': charm_conf['modify_hook_scripts'],
'keep_items': False,
'content_id': content_id,
'cloud_name': charm_conf['cloud_name'],
'item_filters': mirror_info['item_filters']
}
mirror_args = dict(config=config,
objectstore=store,
name_prefix=charm_conf['name_prefix'])
if SIMPLESTREAMS_HAS_PROGRESS:
log.info("Calling DryRun mirror to get item list")
drmirror = glance.ItemInfoDryRunMirror(config=config,
objectstore=store)
drmirror.sync(smirror, path=initial_path)
p = StatusMessageProgressAggregator(drmirror.items,
status_exchange.send_message)
mirror_args['progress_callback'] = p.progress_callback
else:
log.info("Detected simplestreams version without progress"
" update support. Only limited feedback available.")
tmirror = glance.GlanceMirror(**mirror_args)
log.info("calling GlanceMirror.sync")
tmirror.sync(smirror, path=initial_path)
def download_boot_resources(path, store, snapshot_path, product_mapping,
keyring_file=None):
"""Download boot resources for one simplestreams source.
:param path: The Simplestreams URL for this source.
:param store: A simplestreams `ObjectStore` where downloaded resources
should be stored.
:param snapshot_path: Filesystem path to a snapshot of current upstream
boot resources.
:param product_mapping: A `ProductMapping` describing the resources to be
downloaded.
:param keyring_file: Optional path to a keyring file for verifying
signatures.
"""
maaslog.info("Downloading boot resources from %s", path)
writer = RepoWriter(snapshot_path, store, product_mapping)
(mirror, rpath) = path_from_mirror_url(path, None)
policy = get_signing_policy(rpath, keyring_file)
reader = UrlMirrorReader(mirror, policy=policy)
writer.sync(reader, rpath)