def query(self, img_filter):
"""Query streams for latest image given a specific filter.
Args:
img_filter: array of filters as strings format 'key=value'
Returns:
dictionary with latest image information or empty
"""
def policy(content, path): # pylint: disable=W0613
"""TODO."""
return s_util.read_signed(content, keyring=self.keyring_path)
(url, path) = s_util.path_from_mirror_url(self.mirror_url, None)
s_mirror = mirrors.UrlMirrorReader(url, policy=policy)
config = {'filters': filters.get_filters(img_filter)}
self._log.debug('looking for image with the following config:')
self._log.debug(config)
t_mirror = FilterMirror(config)
t_mirror.sync(s_mirror, path)
return t_mirror.json_entries
def mirror(output_d,
source=IMAGE_SRC_URL,
mirror_filters=None,
max_items=1,
keyring=KEYRING,
verbosity=0):
if mirror_filters is None:
mirror_filters = [f for f in ITEM_NAME_FILTERS]
filter_list = filters.get_filters(mirror_filters)
(source_url, initial_path) = sutil.path_from_mirror_url(source, None)
def policy(content, path): # pylint: disable=W0613
if initial_path.endswith('sjson'):
return sutil.read_signed(content, keyring=keyring)
else:
return content
smirror = mirrors.UrlMirrorReader(source_url, policy=policy)
LOG.debug("summary: \n " + '\n '.join([
"source: %s" % source_url,
"path: %s" % initial_path,
"output: %s" % output_d,
"filters: %s" % filter_list,
]) + '\n')
mirror_config = {'max_items': max_items, 'filters': filter_list}
tmirror = CurtinVmTestMirror(config=mirror_config,
out_d=output_d,
verbosity=verbosity)
tmirror.sync(smirror, initial_path)
def download_image_descriptions(
path, keyring=None, user_agent=None, validate_products=True):
"""Download image metadata from upstream Simplestreams repo.
:param path: The path to a Simplestreams repo.
:param keyring: Optional keyring for verifying the repo's signatures.
:param user_agent: Optional user agent string for downloading the image
descriptions.
:return: A `BootImageMapping` describing available boot resources.
"""
maaslog.info("Downloading image descriptions from %s", path)
mirror, rpath = path_from_mirror_url(path, None)
policy = get_signing_policy(rpath, keyring)
if user_agent is None:
reader = UrlMirrorReader(mirror, policy=policy)
else:
try:
reader = UrlMirrorReader(
mirror, policy=policy, user_agent=user_agent)
except TypeError:
# UrlMirrorReader doesn't support the user_agent argument.
# simplestream >=bzr429 is required for this feature.
reader = UrlMirrorReader(mirror, policy=policy)
boot_images_dict = BootImageMapping()
dumper = RepoDumper(boot_images_dict, validate_products=validate_products)
dumper.sync(reader, rpath)
return boot_images_dict
def do_sync(charm_conf, status_exchange):
# NOTE(beisner): the user_agent variable was an unused assignment (lint).
# It may be worth re-visiting its usage, intent and benefit with the
# UrlMirrorReader call below at some point. Leaving it disabled for now,
# and not assigning it since it is not currently utilized.
# user_agent = charm_conf.get("user_agent")
for mirror_info in charm_conf['mirror_list']:
mirror_url, initial_path = path_from_mirror_url(
mirror_info['url'], mirror_info['path'])
log.info("configuring sync for url {}".format(mirror_info))
smirror = UrlMirrorReader(mirror_url, policy=policy)
if charm_conf['use_swift']:
store = SwiftObjectStore(SWIFT_DATA_DIR)
else:
# Use the local apache server to serve product streams
store = FileStore(prefix=APACHE_DATA_DIR)
content_id = charm_conf['content_id_template'].format(
region=charm_conf['region'])
config = {
'max_items': mirror_info['max'],
'modify_hook': charm_conf['modify_hook_scripts'],
'keep_items': True,
'content_id': content_id,
'cloud_name': charm_conf['cloud_name'],
'item_filters': mirror_info['item_filters'],
'hypervisor_mapping': charm_conf.get('hypervisor_mapping', False)
}
mirror_args = dict(config=config,
objectstore=store,
name_prefix=charm_conf['name_prefix'])
mirror_args['custom_properties'] = charm_conf.get(
'custom_properties', False)
if SIMPLESTREAMS_HAS_PROGRESS:
log.info("Calling DryRun mirror to get item list")
drmirror = glance.ItemInfoDryRunMirror(config=config,
objectstore=store)
drmirror.sync(smirror, path=initial_path)
p = StatusMessageProgressAggregator(drmirror.items,
status_exchange.send_message)
mirror_args['progress_callback'] = p.progress_callback
else:
log.info("Detected simplestreams version without progress"
" update support. Only limited feedback available.")
tmirror = GlanceMirrorWithCustomProperties(**mirror_args)
log.info("calling GlanceMirror.sync")
tmirror.sync(smirror, path=initial_path)
def images(s):
center(s.__class__.__name__ + '.images')
retval = []
# Get daily streams, change for releases
(mirror_url, path) = util.path_from_mirror_url('https://cloud-images.ubuntu.com/daily/streams/v1/index.sjson', None)
cdebug(' mirror_url: %s' % mirror_url)
cdebug(' path: %s' % path)
smirror = mirrors.UrlMirrorReader(mirror_url)
# Change the content_id to find results for other clouds or for release images
fl = []
fl.append('content_id=com.ubuntu.cloud:daily:%s' % s.cloud)
if s.series is not None:
fl.append('release=' + s.series)
if s.region is not None:
fl.append('region=' + s.region)
filter_list = filters.get_filters(fl)
cdebug(' fl: %s' % fl)
tmirror = FilterMirror(config={'filters': filter_list})
try:
tmirror.sync(smirror, path)
try:
# Find the latest version
for i in tmirror.json_entries:
# cdebug(i)
cdebug(i['version_name'])
versions = [item['version_name'] for item in tmirror.json_entries]
versions = sorted(list(set(versions)))
cdebug(versions)
latest = versions[-1]
items = [i for i in tmirror.json_entries if i['version_name'] == latest]
for item in items:
retval.append(item)
except IndexError:
pass
# # Print a list of the regions represented in the filtered list
# # as an example of extracting a list of unique keys from all items
# regions = set([item['region'] for item in tmirror.json_entries])
# regions = sorted(list(regions))
# print('Regions: %s' % regions)
except IOError:
pass
cleave(s.__class__.__name__ + '.images')
return retval
def latest_cloud_image(release):
"""Download cloud image of specified release using simplestreams.
This expects to find only a single image for the release for the
specific day.
@param release: string of Ubuntu release image to find
@return: path to unique image for specified release
"""
LOG.info('finding pristine image for %s', (release))
mirror_url = 'https://cloud-images.ubuntu.com/daily'
mirror_dir = '/srv/netplan/'
keyring = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'
(url, path) = s_util.path_from_mirror_url(mirror_url, None)
ss_filter = filters.get_filters([
'arch=%s' % system_architecture(),
'release=%s' % release, 'ftype=disk1.img'
])
mirror_config = {
'filters': ss_filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path=None): # pylint: disable=unused-argument
"""Simplestreams policy function.
@param content: signed content
@param path: not used
@return: policy for simplestreams
"""
return s_util.read_signed(content, keyring=keyring)
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(mirror_dir)
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(mirror_dir, '**', release, '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
return images[0]
def do_sync(charm_conf, status_exchange):
for mirror_info in charm_conf['mirror_list']:
mirror_url, initial_path = path_from_mirror_url(
mirror_info['url'], mirror_info['path'])
log.info("configuring sync for url {}".format(mirror_info))
smirror = UrlMirrorReader(mirror_url, policy=policy)
if charm_conf['use_swift']:
store = SwiftObjectStore(SWIFT_DATA_DIR)
else:
store = None
content_id = charm_conf['content_id_template'].format(
region=charm_conf['region'])
config = {
'max_items': mirror_info['max'],
'modify_hook': charm_conf['modify_hook_scripts'],
'keep_items': False,
'content_id': content_id,
'cloud_name': charm_conf['cloud_name'],
'item_filters': mirror_info['item_filters']
}
mirror_args = dict(config=config,
objectstore=store,
name_prefix=charm_conf['name_prefix'])
if SIMPLESTREAMS_HAS_PROGRESS:
log.info("Calling DryRun mirror to get item list")
drmirror = glance.ItemInfoDryRunMirror(config=config,
objectstore=store)
drmirror.sync(smirror, path=initial_path)
p = StatusMessageProgressAggregator(drmirror.items,
status_exchange.send_message)
mirror_args['progress_callback'] = p.progress_callback
else:
log.info("Detected simplestreams version without progress"
" update support. Only limited feedback available.")
tmirror = glance.GlanceMirror(**mirror_args)
log.info("calling GlanceMirror.sync")
tmirror.sync(smirror, path=initial_path)
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_dpkg_architecture(),
'release=%s' % img_conf['release'],
'ftype=disk1.img',
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) < 1:
raise RuntimeError("No images found under '%s'" % search_d)
if len(images) > 1:
raise RuntimeError("Multiple images found in '%s': %s" %
(search_d, ' '.join(images)))
image = NoCloudKVMImage(self, img_conf, images[0])
return image
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'], 'ftype=disk1.img'
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0])
if img_conf.get('override_templates', False):
image.update_templates(self.config.get('template_overrides', {}),
self.config.get('template_files', {}))
return image
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters(['arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'],
'ftype=disk1.img'])
mirror_config = {'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) < 1:
raise RuntimeError("No images found under '%s'" % search_d)
if len(images) > 1:
raise RuntimeError(
"Multiple images found in '%s': %s" % (search_d,
' '.join(images)))
image = NoCloudKVMImage(self, img_conf, images[0])
return image
def download_boot_resources(path, store, snapshot_path, product_mapping,
keyring_file=None):
"""Download boot resources for one simplestreams source.
:param path: The Simplestreams URL for this source.
:param store: A simplestreams `ObjectStore` where downloaded resources
should be stored.
:param snapshot_path: Filesystem path to a snapshot of current upstream
boot resources.
:param product_mapping: A `ProductMapping` describing the resources to be
downloaded.
:param keyring_file: Optional path to a keyring file for verifying
signatures.
"""
maaslog.info("Downloading boot resources from %s", path)
writer = RepoWriter(snapshot_path, store, product_mapping)
(mirror, rpath) = path_from_mirror_url(path, None)
policy = get_signing_policy(rpath, keyring_file)
reader = UrlMirrorReader(mirror, policy=policy)
writer.sync(reader, rpath)
def _query_streams(img_conf, img_filter):
"""Query streams for latest image given a specific filter.
@param img_conf: configuration for image
@param filters: array of filters as strings format 'key=value'
@return: dictionary with latest image information or empty
"""
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
smirror = mirrors.UrlMirrorReader(url, policy=policy)
config = {'max_items': 1, 'filters': filters.get_filters(img_filter)}
tmirror = FilterMirror(config)
tmirror.sync(smirror, path)
try:
return tmirror.json_entries[0]
except IndexError:
raise RuntimeError('no images found with filter: %s' % img_filter)
def _query_streams(img_conf, img_filter):
"""Query streams for latest image given a specific filter.
@param img_conf: configuration for image
@param filters: array of filters as strings format 'key=value'
@return: dictionary with latest image information or empty
"""
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
smirror = mirrors.UrlMirrorReader(url, policy=policy)
config = {'max_items': 1, 'filters': filters.get_filters(img_filter)}
tmirror = FilterMirror(config)
tmirror.sync(smirror, path)
try:
return tmirror.json_entries[0]
except IndexError:
raise RuntimeError('no images found with filter: %s' % img_filter)
