def query(self, img_filter):
"""Query streams for latest image given a specific filter.
Args:
img_filter: array of filters as strings format 'key=value'
Returns:
dictionary with latest image information or empty
"""
def policy(content, path): # pylint: disable=W0613
"""TODO."""
return s_util.read_signed(content, keyring=self.keyring_path)
(url, path) = s_util.path_from_mirror_url(self.mirror_url, None)
s_mirror = mirrors.UrlMirrorReader(url, policy=policy)
config = {'filters': filters.get_filters(img_filter)}
self._log.debug('looking for image with the following config:')
self._log.debug(config)
t_mirror = FilterMirror(config)
t_mirror.sync(s_mirror, path)
return t_mirror.json_entries
def mirror(output_d,
source=IMAGE_SRC_URL,
mirror_filters=None,
max_items=1,
keyring=KEYRING,
verbosity=0):
if mirror_filters is None:
mirror_filters = [f for f in ITEM_NAME_FILTERS]
filter_list = filters.get_filters(mirror_filters)
(source_url, initial_path) = sutil.path_from_mirror_url(source, None)
def policy(content, path): # pylint: disable=W0613
if initial_path.endswith('sjson'):
return sutil.read_signed(content, keyring=keyring)
else:
return content
smirror = mirrors.UrlMirrorReader(source_url, policy=policy)
LOG.debug("summary: \n " + '\n '.join([
"source: %s" % source_url,
"path: %s" % initial_path,
"output: %s" % output_d,
"filters: %s" % filter_list,
]) + '\n')
mirror_config = {'max_items': max_items, 'filters': filter_list}
tmirror = CurtinVmTestMirror(config=mirror_config,
out_d=output_d,
verbosity=verbosity)
tmirror.sync(smirror, initial_path)
def images(s):
center(s.__class__.__name__ + '.images')
retval = []
# Get daily streams, change for releases
(mirror_url, path) = util.path_from_mirror_url('https://cloud-images.ubuntu.com/daily/streams/v1/index.sjson', None)
cdebug(' mirror_url: %s' % mirror_url)
cdebug(' path: %s' % path)
smirror = mirrors.UrlMirrorReader(mirror_url)
# Change the content_id to find results for other clouds or for release images
fl = []
fl.append('content_id=com.ubuntu.cloud:daily:%s' % s.cloud)
if s.series is not None:
fl.append('release=' + s.series)
if s.region is not None:
fl.append('region=' + s.region)
filter_list = filters.get_filters(fl)
cdebug(' fl: %s' % fl)
tmirror = FilterMirror(config={'filters': filter_list})
try:
tmirror.sync(smirror, path)
try:
# Find the latest version
for i in tmirror.json_entries:
# cdebug(i)
cdebug(i['version_name'])
versions = [item['version_name'] for item in tmirror.json_entries]
versions = sorted(list(set(versions)))
cdebug(versions)
latest = versions[-1]
items = [i for i in tmirror.json_entries if i['version_name'] == latest]
for item in items:
retval.append(item)
except IndexError:
pass
# # Print a list of the regions represented in the filtered list
# # as an example of extracting a list of unique keys from all items
# regions = set([item['region'] for item in tmirror.json_entries])
# regions = sorted(list(regions))
# print('Regions: %s' % regions)
except IOError:
pass
cleave(s.__class__.__name__ + '.images')
return retval
def latest_cloud_image(release):
"""Download cloud image of specified release using simplestreams.
This expects to find only a single image for the release for the
specific day.
@param release: string of Ubuntu release image to find
@return: path to unique image for specified release
"""
LOG.info('finding pristine image for %s', (release))
mirror_url = 'https://cloud-images.ubuntu.com/daily'
mirror_dir = '/srv/netplan/'
keyring = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'
(url, path) = s_util.path_from_mirror_url(mirror_url, None)
ss_filter = filters.get_filters([
'arch=%s' % system_architecture(),
'release=%s' % release, 'ftype=disk1.img'
])
mirror_config = {
'filters': ss_filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path=None): # pylint: disable=unused-argument
"""Simplestreams policy function.
@param content: signed content
@param path: not used
@return: policy for simplestreams
"""
return s_util.read_signed(content, keyring=keyring)
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(mirror_dir)
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(mirror_dir, '**', release, '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
return images[0]
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_dpkg_architecture(),
'release=%s' % img_conf['release'],
'ftype=disk1.img',
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) < 1:
raise RuntimeError("No images found under '%s'" % search_d)
if len(images) > 1:
raise RuntimeError("Multiple images found in '%s': %s" %
(search_d, ' '.join(images)))
image = NoCloudKVMImage(self, img_conf, images[0])
return image
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'], 'ftype=disk1.img'
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0])
if img_conf.get('override_templates', False):
image.update_templates(self.config.get('template_overrides', {}),
self.config.get('template_files', {}))
return image
def _query_streams(img_conf, img_filter):
"""Query streams for latest image given a specific filter.
@param img_conf: configuration for image
@param filters: array of filters as strings format 'key=value'
@return: dictionary with latest image information or empty
"""
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
smirror = mirrors.UrlMirrorReader(url, policy=policy)
config = {'max_items': 1, 'filters': filters.get_filters(img_filter)}
tmirror = FilterMirror(config)
tmirror.sync(smirror, path)
try:
return tmirror.json_entries[0]
except IndexError:
raise RuntimeError('no images found with filter: %s' % img_filter)
def items_from_url(cls, url):
source = mirrors.UrlMirrorReader(url)
target = cls()
target.sync(source, 'com.ubuntu.cloud:released:azure.sjson')
return target.items