def __init__(self, config, objectstore=None, region=None, name_prefix=None, progress_callback=None, client=None): super(GlanceMirror, self).__init__(config=config) self.item_filters = self.config.get('item_filters', []) if len(self.item_filters) == 0: self.item_filters = [ 'ftype~(disk1.img|disk.img)', 'arch~(x86_64|amd64|i386)' ] self.item_filters = filters.get_filters(self.item_filters) self.index_filters = self.config.get('index_filters', []) if len(self.index_filters) == 0: self.index_filters = ['datatype=image-downloads'] self.index_filters = filters.get_filters(self.index_filters) self.loaded_content = {} self.store = objectstore if client is None: client = openstack self.keystone_creds = client.load_keystone_creds() self.name_prefix = name_prefix or "" if region is not None: self.keystone_creds['region_name'] = region self.progress_callback = progress_callback conn_info = client.get_service_conn_info('image', **self.keystone_creds) self.glance_api_version = conn_info['glance_version'] self.gclient = get_glanceclient(version=self.glance_api_version, **conn_info) self.tenant_id = conn_info['tenant_id'] self.region = self.keystone_creds.get('region_name', 'nullregion') self.cloudname = config.get("cloud_name", 'nullcloud') self.crsn = '-'.join(( self.cloudname, self.region, )) self.auth_url = self.keystone_creds['auth_url'] self.content_id = config.get("content_id") self.modify_hook = config.get("modify_hook") self.inserts = {} if not self.content_id: raise TypeError("content_id is required")
def query(self, img_filter): """Query streams for latest image given a specific filter. Args: img_filter: array of filters as strings format 'key=value' Returns: dictionary with latest image information or empty """ def policy(content, path): # pylint: disable=W0613 """TODO.""" return s_util.read_signed(content, keyring=self.keyring_path) (url, path) = s_util.path_from_mirror_url(self.mirror_url, None) s_mirror = mirrors.UrlMirrorReader(url, policy=policy) config = {'filters': filters.get_filters(img_filter)} self._log.debug('looking for image with the following config:') self._log.debug(config) t_mirror = FilterMirror(config) t_mirror.sync(s_mirror, path) return t_mirror.json_entries
def mirror(output_d, source=IMAGE_SRC_URL, mirror_filters=None, max_items=1, keyring=KEYRING, verbosity=0): if mirror_filters is None: mirror_filters = [f for f in ITEM_NAME_FILTERS] filter_list = filters.get_filters(mirror_filters) (source_url, initial_path) = sutil.path_from_mirror_url(source, None) def policy(content, path): # pylint: disable=W0613 if initial_path.endswith('sjson'): return sutil.read_signed(content, keyring=keyring) else: return content smirror = mirrors.UrlMirrorReader(source_url, policy=policy) LOG.debug("summary: \n " + '\n '.join([ "source: %s" % source_url, "path: %s" % initial_path, "output: %s" % output_d, "filters: %s" % filter_list, ]) + '\n') mirror_config = {'max_items': max_items, 'filters': filter_list} tmirror = CurtinVmTestMirror(config=mirror_config, out_d=output_d, verbosity=verbosity) tmirror.sync(smirror, initial_path)
def test_ObjectFilterMirror_does_item_filter(self): src = get_mirror_reader("foocloud") filter_list = get_filters(['ftype!=disk1.img']) config = {'filters': filter_list} objectstore = MemoryObjectStore(None) target = ObjectFilterMirror(config, objectstore) target.sync(src, "streams/v1/index.json") unexpected = [f for f in objectstore.data if 'disk' in f] assert len(unexpected) == 0 assert len(objectstore.data) != 0
def images(s): center(s.__class__.__name__ + '.images') retval = [] # Get daily streams, change for releases (mirror_url, path) = util.path_from_mirror_url('https://cloud-images.ubuntu.com/daily/streams/v1/index.sjson', None) cdebug(' mirror_url: %s' % mirror_url) cdebug(' path: %s' % path) smirror = mirrors.UrlMirrorReader(mirror_url) # Change the content_id to find results for other clouds or for release images fl = [] fl.append('content_id=com.ubuntu.cloud:daily:%s' % s.cloud) if s.series is not None: fl.append('release=' + s.series) if s.region is not None: fl.append('region=' + s.region) filter_list = filters.get_filters(fl) cdebug(' fl: %s' % fl) tmirror = FilterMirror(config={'filters': filter_list}) try: tmirror.sync(smirror, path) try: # Find the latest version for i in tmirror.json_entries: # cdebug(i) cdebug(i['version_name']) versions = [item['version_name'] for item in tmirror.json_entries] versions = sorted(list(set(versions))) cdebug(versions) latest = versions[-1] items = [i for i in tmirror.json_entries if i['version_name'] == latest] for item in items: retval.append(item) except IndexError: pass # # Print a list of the regions represented in the filtered list # # as an example of extracting a list of unique keys from all items # regions = set([item['region'] for item in tmirror.json_entries]) # regions = sorted(list(regions)) # print('Regions: %s' % regions) except IOError: pass cleave(s.__class__.__name__ + '.images') return retval
def latest_cloud_image(release): """Download cloud image of specified release using simplestreams. This expects to find only a single image for the release for the specific day. @param release: string of Ubuntu release image to find @return: path to unique image for specified release """ LOG.info('finding pristine image for %s', (release)) mirror_url = 'https://cloud-images.ubuntu.com/daily' mirror_dir = '/srv/netplan/' keyring = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg' (url, path) = s_util.path_from_mirror_url(mirror_url, None) ss_filter = filters.get_filters([ 'arch=%s' % system_architecture(), 'release=%s' % release, 'ftype=disk1.img' ]) mirror_config = { 'filters': ss_filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path=None): # pylint: disable=unused-argument """Simplestreams policy function. @param content: signed content @param path: not used @return: policy for simplestreams """ return s_util.read_signed(content, keyring=keyring) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(mirror_dir) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(mirror_dir, '**', release, '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) != 1: raise Exception('No unique images found') return images[0]
def query(mirror, max_items=1, filter_list=None, verbosity=0): if filter_list is None: filter_list = [] ifilters = filters.get_filters(filter_list) def fpath(path): return os.path.join(mirror, path) return next( (q for q in (query_ptree(sutil.load_content(util.load_file(fpath(path))), max_num=max_items, ifilters=ifilters, path2url=fpath) for path in VMTEST_CONTENT_ID_PATH_MAP.values() if os.path.exists(fpath(path))) if q), [])
def __init__(self, local_path, config=None, delete=False, item_filters=None, product_regex=PRODUCTS_REGEX): self.local_path = os.path.abspath(local_path) self.delete = delete # Any user specified filters such as arch~(amd64|i386) are in # addition to our selecting only tar.gz files. That's the only type # of file we know how to unpack. self.item_filters = item_filters or [] self.item_filters.append('ftype=tar.gz') self.item_filters = filters.get_filters(self.item_filters) self.product_filters = [ filters.ItemFilter('product_name~' + product_regex)] objectstore = objectstores.FileStore(self._simplestreams_path()) super(MAASMirrorWriter, self).__init__(config, objectstore)
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters([ 'arch=%s' % c_util.get_dpkg_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img', ]) mirror_config = { 'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) < 1: raise RuntimeError("No images found under '%s'" % search_d) if len(images) > 1: raise RuntimeError("Multiple images found in '%s': %s" % (search_d, ' '.join(images))) image = NoCloudKVMImage(self, img_conf, images[0]) return image
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters([ 'arch=%s' % c_util.get_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img' ]) mirror_config = { 'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) != 1: raise Exception('No unique images found') image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) if img_conf.get('override_templates', False): image.update_templates(self.config.get('template_overrides', {}), self.config.get('template_files', {})) return image
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img']) mirror_config = {'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) < 1: raise RuntimeError("No images found under '%s'" % search_d) if len(images) > 1: raise RuntimeError( "Multiple images found in '%s': %s" % (search_d, ' '.join(images))) image = NoCloudKVMImage(self, img_conf, images[0]) return image
def _query_streams(img_conf, img_filter): """Query streams for latest image given a specific filter. @param img_conf: configuration for image @param filters: array of filters as strings format 'key=value' @return: dictionary with latest image information or empty """ def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) smirror = mirrors.UrlMirrorReader(url, policy=policy) config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} tmirror = FilterMirror(config) tmirror.sync(smirror, path) try: return tmirror.json_entries[0] except IndexError: raise RuntimeError('no images found with filter: %s' % img_filter)
def _query_streams(img_conf, img_filter): """Query streams for latest image given a specific filter. @param img_conf: configuration for image @param filters: array of filters as strings format 'key=value' @return: dictionary with latest image information or empty """ def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) smirror = mirrors.UrlMirrorReader(url, policy=policy) config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} tmirror = FilterMirror(config) tmirror.sync(smirror, path) try: return tmirror.json_entries[0] except IndexError: raise RuntimeError('no images found with filter: %s' % img_filter)