def query(self, img_filter): """Query streams for latest image given a specific filter. Args: img_filter: array of filters as strings format 'key=value' Returns: dictionary with latest image information or empty """ def policy(content, path): # pylint: disable=W0613 """TODO.""" return s_util.read_signed(content, keyring=self.keyring_path) (url, path) = s_util.path_from_mirror_url(self.mirror_url, None) s_mirror = mirrors.UrlMirrorReader(url, policy=policy) config = {'filters': filters.get_filters(img_filter)} self._log.debug('looking for image with the following config:') self._log.debug(config) t_mirror = FilterMirror(config) t_mirror.sync(s_mirror, path) return t_mirror.json_entries
def mirror(output_d, source=IMAGE_SRC_URL, mirror_filters=None, max_items=1, keyring=KEYRING, verbosity=0): if mirror_filters is None: mirror_filters = [f for f in ITEM_NAME_FILTERS] filter_list = filters.get_filters(mirror_filters) (source_url, initial_path) = sutil.path_from_mirror_url(source, None) def policy(content, path): # pylint: disable=W0613 if initial_path.endswith('sjson'): return sutil.read_signed(content, keyring=keyring) else: return content smirror = mirrors.UrlMirrorReader(source_url, policy=policy) LOG.debug("summary: \n " + '\n '.join([ "source: %s" % source_url, "path: %s" % initial_path, "output: %s" % output_d, "filters: %s" % filter_list, ]) + '\n') mirror_config = {'max_items': max_items, 'filters': filter_list} tmirror = CurtinVmTestMirror(config=mirror_config, out_d=output_d, verbosity=verbosity) tmirror.sync(smirror, initial_path)
def download_image_descriptions( path, keyring=None, user_agent=None, validate_products=True): """Download image metadata from upstream Simplestreams repo. :param path: The path to a Simplestreams repo. :param keyring: Optional keyring for verifying the repo's signatures. :param user_agent: Optional user agent string for downloading the image descriptions. :return: A `BootImageMapping` describing available boot resources. """ maaslog.info("Downloading image descriptions from %s", path) mirror, rpath = path_from_mirror_url(path, None) policy = get_signing_policy(rpath, keyring) if user_agent is None: reader = UrlMirrorReader(mirror, policy=policy) else: try: reader = UrlMirrorReader( mirror, policy=policy, user_agent=user_agent) except TypeError: # UrlMirrorReader doesn't support the user_agent argument. # simplestream >=bzr429 is required for this feature. reader = UrlMirrorReader(mirror, policy=policy) boot_images_dict = BootImageMapping() dumper = RepoDumper(boot_images_dict, validate_products=validate_products) dumper.sync(reader, rpath) return boot_images_dict
def do_sync(charm_conf, status_exchange): # NOTE(beisner): the user_agent variable was an unused assignment (lint). # It may be worth re-visiting its usage, intent and benefit with the # UrlMirrorReader call below at some point. Leaving it disabled for now, # and not assigning it since it is not currently utilized. # user_agent = charm_conf.get("user_agent") for mirror_info in charm_conf['mirror_list']: mirror_url, initial_path = path_from_mirror_url( mirror_info['url'], mirror_info['path']) log.info("configuring sync for url {}".format(mirror_info)) smirror = UrlMirrorReader(mirror_url, policy=policy) if charm_conf['use_swift']: store = SwiftObjectStore(SWIFT_DATA_DIR) else: # Use the local apache server to serve product streams store = FileStore(prefix=APACHE_DATA_DIR) content_id = charm_conf['content_id_template'].format( region=charm_conf['region']) config = { 'max_items': mirror_info['max'], 'modify_hook': charm_conf['modify_hook_scripts'], 'keep_items': True, 'content_id': content_id, 'cloud_name': charm_conf['cloud_name'], 'item_filters': mirror_info['item_filters'], 'hypervisor_mapping': charm_conf.get('hypervisor_mapping', False) } mirror_args = dict(config=config, objectstore=store, name_prefix=charm_conf['name_prefix']) mirror_args['custom_properties'] = charm_conf.get( 'custom_properties', False) if SIMPLESTREAMS_HAS_PROGRESS: log.info("Calling DryRun mirror to get item list") drmirror = glance.ItemInfoDryRunMirror(config=config, objectstore=store) drmirror.sync(smirror, path=initial_path) p = StatusMessageProgressAggregator(drmirror.items, status_exchange.send_message) mirror_args['progress_callback'] = p.progress_callback else: log.info("Detected simplestreams version without progress" " update support. Only limited feedback available.") tmirror = GlanceMirrorWithCustomProperties(**mirror_args) log.info("calling GlanceMirror.sync") tmirror.sync(smirror, path=initial_path)
def images(s): center(s.__class__.__name__ + '.images') retval = [] # Get daily streams, change for releases (mirror_url, path) = util.path_from_mirror_url('https://cloud-images.ubuntu.com/daily/streams/v1/index.sjson', None) cdebug(' mirror_url: %s' % mirror_url) cdebug(' path: %s' % path) smirror = mirrors.UrlMirrorReader(mirror_url) # Change the content_id to find results for other clouds or for release images fl = [] fl.append('content_id=com.ubuntu.cloud:daily:%s' % s.cloud) if s.series is not None: fl.append('release=' + s.series) if s.region is not None: fl.append('region=' + s.region) filter_list = filters.get_filters(fl) cdebug(' fl: %s' % fl) tmirror = FilterMirror(config={'filters': filter_list}) try: tmirror.sync(smirror, path) try: # Find the latest version for i in tmirror.json_entries: # cdebug(i) cdebug(i['version_name']) versions = [item['version_name'] for item in tmirror.json_entries] versions = sorted(list(set(versions))) cdebug(versions) latest = versions[-1] items = [i for i in tmirror.json_entries if i['version_name'] == latest] for item in items: retval.append(item) except IndexError: pass # # Print a list of the regions represented in the filtered list # # as an example of extracting a list of unique keys from all items # regions = set([item['region'] for item in tmirror.json_entries]) # regions = sorted(list(regions)) # print('Regions: %s' % regions) except IOError: pass cleave(s.__class__.__name__ + '.images') return retval
def latest_cloud_image(release): """Download cloud image of specified release using simplestreams. This expects to find only a single image for the release for the specific day. @param release: string of Ubuntu release image to find @return: path to unique image for specified release """ LOG.info('finding pristine image for %s', (release)) mirror_url = 'https://cloud-images.ubuntu.com/daily' mirror_dir = '/srv/netplan/' keyring = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg' (url, path) = s_util.path_from_mirror_url(mirror_url, None) ss_filter = filters.get_filters([ 'arch=%s' % system_architecture(), 'release=%s' % release, 'ftype=disk1.img' ]) mirror_config = { 'filters': ss_filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path=None): # pylint: disable=unused-argument """Simplestreams policy function. @param content: signed content @param path: not used @return: policy for simplestreams """ return s_util.read_signed(content, keyring=keyring) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(mirror_dir) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(mirror_dir, '**', release, '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) != 1: raise Exception('No unique images found') return images[0]
def do_sync(charm_conf, status_exchange): for mirror_info in charm_conf['mirror_list']: mirror_url, initial_path = path_from_mirror_url( mirror_info['url'], mirror_info['path']) log.info("configuring sync for url {}".format(mirror_info)) smirror = UrlMirrorReader(mirror_url, policy=policy) if charm_conf['use_swift']: store = SwiftObjectStore(SWIFT_DATA_DIR) else: store = None content_id = charm_conf['content_id_template'].format( region=charm_conf['region']) config = { 'max_items': mirror_info['max'], 'modify_hook': charm_conf['modify_hook_scripts'], 'keep_items': False, 'content_id': content_id, 'cloud_name': charm_conf['cloud_name'], 'item_filters': mirror_info['item_filters'] } mirror_args = dict(config=config, objectstore=store, name_prefix=charm_conf['name_prefix']) if SIMPLESTREAMS_HAS_PROGRESS: log.info("Calling DryRun mirror to get item list") drmirror = glance.ItemInfoDryRunMirror(config=config, objectstore=store) drmirror.sync(smirror, path=initial_path) p = StatusMessageProgressAggregator(drmirror.items, status_exchange.send_message) mirror_args['progress_callback'] = p.progress_callback else: log.info("Detected simplestreams version without progress" " update support. Only limited feedback available.") tmirror = glance.GlanceMirror(**mirror_args) log.info("calling GlanceMirror.sync") tmirror.sync(smirror, path=initial_path)
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters([ 'arch=%s' % c_util.get_dpkg_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img', ]) mirror_config = { 'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) < 1: raise RuntimeError("No images found under '%s'" % search_d) if len(images) > 1: raise RuntimeError("Multiple images found in '%s': %s" % (search_d, ' '.join(images))) image = NoCloudKVMImage(self, img_conf, images[0]) return image
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters([ 'arch=%s' % c_util.get_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img' ]) mirror_config = { 'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) != 1: raise Exception('No unique images found') image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) if img_conf.get('override_templates', False): image.update_templates(self.config.get('template_overrides', {}), self.config.get('template_files', {})) return image
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img']) mirror_config = {'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) < 1: raise RuntimeError("No images found under '%s'" % search_d) if len(images) > 1: raise RuntimeError( "Multiple images found in '%s': %s" % (search_d, ' '.join(images))) image = NoCloudKVMImage(self, img_conf, images[0]) return image
def download_boot_resources(path, store, snapshot_path, product_mapping, keyring_file=None): """Download boot resources for one simplestreams source. :param path: The Simplestreams URL for this source. :param store: A simplestreams `ObjectStore` where downloaded resources should be stored. :param snapshot_path: Filesystem path to a snapshot of current upstream boot resources. :param product_mapping: A `ProductMapping` describing the resources to be downloaded. :param keyring_file: Optional path to a keyring file for verifying signatures. """ maaslog.info("Downloading boot resources from %s", path) writer = RepoWriter(snapshot_path, store, product_mapping) (mirror, rpath) = path_from_mirror_url(path, None) policy = get_signing_policy(rpath, keyring_file) reader = UrlMirrorReader(mirror, policy=policy) writer.sync(reader, rpath)
def _query_streams(img_conf, img_filter): """Query streams for latest image given a specific filter. @param img_conf: configuration for image @param filters: array of filters as strings format 'key=value' @return: dictionary with latest image information or empty """ def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) smirror = mirrors.UrlMirrorReader(url, policy=policy) config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} tmirror = FilterMirror(config) tmirror.sync(smirror, path) try: return tmirror.json_entries[0] except IndexError: raise RuntimeError('no images found with filter: %s' % img_filter)
def _query_streams(img_conf, img_filter): """Query streams for latest image given a specific filter. @param img_conf: configuration for image @param filters: array of filters as strings format 'key=value' @return: dictionary with latest image information or empty """ def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) smirror = mirrors.UrlMirrorReader(url, policy=policy) config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} tmirror = FilterMirror(config) tmirror.sync(smirror, path) try: return tmirror.json_entries[0] except IndexError: raise RuntimeError('no images found with filter: %s' % img_filter)