def test_binopen_seek(self): tcs = objectstores.FileStore(self.target) scs = contentsource.UrlContentSource(self.furl_for('foo')) tcs.insert('foo', scs) with open(join(self.target, 'foo'), 'rb') as f: contents = f.read() assert contents == b'hello world\n', contents
def test_url_seek(self): tcs = objectstores.FileStore(self.target) loc = self.url_for('foo') scs = contentsource.UrlContentSource(loc) tcs.insert('foo', scs) with open(join(self.target, 'foo'), 'rb') as f: contents = f.read() # Unfortunately, SimpleHTTPServer doesn't support the Range # header, so we get two 'hello's. assert contents == b'hellohello world\n', contents
def get_mirror_reader(name, docdir=None, signed=False): if docdir is None: docdir = EXAMPLES_DIR src_d = os.path.join(EXAMPLES_DIR, name) sstore = objectstores.FileStore(src_d) def policy(content, path): # pylint: disable=W0613 return content kwargs = {} if signed else {"policy": policy} return mirrors.ObjectStoreMirrorReader(sstore, **kwargs)
def test_percent_callback(self): data = {'dld': 0} def handler(path, downloaded, total): data['dld'] = downloaded tcs = objectstores.FileStore(self.target, complete_callback=handler) loc = self.url_for('foo') scs = contentsource.UrlContentSource(loc) tcs.insert('foo', scs, size=len('hellohello world')) assert data['dld'] > 0 # just make sure it was called
def latest_cloud_image(release): """Download cloud image of specified release using simplestreams. This expects to find only a single image for the release for the specific day. @param release: string of Ubuntu release image to find @return: path to unique image for specified release """ LOG.info('finding pristine image for %s', (release)) mirror_url = 'https://cloud-images.ubuntu.com/daily' mirror_dir = '/srv/netplan/' keyring = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg' (url, path) = s_util.path_from_mirror_url(mirror_url, None) ss_filter = filters.get_filters([ 'arch=%s' % system_architecture(), 'release=%s' % release, 'ftype=disk1.img' ]) mirror_config = { 'filters': ss_filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path=None): # pylint: disable=unused-argument """Simplestreams policy function. @param content: signed content @param path: not used @return: policy for simplestreams """ return s_util.read_signed(content, keyring=keyring) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(mirror_dir) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(mirror_dir, '**', release, '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) != 1: raise Exception('No unique images found') return images[0]
def __init__(self, config, out_d, verbosity=0): self.config = config self.filters = self.config.get('filters', []) self.out_d = os.path.abspath(out_d) self.objectstore = objectstores.FileStore( out_d, complete_callback=self.callback) self.file_info = {} self.data_path = ".vmtest-data" super(CurtinVmTestMirror, self).__init__(config=config, objectstore=self.objectstore) self.verbosity = verbosity self.dlstatus = {'columns': 80, 'total': 0, 'curpath': None}
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters([ 'arch=%s' % c_util.get_dpkg_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img', ]) mirror_config = { 'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) < 1: raise RuntimeError("No images found under '%s'" % search_d) if len(images) > 1: raise RuntimeError("Multiple images found in '%s': %s" % (search_d, ' '.join(images))) image = NoCloudKVMImage(self, img_conf, images[0]) return image
def get_image(self, img_conf): """Get image using specified image configuration. @param img_conf: configuration for image @return_value: cloud_tests.images instance """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) filter = filters.get_filters([ 'arch=%s' % c_util.get_architecture(), 'release=%s' % img_conf['release'], 'ftype=disk1.img' ]) mirror_config = { 'filters': filter, 'keep_items': False, 'max_items': 1, 'checksumming_reader': True, 'item_download': True } def policy(content, path): return s_util.read_signed(content, keyring=img_conf['keyring']) smirror = mirrors.UrlMirrorReader(url, policy=policy) tstore = objectstores.FileStore(img_conf['mirror_dir']) tmirror = mirrors.ObjectFilterMirror(config=mirror_config, objectstore=tstore) tmirror.sync(smirror, path) search_d = os.path.join(img_conf['mirror_dir'], '**', img_conf['release'], '**', '*.img') images = [] for fname in glob.iglob(search_d, recursive=True): images.append(fname) if len(images) != 1: raise Exception('No unique images found') image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) if img_conf.get('override_templates', False): image.update_templates(self.config.get('template_overrides', {}), self.config.get('template_files', {})) return image
def test_corrupted_mirror_resume(self): # test corrupted .part file is caught smirror = get_mirror_reader("foocloud") # create a corrupt .part file tfile = os.path.join(self.target, FOOCLOUD_FILE) os.makedirs(os.path.dirname(tfile)) with open(tfile + ".part", "w") as fw: # just write some invalid data fw.write("--bogus--") target_objstore = objectstores.FileStore(self.target) tmirror = mirrors.ObjectStoreMirrorWriter(config=None, objectstore=target_objstore) self.assertRaisesRegexp(Exception, r".*%s.*" % FOOCLOUD_FILE, tmirror.sync, smirror, "streams/v1/index.json") # now the .part file should be removed, and trying again should succeed self.assertFalse(os.path.exists(tfile + ".part")) tmirror.sync(smirror, "streams/v1/index.json") self.assertFalse(os.path.exists(tfile + ".part"))
def test_mirror_resume(self): # test mirror resuming from filestore smirror = get_mirror_reader("foocloud") # as long as this is less than size of file, its valid part_size = 10 # create a valid .part file tfile = os.path.join(self.target, FOOCLOUD_FILE) os.makedirs(os.path.dirname(tfile)) with open(tfile + ".part", "wb") as fw: with smirror.source(FOOCLOUD_FILE) as fr: fw.write(fr.read(part_size)) target_objstore = objectstores.FileStore(self.target) tmirror = mirrors.ObjectStoreMirrorWriter(config=None, objectstore=target_objstore) tmirror.sync(smirror, "streams/v1/index.json") # the part file should have been cleaned up. If this fails, then # likely the part file wasn't used, and this test is no longer valid self.assertFalse(os.path.exists(tfile + ".part"))
def _tmp_reader(): sstore = objectstores.FileStore(tempfile.gettempdir()) return mirrors.ObjectStoreMirrorReader(sstore)