def test_binopen_seek(self):
tcs = objectstores.FileStore(self.target)
scs = contentsource.UrlContentSource(self.furl_for('foo'))
tcs.insert('foo', scs)
with open(join(self.target, 'foo'), 'rb') as f:
contents = f.read()
assert contents == b'hello world\n', contents
def test_url_seek(self):
tcs = objectstores.FileStore(self.target)
loc = self.url_for('foo')
scs = contentsource.UrlContentSource(loc)
tcs.insert('foo', scs)
with open(join(self.target, 'foo'), 'rb') as f:
contents = f.read()
# Unfortunately, SimpleHTTPServer doesn't support the Range
# header, so we get two 'hello's.
assert contents == b'hellohello world\n', contents
def get_mirror_reader(name, docdir=None, signed=False):
if docdir is None:
docdir = EXAMPLES_DIR
src_d = os.path.join(EXAMPLES_DIR, name)
sstore = objectstores.FileStore(src_d)
def policy(content, path): # pylint: disable=W0613
return content
kwargs = {} if signed else {"policy": policy}
return mirrors.ObjectStoreMirrorReader(sstore, **kwargs)
def test_percent_callback(self):
data = {'dld': 0}
def handler(path, downloaded, total):
data['dld'] = downloaded
tcs = objectstores.FileStore(self.target, complete_callback=handler)
loc = self.url_for('foo')
scs = contentsource.UrlContentSource(loc)
tcs.insert('foo', scs, size=len('hellohello world'))
assert data['dld'] > 0 # just make sure it was called
def latest_cloud_image(release):
"""Download cloud image of specified release using simplestreams.
This expects to find only a single image for the release for the
specific day.
@param release: string of Ubuntu release image to find
@return: path to unique image for specified release
"""
LOG.info('finding pristine image for %s', (release))
mirror_url = 'https://cloud-images.ubuntu.com/daily'
mirror_dir = '/srv/netplan/'
keyring = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'
(url, path) = s_util.path_from_mirror_url(mirror_url, None)
ss_filter = filters.get_filters([
'arch=%s' % system_architecture(),
'release=%s' % release, 'ftype=disk1.img'
])
mirror_config = {
'filters': ss_filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path=None): # pylint: disable=unused-argument
"""Simplestreams policy function.
@param content: signed content
@param path: not used
@return: policy for simplestreams
"""
return s_util.read_signed(content, keyring=keyring)
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(mirror_dir)
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(mirror_dir, '**', release, '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
return images[0]
def __init__(self, config, out_d, verbosity=0):
self.config = config
self.filters = self.config.get('filters', [])
self.out_d = os.path.abspath(out_d)
self.objectstore = objectstores.FileStore(
out_d, complete_callback=self.callback)
self.file_info = {}
self.data_path = ".vmtest-data"
super(CurtinVmTestMirror, self).__init__(config=config,
objectstore=self.objectstore)
self.verbosity = verbosity
self.dlstatus = {'columns': 80, 'total': 0, 'curpath': None}
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_dpkg_architecture(),
'release=%s' % img_conf['release'],
'ftype=disk1.img',
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) < 1:
raise RuntimeError("No images found under '%s'" % search_d)
if len(images) > 1:
raise RuntimeError("Multiple images found in '%s': %s" %
(search_d, ' '.join(images)))
image = NoCloudKVMImage(self, img_conf, images[0])
return image
def get_image(self, img_conf):
"""Get image using specified image configuration.
@param img_conf: configuration for image
@return_value: cloud_tests.images instance
"""
(url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None)
filter = filters.get_filters([
'arch=%s' % c_util.get_architecture(),
'release=%s' % img_conf['release'], 'ftype=disk1.img'
])
mirror_config = {
'filters': filter,
'keep_items': False,
'max_items': 1,
'checksumming_reader': True,
'item_download': True
}
def policy(content, path):
return s_util.read_signed(content, keyring=img_conf['keyring'])
smirror = mirrors.UrlMirrorReader(url, policy=policy)
tstore = objectstores.FileStore(img_conf['mirror_dir'])
tmirror = mirrors.ObjectFilterMirror(config=mirror_config,
objectstore=tstore)
tmirror.sync(smirror, path)
search_d = os.path.join(img_conf['mirror_dir'], '**',
img_conf['release'], '**', '*.img')
images = []
for fname in glob.iglob(search_d, recursive=True):
images.append(fname)
if len(images) != 1:
raise Exception('No unique images found')
image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0])
if img_conf.get('override_templates', False):
image.update_templates(self.config.get('template_overrides', {}),
self.config.get('template_files', {}))
return image
def test_corrupted_mirror_resume(self):
# test corrupted .part file is caught
smirror = get_mirror_reader("foocloud")
# create a corrupt .part file
tfile = os.path.join(self.target, FOOCLOUD_FILE)
os.makedirs(os.path.dirname(tfile))
with open(tfile + ".part", "w") as fw:
# just write some invalid data
fw.write("--bogus--")
target_objstore = objectstores.FileStore(self.target)
tmirror = mirrors.ObjectStoreMirrorWriter(config=None,
objectstore=target_objstore)
self.assertRaisesRegexp(Exception, r".*%s.*" % FOOCLOUD_FILE,
tmirror.sync, smirror, "streams/v1/index.json")
# now the .part file should be removed, and trying again should succeed
self.assertFalse(os.path.exists(tfile + ".part"))
tmirror.sync(smirror, "streams/v1/index.json")
self.assertFalse(os.path.exists(tfile + ".part"))
def test_mirror_resume(self):
# test mirror resuming from filestore
smirror = get_mirror_reader("foocloud")
# as long as this is less than size of file, its valid
part_size = 10
# create a valid .part file
tfile = os.path.join(self.target, FOOCLOUD_FILE)
os.makedirs(os.path.dirname(tfile))
with open(tfile + ".part", "wb") as fw:
with smirror.source(FOOCLOUD_FILE) as fr:
fw.write(fr.read(part_size))
target_objstore = objectstores.FileStore(self.target)
tmirror = mirrors.ObjectStoreMirrorWriter(config=None,
objectstore=target_objstore)
tmirror.sync(smirror, "streams/v1/index.json")
# the part file should have been cleaned up. If this fails, then
# likely the part file wasn't used, and this test is no longer valid
self.assertFalse(os.path.exists(tfile + ".part"))
def _tmp_reader():
sstore = objectstores.FileStore(tempfile.gettempdir())
return mirrors.ObjectStoreMirrorReader(sstore)