def make_env(env_type, base, top): img_id = make_uuid() base_id = make_uuid() top_id = make_uuid() if env_type == "block" and base.format == "raw": prealloc = sc.PREALLOCATED_VOL else: prealloc = sc.SPARSE_VOL with fake_env(env_type) as env: env.make_volume(base.virtual * GB, img_id, base_id, vol_format=sc.name2type(base.format), prealloc=prealloc) env.make_volume(top.virtual * GB, img_id, top_id, parent_vol_id=base_id, vol_format=sc.COW_FORMAT) env.subchain = merge.SubchainInfo( dict(sd_id=env.sd_manifest.sdUUID, img_id=img_id, base_id=base_id, top_id=top_id), 0 ) if env_type == "block": # Simulate allocation by adjusting the LV sizes env.lvm.extendLV(env.sd_manifest.sdUUID, base_id, base.physical * GB / MB) env.lvm.extendLV(env.sd_manifest.sdUUID, top_id, top.physical * GB / MB) rm = FakeResourceManager() with MonkeyPatchScope( [ (guarded, "context", fake_guarded_context()), (merge, "sdCache", env.sdcache), (blockVolume, "rm", rm), (blockVolume, "sdCache", env.sdcache), (image.Image, "getChain", lambda self, sdUUID, imgUUID: [env.subchain.base_vol, env.subchain.top_vol]), (blockVolume.BlockVolume, "extendSize", partial(fake_blockVolume_extendSize, env)), (fileVolume.FileVolume, "extendSize", partial(fake_fileVolume_extendSize, env)), ] ): yield env
def make_volume(env, size, md_fmt, real_fmt): img_id = make_uuid() vol_id = make_uuid() env.make_volume(size, img_id, vol_id, vol_format=md_formats[md_fmt]) vol = env.sd_manifest.produceVolume(img_id, vol_id) qemuimg.create(vol.getVolumePath(), size, qemu_formats[real_fmt]) return vol
def test_volume_accessibility(self): with fake_block_env() as env: sd_id = env.sd_manifest.sdUUID img_id = make_uuid() vol_id = make_uuid() make_block_volume(env.lvm, env.sd_manifest, 1 * MB, img_id, vol_id) self.assertTrue(os.path.isfile(env.lvm.lvPath(sd_id, vol_id))) domain_path = os.path.join(env.sd_manifest.domaindir, sd.DOMAIN_IMAGES, img_id, vol_id) repo_path = os.path.join(env.sd_manifest.getRepoPath(), sd_id, sd.DOMAIN_IMAGES, img_id, vol_id) self.assertNotEqual(repo_path, domain_path) # The links to the dev are created only when producing the volume self.assertFalse(os.path.isfile(domain_path)) self.assertFalse(os.path.isfile(repo_path)) env.sd_manifest.produceVolume(img_id, vol_id) self.assertTrue(os.path.samefile(repo_path, domain_path))
def volume(self): img_id = make_uuid() vol_id = make_uuid() with fake_env('file') as env: env.make_volume(MB, img_id, vol_id) vol = env.sd_manifest.produceVolume(img_id, vol_id) yield vol
def fake_volume(self, vol_fmt, sd_version=3): with fake_file_env(sd_version=sd_version) as env: img_id = make_uuid() vol_id = make_uuid() make_file_volume(env.sd_manifest, self.SIZE, img_id, vol_id, vol_format=vol_fmt) yield env.sd_manifest.produceVolume(img_id, vol_id)
def fake_volume(storage_type='file', size=MB, format=sc.RAW_FORMAT): img_id = make_uuid() vol_id = make_uuid() with fake_env(storage_type) as env: env.make_volume(size, img_id, vol_id, vol_format=format) vol = env.sd_manifest.produceVolume(img_id, vol_id) yield vol
def make_blocksd_manifest(tmpdir, fake_lvm, sduuid=None, devices=None, sd_version=3): if sduuid is None: sduuid = make_uuid() if devices is None: devices = get_random_devices() spuuid = make_uuid() fake_lvm.createVG(sduuid, devices, blockSD.STORAGE_DOMAIN_TAG, blockSD.VG_METADATASIZE) fake_lvm.createLV(sduuid, sd.METADATA, blockSD.SD_METADATA_SIZE) # Create the rest of the special LVs special = blockSD.BlockStorageDomainManifest.special_volumes(sd_version) for name, size_mb in sd.SPECIAL_VOLUME_SIZES_MIB.iteritems(): if name in special: fake_lvm.createLV(sduuid, name, size_mb) fake_lvm.createLV(sduuid, blockSD.MASTERLV, blockSD.MASTER_LV_SIZE_MB) # We'll store the domain metadata in the VG's tags metadata = make_sd_metadata(sduuid, version=sd_version, pools=[spuuid]) assert(metadata[sd.DMDK_VERSION] >= 3) # Tag based MD is V3 and above tag_md = blockSD.TagBasedSDMetadata(sduuid) tag_md.update(metadata) manifest = blockSD.BlockStorageDomainManifest(sduuid, tag_md) os.makedirs(os.path.join(manifest.domaindir, sd.DOMAIN_IMAGES)) # Make the repo directory structure repo_pool_dir = os.path.join(tmpdir, spuuid) os.mkdir(repo_pool_dir) os.symlink(manifest.domaindir, os.path.join(repo_pool_dir, sduuid)) return manifest
def _get_args(self): job_id = make_uuid() host_id = 1 sd_manifest = FakeDomainManifest(make_uuid()) vol_info = _get_vol_info() vol_info_obj = storage.sdm.api.create_volume.CreateVolumeInfo(vol_info) return dict(job_id=job_id, host_id=host_id, sd_manifest=sd_manifest, vol_info=vol_info_obj)
def test_volume_type(self, vol_type): with fake_block_env() as env: img_id = make_uuid() vol_id = make_uuid() make_block_volume(env.lvm, env.sd_manifest, 0, img_id, vol_id, vol_type=vol_type) vol = env.sd_manifest.produceVolume(img_id, vol_id) self.assertEqual(vol.getVolType(), sc.type2name(vol_type))
def test_volume_structure(self): with fake_file_env() as env: img_id = make_uuid() vol_id = make_uuid() make_file_volume(env.sd_manifest, 0, img_id, vol_id) image_dir = env.sd_manifest.getImagePath(img_id) files = (vol_id, vol_id + sc.LEASE_FILEEXT, vol_id + fileVolume.META_FILEEXT) for f in files: path = os.path.join(image_dir, f) self.assertTrue(os.path.exists(path))
def make_volume(self, size, storage_type='block', format=sc.RAW_FORMAT): img_id = make_uuid() vol_id = make_uuid() # TODO fix make_volume helper to create the qcow image when needed with fake_env(storage_type) as env: if format == sc.RAW_FORMAT: env.make_volume(size, img_id, vol_id, vol_format=format) vol = env.sd_manifest.produceVolume(img_id, vol_id) yield vol else: chain = make_qemu_chain(env, size, format, 1) yield chain[0]
def test_volume_metadata_io(self): with fake_file_env() as env: size = 1 * MB img_id = make_uuid() vol_id = make_uuid() make_file_volume(env.sd_manifest, size, img_id, vol_id) vol = env.sd_manifest.produceVolume(img_id, vol_id) desc = 'foo' vol.setDescription(desc) # Test that metadata is persisted to our temporary storage area vol = env.sd_manifest.produceVolume(img_id, vol_id) self.assertEqual(desc, vol.getDescription())
def test_get_image_volumes(self): img_id = make_uuid() vol_id = make_uuid() remote_path = "[2001:db8:85a3::8a2e:370:7334]:1234:/path" size = 5 * MEGAB # Simulate a domain with an ipv6 address with fake_env(storage_type='file', remote_path=remote_path) as env: env.make_volume(size, img_id, vol_id) vol = env.sd_manifest.produceVolume(img_id, vol_id) vol_path = vol.getVolumePath() sduuid = fileVolume.getDomUuidFromVolumePath(vol_path) assert vol.getImageVolumes(sduuid, img_id) == [vol_id]
def make_filesd_manifest(mnt_dir, sd_version=3): spuuid = make_uuid() sduuid = make_uuid() domain_path = os.path.join(mnt_dir, sduuid) metafile = get_metafile_path(domain_path) make_file(metafile) metadata = fileSD.FileSDMetadata(metafile) metadata.update(make_sd_metadata(sduuid, version=sd_version, pools=[spuuid])) manifest = fileSD.FileStorageDomainManifest(domain_path, metadata) os.makedirs(os.path.join(manifest.domaindir, sd.DOMAIN_IMAGES)) return manifest
def make_init_params(**kwargs): res = dict( domain=make_uuid(), image=make_uuid(), puuid=make_uuid(), capacity=1024 * MB, format=sc.type2name(sc.RAW_FORMAT), type=sc.type2name(sc.SPARSE_VOL), voltype=sc.type2name(sc.LEAF_VOL), disktype=image.SYSTEM_DISK_TYPE, description="", legality=sc.LEGAL_VOL, generation=sc.DEFAULT_GENERATION) res.update(kwargs) return res
def test_copy_bitmaps_fail_raw_format(user_mount, fake_scheduler, env_type, dst_fmt): job_id = make_uuid() data_center = os.path.join(user_mount.path, "data-center") with make_env(env_type, sc.COW_FORMAT, dst_fmt, sd_version=5, src_qcow2_compat='1.1', data_center=data_center) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] op = qemuimg.bitmap_add(src_vol.getVolumePath(), 'bitmap') op.run() source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = copy_data.Job(job_id, 0, source, dest, copy_bitmaps=True) job.run() # copy bitmaps are not supported for raw volumes assert jobs.STATUS.FAILED == job.status assert 'error' in job.info()
def test_volume_size_alignment(self, size_param): with fake_block_env() as env: sd_id = env.sd_manifest.sdUUID img_id = make_uuid() vol_id = make_uuid() make_block_volume(env.lvm, env.sd_manifest, size_param, img_id, vol_id) vol = env.sd_manifest.produceVolume(img_id, vol_id) extent_size = sc.VG_EXTENT_SIZE_MB * MB expected_size = utils.round(size_param, extent_size) self.assertEqual(expected_size / sc.BLOCK_SIZE, vol.getSize()) self.assertEqual(expected_size, int(env.lvm.getLV(sd_id, vol_id).size)) lv_file_size = os.stat(env.lvm.lvPath(sd_id, vol_id)).st_size self.assertEqual(expected_size, lv_file_size)
def test_intra_domain_copy(env_type, src_fmt, dst_fmt): src_fmt = sc.name2type(src_fmt) dst_fmt = sc.name2type(dst_fmt) job_id = make_uuid() with make_env(env_type, src_fmt, dst_fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] write_qemu_chain(env.src_chain) with pytest.raises(qemuio.VerificationError): verify_qemu_chain(env.dst_chain) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = copy_data.Job(job_id, 0, source, dest) job.run() assert (sorted(expected_locks(src_vol, dst_vol)) == sorted( guarded.context.locks)) assert jobs.STATUS.DONE == job.status assert 100.0 == job.progress assert 'error' not in job.info() verify_qemu_chain(env.dst_chain) assert (sc.fmt2str(dst_fmt) == qemuimg.info( dst_vol.volumePath)['format'])
def test_remove_invalid_bitmap(fake_scheduler, env_type): bitmap = "bitmap" with make_env(env_type, sc.name2type('cow')) as env: base_vol = env.chain[0] # Add bitmap to base volume op = qemuimg.bitmap_add( base_vol.getVolumePath(), bitmap, ) op.run() # Simulate qemu crash, leaving bitmaps with the "in-use" # flag by opening the image for writing and killing the process. qemuio.abort(base_vol.getVolumePath()) generation = base_vol.getMetaParam(sc.GENERATION) vol = { 'endpoint_type': 'div', 'sd_id': base_vol.sdUUID, 'img_id': base_vol.imgUUID, 'vol_id': base_vol.volUUID, 'generation': generation } job = remove_bitmap.Job(make_uuid(), 0, vol, bitmap) job.run() assert jobs.STATUS.DONE == job.status vol_info = qemuimg.info(base_vol.getVolumePath()) bitmaps = vol_info["format-specific"]["data"].get("bitmaps", []) assert not bitmaps assert base_vol.getMetaParam(sc.GENERATION) == generation + 1
def test_bad_vm_configuration_volume(self): """ When copying a volume containing VM configuration information the volume format may be set incorrectly due to an old bug. Check that the workaround we have in place allows the copy to proceed without error. """ job_id = make_uuid() vm_conf_size = workarounds.VM_CONF_SIZE_BLK * sc.BLOCK_SIZE vm_conf_data = "VM Configuration" with self.make_env('file', sc.COW_FORMAT, sc.COW_FORMAT, size=vm_conf_size) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] # Corrupt the COW volume by writing raw data. This simulates how # these "problem" volumes were created in the first place. with open(src_vol.getVolumePath(), "w") as f: f.write(vm_conf_data) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest) job.run() wait_for_job(job) self.assertEqual(jobs.STATUS.DONE, job.status) # Verify that the copy succeeded with open(dst_vol.getVolumePath(), "r") as f: # Qemu pads the file to a 1k boundary with null bytes self.assertTrue(f.read().startswith(vm_conf_data))
def make_sd_metadata(sduuid, version=3, dom_class=sd.DATA_DOMAIN, pools=None): md = FakeMetadata() md[sd.DMDK_SDUUID] = sduuid md[sd.DMDK_VERSION] = version md[sd.DMDK_CLASS] = dom_class md[sd.DMDK_POOLS] = pools if pools is not None else [make_uuid()] return md
def test_volume_size_alignment(self, size_param): with fake_block_env() as env: sd_id = env.sd_manifest.sdUUID img_id = make_uuid() vol_id = make_uuid() make_block_volume(env.lvm, env.sd_manifest, size_param, img_id, vol_id) vol = env.sd_manifest.produceVolume(img_id, vol_id) extent_size = sc.VG_EXTENT_SIZE_MB * MB expected_size = utils.round(size_param, extent_size) self.assertEqual(expected_size // sc.BLOCK_SIZE, vol.getSize()) self.assertEqual(expected_size, int(env.lvm.getLV(sd_id, vol_id).size)) lv_file_size = os.stat(env.lvm.lvPath(sd_id, vol_id)).st_size self.assertEqual(expected_size, lv_file_size)
def test_abort_during_copy(self, env_type): fmt = sc.RAW_FORMAT with self.make_env(env_type, fmt, fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] gen_id = dst_vol.getMetaParam(sc.GENERATION) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID, generation=0) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID, generation=gen_id) fake_convert = FakeQemuConvertChecker(src_vol, dst_vol, wait_for_abort=True) with MonkeyPatchScope([(qemuimg, 'convert', fake_convert)]): job_id = make_uuid() job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest) t = start_thread(job.run) if not fake_convert.ready_event.wait(1): raise RuntimeError("Timeout waiting for thread") job.abort() t.join(1) if t.isAlive(): raise RuntimeError("Timeout waiting for thread") self.assertEqual(jobs.STATUS.ABORTED, job.status) self.assertEqual(sc.ILLEGAL_VOL, dst_vol.getLegality()) self.assertEqual(gen_id, dst_vol.getMetaParam(sc.GENERATION))
def test_subchain_validation(self): job_id = make_uuid() with self.make_env(sd_type='file', chain_len=2) as env: write_qemu_chain(env.chain) base_index = 0 top_index = 1 base_vol = env.chain[base_index] base_vol.setLegality(sc.ILLEGAL_VOL) top_vol = env.chain[top_index] subchain_info = dict(sd_id=top_vol.sdUUID, img_id=top_vol.imgUUID, base_id=base_vol.imgUUID, top_id=top_vol.volUUID, base_generation=0) subchain = merge.SubchainInfo(subchain_info, 0) def fail(): raise se.VolumeIsNotInChain(None, None, None) # We already tested that subchain validate does the right thing, # here we test that this job care to call subchain validate. subchain.validate = fail job = api_merge.Job(job_id, subchain) job.run() self.assertEqual(job.status, jobs.STATUS.FAILED) self.assertEqual(type(job.error), se.VolumeIsNotInChain) # Check that validate is called *before* attempting - verify that # the chain data was *not* merged offset = base_index * KiB pattern = 0xf0 + base_index verify_pattern(base_vol.volumePath, qemuimg.FORMAT.RAW, offset=offset, len=KiB, pattern=pattern) self.assertEqual(base_vol.getMetaParam(sc.GENERATION), 0)
def test_clear_invalid_bitmaps(fake_scheduler, env_type): with make_env(env_type, sc.name2type('cow')) as env: top_vol = env.chain[0] # Add new invalid bitmaps to top volume for bitmap in ['bitmap_1', 'bitmap_2']: op = qemuimg.bitmap_add(top_vol.getVolumePath(), bitmap, enable=False) op.run() # Clear the created bitmap generation = top_vol.getMetaParam(sc.GENERATION) vol = { 'endpoint_type': 'div', 'sd_id': top_vol.sdUUID, 'img_id': top_vol.imgUUID, 'vol_id': top_vol.volUUID, 'generation': generation } job = clear_bitmaps.Job(make_uuid(), 0, vol) job.run() assert jobs.STATUS.DONE == job.status vol_info = qemuimg.info(top_vol.getVolumePath()) assert "bitmaps" not in vol_info["format-specific"]["data"] assert top_vol.getMetaParam(sc.GENERATION) == generation + 1
def test_add_remove_bitmap(fake_scheduler, env_type): bitmap1 = "bitmap1" bitmap2 = "bitmap2" with make_env(env_type, sc.name2type('cow')) as env: top_vol = env.chain[0] # Add bitmaps to the volume for bitmap in [bitmap1, bitmap2]: op = qemuimg.bitmap_add(top_vol.getVolumePath(), bitmap) op.run() # Remove one of the created bitmap generation = top_vol.getMetaParam(sc.GENERATION) vol = { 'endpoint_type': 'div', 'sd_id': top_vol.sdUUID, 'img_id': top_vol.imgUUID, 'vol_id': top_vol.volUUID, 'generation': generation } job = remove_bitmap.Job(make_uuid(), 0, vol, bitmap1) job.run() assert jobs.STATUS.DONE == job.status vol_info = qemuimg.info(top_vol.getVolumePath()) bitmaps = [ b["name"] for b in vol_info["format-specific"]["data"].get("bitmaps", []) ] assert bitmap1 not in bitmaps and bitmap2 in bitmaps assert top_vol.getMetaParam(sc.GENERATION) == generation + 1
def test_abort_during_copy(self, env_type): fmt = sc.RAW_FORMAT with make_env(env_type, fmt, fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] gen_id = dst_vol.getMetaParam(sc.GENERATION) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID, generation=0) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID, generation=gen_id) fake_convert = FakeQemuConvertChecker(src_vol, dst_vol, wait_for_abort=True) with MonkeyPatchScope([(qemuimg, 'convert', fake_convert)]): job_id = make_uuid() job = copy_data.Job(job_id, 0, source, dest) t = start_thread(job.run) if not fake_convert.ready_event.wait(1): raise RuntimeError("Timeout waiting for thread") job.abort() t.join(1) if t.is_alive(): raise RuntimeError("Timeout waiting for thread") self.assertEqual(jobs.STATUS.ABORTED, job.status) self.assertEqual(sc.ILLEGAL_VOL, dst_vol.getLegality()) self.assertEqual(gen_id, dst_vol.getMetaParam(sc.GENERATION))
def test_qcow2_compat(self, env_type, qcow2_compat, sd_version): src_fmt = sc.name2type("cow") dst_fmt = sc.name2type("cow") job_id = make_uuid() with self.make_env(env_type, src_fmt, dst_fmt, sd_version=sd_version, src_qcow2_compat=qcow2_compat) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = copy_data.Job(job_id, 0, source, dest) job.run() wait_for_job(job) actual_compat = qemuimg.info(dst_vol.volumePath)['compat'] self.assertEqual(actual_compat, env.sd_manifest.qcow2_compat())
def test_lookup_exists(self, tmp_vol, fake_sanlock): vol = xlease.LeasesVolume(tmp_vol.backend) with utils.closing(vol): lease_id = make_uuid() add_info = vol.add(lease_id) lookup_info = vol.lookup(lease_id) assert add_info == lookup_info
def test_volume_chain_copy(self, env_type, src_fmt, dst_fmt, copy_seq): src_fmt = sc.name2type(src_fmt) dst_fmt = sc.name2type(dst_fmt) nr_vols = len(copy_seq) with self.make_env(env_type, src_fmt, dst_fmt, chain_length=nr_vols) as env: write_qemu_chain(env.src_chain) for index in copy_seq: job_id = make_uuid() src_vol = env.src_chain[index] dst_vol = env.dst_chain[index] source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = copy_data.Job(job_id, 0, source, dest) job.run() wait_for_job(job) self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)), sorted(guarded.context.locks)) verify_qemu_chain(env.dst_chain)
def test_lookup_updating(self): record = xlease.Record(make_uuid(), 0, updating=True) with make_volume((42, record)) as vol: leases = vol.leases() self.assertTrue(leases[record.resource]["updating"]) with self.assertRaises(xlease.LeaseUpdating): vol.lookup(record.resource)
def test_volume_operation(self, env_type, error, final_legality, final_status, final_gen): job_id = make_uuid() fmt = sc.RAW_FORMAT with self.make_env(env_type, fmt, fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] self.assertEqual(sc.LEGAL_VOL, dst_vol.getLegality()) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID, generation=0) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID, generation=0) fake_convert = FakeQemuConvertChecker(src_vol, dst_vol, error=error) with MonkeyPatchScope([(qemuimg, 'convert', fake_convert)]): job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest) job.run() self.assertEqual(final_status, job.status) self.assertEqual(final_legality, dst_vol.getLegality()) self.assertEqual(final_gen, dst_vol.getMetaParam(sc.GENERATION))
def test_remove_inactive_bitmap(fake_scheduler, env_type): bitmap = "bitmap" with make_env(env_type, sc.name2type('cow')) as env: base_vol = env.chain[0] # Add inactive bitmap to base volume op = qemuimg.bitmap_add(base_vol.getVolumePath(), bitmap, enable=False) op.run() generation = base_vol.getMetaParam(sc.GENERATION) vol = { 'endpoint_type': 'div', 'sd_id': base_vol.sdUUID, 'img_id': base_vol.imgUUID, 'vol_id': base_vol.volUUID, 'generation': generation } job = remove_bitmap.Job(make_uuid(), 0, vol, bitmap) job.run() assert jobs.STATUS.DONE == job.status vol_info = qemuimg.info(base_vol.getVolumePath()) bitmaps = vol_info["format-specific"]["data"].get("bitmaps", []) assert not bitmaps assert base_vol.getMetaParam(sc.GENERATION) == generation + 1
def test_volume_operation(self, env_type, error, final_legality, final_status, final_gen): job_id = make_uuid() fmt = sc.RAW_FORMAT with make_env(env_type, fmt, fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] self.assertEqual(sc.LEGAL_VOL, dst_vol.getLegality()) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID, generation=0) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID, generation=0) fake_convert = FakeQemuConvertChecker(src_vol, dst_vol, error=error) with MonkeyPatchScope([(qemuimg, 'convert', fake_convert)]): job = copy_data.Job(job_id, 0, source, dest) job.run() self.assertEqual(final_status, job.status) self.assertEqual(final_legality, dst_vol.getLegality()) self.assertEqual(final_gen, dst_vol.getMetaParam(sc.GENERATION))
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt): src_fmt = sc.name2type(src_fmt) dst_fmt = sc.name2type(dst_fmt) job_id = make_uuid() with self.make_env(env_type, src_fmt, dst_fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] write_qemu_chain(env.src_chain) self.assertRaises(ChainVerificationError, verify_qemu_chain, env.dst_chain) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest) job.run() wait_for_job(job) self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)), sorted(guarded.context.locks)) self.assertEqual(jobs.STATUS.DONE, job.status) self.assertEqual(100.0, job.progress) self.assertNotIn('error', job.info()) verify_qemu_chain(env.dst_chain) self.assertEqual(sc.fmt2str(dst_fmt), qemuimg.info(dst_vol.volumePath)['format'])
def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt): src_fmt = sc.name2type(src_fmt) dst_fmt = sc.name2type(dst_fmt) job_id = make_uuid() with self.make_env(env_type, src_fmt, dst_fmt) as env: src_vol = env.src_chain[0] dst_vol = env.dst_chain[0] write_qemu_chain(env.src_chain) self.assertRaises(qemuio.VerificationError, verify_qemu_chain, env.dst_chain) source = dict(endpoint_type='div', sd_id=src_vol.sdUUID, img_id=src_vol.imgUUID, vol_id=src_vol.volUUID) dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID, img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID) job = copy_data.Job(job_id, 0, source, dest) job.run() wait_for_job(job) self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)), sorted(guarded.context.locks)) self.assertEqual(jobs.STATUS.DONE, job.status) self.assertEqual(100.0, job.progress) self.assertNotIn('error', job.info()) verify_qemu_chain(env.dst_chain) self.assertEqual(sc.fmt2str(dst_fmt), qemuimg.info(dst_vol.volumePath)['format'])
def test_subchain_validation(self): job_id = make_uuid() with self.make_env(sd_type='file', chain_len=2) as env: write_qemu_chain(env.chain) base_index = 0 top_index = 1 base_vol = env.chain[base_index] base_vol.setLegality(sc.ILLEGAL_VOL) top_vol = env.chain[top_index] subchain_info = dict(sd_id=top_vol.sdUUID, img_id=top_vol.imgUUID, base_id=base_vol.imgUUID, top_id=top_vol.volUUID, base_generation=0) subchain = merge.SubchainInfo(subchain_info, 0) def fail(): raise se.VolumeIsNotInChain(None, None, None) # We already tested that subchain validate does the right thing, # here we test that this job care to call subchain validate. subchain.validate = fail job = storage.sdm.api.merge.Job(job_id, subchain) job.run() wait_for_job(job) self.assertEqual(job.status, jobs.STATUS.FAILED) self.assertEqual(type(job.error), se.VolumeIsNotInChain) # Check that validate is called *before* attempting - verify that # the chain data was *not* merged offset = base_index * 1024 pattern = 0xf0 + base_index qemu_pattern_verify(base_vol.volumePath, qemuimg.FORMAT.RAW, offset=offset, len=1024, pattern=pattern) self.assertEqual(base_vol.getMetaParam(sc.GENERATION), 0)
def test_lookup_updating(self): record = xlease.Record(make_uuid(), 0, updating=True) with make_volume((42, record)) as vol: leases = vol.leases() assert leases[record.resource]["updating"] with pytest.raises(xlease.LeaseUpdating): vol.lookup(record.resource)
def make_env(env_type, base, top): img_id = make_uuid() base_id = make_uuid() top_id = make_uuid() if env_type == 'block' and base.format == 'raw': prealloc = sc.PREALLOCATED_VOL else: prealloc = sc.SPARSE_VOL with fake_env(env_type) as env: env.make_volume(base.virtual * GB, img_id, base_id, vol_format=sc.name2type(base.format), prealloc=prealloc) env.make_volume(top.virtual * GB, img_id, top_id, parent_vol_id=base_id, vol_format=sc.COW_FORMAT) env.subchain = merge.SubchainInfo( dict(sd_id=env.sd_manifest.sdUUID, img_id=img_id, base_id=base_id, top_id=top_id), 0) if env_type == 'block': # Simulate allocation by adjusting the LV sizes env.lvm.extendLV(env.sd_manifest.sdUUID, base_id, base.physical * GB // MB) env.lvm.extendLV(env.sd_manifest.sdUUID, top_id, top.physical * GB // MB) with MonkeyPatch().context() as mp: mp.setattr(guarded, 'context', fake_guarded_context()) mp.setattr(merge, 'sdCache', env.sdcache) mp.setattr(blockVolume, 'rm', FakeResourceManager()) mp.setattr(blockVolume, 'sdCache', env.sdcache) mp.setattr( image.Image, 'getChain', lambda self, sdUUID, imgUUID: [env.subchain.base_vol, env.subchain.top_vol]) mp.setattr(blockVolume.BlockVolume, 'extendSize', partial(fake_blockVolume_extendSize, env)) mp.setattr(fileVolume.FileVolume, 'extendSize', partial(fake_fileVolume_extendSize, env)) yield env
def test_lookup_missing(self, tmp_vol): vol = xlease.LeasesVolume( tmp_vol.backend, alignment=tmp_vol.alignment, block_size=tmp_vol.block_size) with utils.closing(vol): with pytest.raises(se.NoSuchLease): vol.lookup(make_uuid())
def test_volume_metadata_io_block_env(): with fake_block_env() as env: sd_id = env.sd_manifest.sdUUID img_id = make_uuid() vol_id = make_uuid() size = sc.VG_EXTENT_SIZE make_block_volume(env.lvm, env.sd_manifest, size, img_id, vol_id) assert vol_id == env.lvm.getLV(sd_id, vol_id).name vol = env.sd_manifest.produceVolume(img_id, vol_id) assert size == vol.getCapacity() desc = 'foo' vol.setDescription(desc) # Test that metadata is persisted to our temporary storage area. vol = env.sd_manifest.produceVolume(img_id, vol_id) assert desc == vol.getDescription()
def test_remove_missing(self, tmp_vol): vol = xlease.LeasesVolume(tmp_vol.backend, alignment=tmp_vol.alignment, block_size=tmp_vol.block_size) with utils.closing(vol): lease_id = make_uuid() with pytest.raises(se.NoSuchLease): vol.remove(lease_id)
def test_merge_subchain_with_bitmaps( self, sd_type, chain_len, base_index, top_index): job_id = make_uuid() bitmap1_name = 'bitmap1' bitmap2_name = 'bitmap2' with self.make_env( sd_type=sd_type, chain_len=chain_len, base_format=sc.COW_FORMAT, qcow2_compat='1.1') as env: base_vol = env.chain[base_index] top_vol = env.chain[top_index] # Add new bitmap to base_vol and top_vol for vol in [base_vol, top_vol]: op = qemuimg.bitmap_add( vol.getVolumePath(), bitmap1_name, ) op.run() # Add another bitmap to top_vol only # to test add + merge op = qemuimg.bitmap_add( top_vol.getVolumePath(), bitmap2_name, ) op.run() # Writing data to the chain to modify the bitmaps write_qemu_chain(env.chain) subchain_info = dict(sd_id=base_vol.sdUUID, img_id=base_vol.imgUUID, base_id=base_vol.volUUID, top_id=top_vol.volUUID, base_generation=0) subchain = merge.SubchainInfo(subchain_info, 0) job = api_merge.Job(job_id, subchain, merge_bitmaps=True) job.run() self.assertEqual(job.status, jobs.STATUS.DONE) info = qemuimg.info(base_vol.getVolumePath()) # TODO: we should improve this test by adding a # a verification to the extents that are reported # by qemu-nbd. assert info['format-specific']['data']['bitmaps'] == [ { "flags": ["auto"], "name": bitmap1_name, "granularity": 65536 }, { "flags": ["auto"], "name": bitmap2_name, "granularity": 65536 }, ]
def test_add_exists(self, fake_sanlock): with make_volume() as vol: lease_id = make_uuid() lease = vol.add(lease_id) with pytest.raises(xlease.LeaseExists): vol.add(lease_id) res = fake_sanlock.read_resource(lease.path, lease.offset) assert res["lockspace"] == lease.lockspace assert res["resource"] == lease.resource
def test_set_generation(self, env_type): with self.make_env(env_type) as env: vol = env.chain[0] job = update_volume.Job(make_uuid(), 0, make_endpoint_from_volume(vol), dict(generation=44)) job.run() self.assertEqual(jobs.STATUS.DONE, job.status) self.assertEqual(44, vol.getMetaParam(sc.GENERATION))
def test_volume_metadata_io(self): with fake_block_env() as env: sd_id = env.sd_manifest.sdUUID img_id = make_uuid() vol_id = make_uuid() size_mb = sc.VG_EXTENT_SIZE_MB size = size_mb * MB make_block_volume(env.lvm, env.sd_manifest, size, img_id, vol_id) self.assertEqual(vol_id, env.lvm.getLV(sd_id, vol_id).name) vol = env.sd_manifest.produceVolume(img_id, vol_id) self.assertEqual(size, vol.getCapacity()) desc = 'foo' vol.setDescription(desc) # Test that metadata is persisted to our temporary storage area vol = env.sd_manifest.produceVolume(img_id, vol_id) self.assertEqual(desc, vol.getDescription())
def roundtrip(self, mboxfiles, delay, messages): with make_hsm_mailbox(mboxfiles, 7) as hsm_mb: with make_spm_mailbox(mboxfiles) as spm_mm: pool = FakePool(spm_mm) spm_callback = partial(sm.SPM_Extend_Message.processRequest, pool) spm_mm.registerMessageType(sm.EXTEND_CODE, spm_callback) done = threading.Event() start = {} end = {} def reply_msg_callback(vol_data): vol_id = vol_data['volumeID'] assert vol_id in start, "Missing request" assert vol_id not in end, "Duplicate request" end[vol_id] = time.monotonic() log.info("got extension reply for volume %s, elapsed %s", vol_id, end[vol_id] - start[vol_id]) if len(end) == messages: log.info("done gathering all replies") done.set() for _ in range(messages): vol_id = make_uuid() start[vol_id] = time.monotonic() log.info("requesting to extend volume %s (delay=%.3f)", vol_id, delay) hsm_mb.sendExtendMsg(volume_data(vol_id), 2 * GiB, callbackFunction=reply_msg_callback) time.sleep(delay) log.info("waiting for all replies") if not done.wait(MAILER_TIMEOUT): raise RuntimeError("Roundtrip did not finish in time") log.info("waiting for messages clearing in SPM inbox") deadline = time.monotonic() + MAILER_TIMEOUT while True: with io.open(mboxfiles.inbox, "rb") as f: # Skip the event block, checking it is racy since hsm # mail monitor writes events to this block. f.seek(sm.MAILBOX_SIZE) # check that SPM inbox was cleared if f.read(sm.MAILBOX_SIZE) == sm.EMPTYMAILBOX: break if time.monotonic() >= deadline: raise RuntimeError("Timeout clearing SPM inbox") time.sleep(EVENT_INTERVAL) times = [end[k] - start[k] for k in start] times.sort() return times
def test_lookup_updating(self, tmp_vol): record = xlease.Record(make_uuid(), 0, updating=True) tmp_vol.write_records((42, record)) vol = xlease.LeasesVolume(tmp_vol.backend) with utils.closing(vol): leases = vol.leases() assert leases[record.resource]["updating"] with pytest.raises(xlease.LeaseUpdating): vol.lookup(record.resource)
def test_leases(self): with make_volume() as vol: uuid = make_uuid() lease_info = vol.add(uuid) leases = vol.leases() self.assertEqual(len(leases), 1) self.assertEqual(leases[uuid]["offset"], xlease.LEASE_BASE) self.assertEqual(leases[uuid]["state"], "USED") self.assertEqual(leases[uuid]["modified"], lease_info.modified)
def make_env(env_type, base, top): img_id = make_uuid() base_id = make_uuid() top_id = make_uuid() if env_type == 'block' and base.format == 'raw': prealloc = sc.PREALLOCATED_VOL else: prealloc = sc.SPARSE_VOL with fake_env(env_type) as env: env.make_volume(base.virtual * GB, img_id, base_id, vol_format=sc.name2type(base.format), prealloc=prealloc) env.make_volume(top.virtual * GB, img_id, top_id, parent_vol_id=base_id, vol_format=sc.COW_FORMAT) env.subchain = merge.SubchainInfo( dict(sd_id=env.sd_manifest.sdUUID, img_id=img_id, base_id=base_id, top_id=top_id), 0) if env_type == 'block': # Simulate allocation by adjusting the LV sizes env.lvm.extendLV(env.sd_manifest.sdUUID, base_id, base.physical * GB // MB) env.lvm.extendLV(env.sd_manifest.sdUUID, top_id, top.physical * GB // MB) with MonkeyPatch().context() as mp: mp.setattr(guarded, 'context', fake_guarded_context()) mp.setattr(merge, 'sdCache', env.sdcache) mp.setattr(blockVolume, 'rm', FakeResourceManager()) mp.setattr(blockVolume, 'sdCache', env.sdcache) mp.setattr( image.Image, 'getChain', lambda self, sdUUID, imgUUID: [env.subchain.base_vol, env.subchain.top_vol]) mp.setattr( blockVolume.BlockVolume, 'extendSize', partial(fake_blockVolume_extendSize, env)) mp.setattr( fileVolume.FileVolume, 'extendSize', partial(fake_fileVolume_extendSize, env)) yield env
def test_create_additional_raw_vol(self): with self.fake_env() as env: first = env.sd_manifest.get_volume_artifacts( self.img_id, self.vol_id) first.create(*BASE_PARAMS[sc.RAW_FORMAT]) first.commit() second = env.sd_manifest.get_volume_artifacts( self.img_id, make_uuid()) self.assertRaises(se.InvalidParameterException, second.create, *BASE_PARAMS[sc.RAW_FORMAT])
def test_metaslot_selection(self, used_slots, free_slot): with fake_block_env() as env: for offset in used_slots: lv = make_uuid() sduuid = env.sd_manifest.sdUUID env.lvm.createLV(sduuid, lv, VOLSIZE / MB) tag = sc.TAG_PREFIX_MD + str(offset) env.lvm.addtag(sduuid, lv, tag) with env.sd_manifest.acquireVolumeMetadataSlot(None, 1) as mdSlot: self.assertEqual(mdSlot, free_slot)
def make_qemu_chain(env, size, base_vol_fmt, chain_len, qcow2_compat='0.10', prealloc=sc.SPARSE_VOL): vol_list = [] img_id = make_uuid() parent_vol_id = sc.BLANK_UUID vol_fmt = base_vol_fmt for i in range(chain_len): vol_id = make_uuid() if parent_vol_id != sc.BLANK_UUID: vol_fmt = sc.COW_FORMAT vol_type = sc.LEAF_VOL if i == chain_len - 1 else sc.INTERNAL_VOL env.make_volume(size, img_id, vol_id, parent_vol_id=parent_vol_id, vol_format=vol_fmt, vol_type=vol_type, prealloc=prealloc, qcow2_compat=qcow2_compat) vol = env.sd_manifest.produceVolume(img_id, vol_id) vol_list.append(vol) parent_vol_id = vol_id return vol_list
def test_add(self, fake_sanlock): with make_volume() as vol: lease_id = make_uuid() lease = vol.add(lease_id) assert lease.lockspace == vol.lockspace assert lease.resource == lease_id assert lease.path == vol.path res = fake_sanlock.read_resource(lease.path, lease.offset) assert res["lockspace"] == lease.lockspace assert res["resource"] == lease.resource
def _metaslot_selection(self, used_slots, free_slot, sd_version): with fake_block_env(sd_version=sd_version) as env: for offset in used_slots: lv = make_uuid() sduuid = env.sd_manifest.sdUUID env.lvm.createLV(sduuid, lv, VOLSIZE // MiB) tag = sc.TAG_PREFIX_MD + str(offset) env.lvm.changeLVsTags(sduuid, (lv,), addTags=(tag,)) with env.sd_manifest.acquireVolumeMetadataSlot(None) as mdSlot: assert mdSlot == free_slot
def _metaslot_selection(self, used_slots, free_slot, sd_version): with fake_block_env(sd_version=sd_version) as env: for offset in used_slots: lv = make_uuid() sduuid = env.sd_manifest.sdUUID env.lvm.createLV(sduuid, lv, VOLSIZE // MB) tag = sc.TAG_PREFIX_MD + str(offset) env.lvm.addtag(sduuid, lv, tag) with env.sd_manifest.acquireVolumeMetadataSlot(None) as mdSlot: assert mdSlot == free_slot