예제 #1
0
파일: volume.py 프로젝트: bronhaim/vdsm
    def rebaseVolumeRollback(cls, taskObj, sdUUID, srcImg, srcVol, dstFormat,
                             srcParent, unsafe):
        """
        Rebase volume rollback
        """
        cls.log.info(
            'rebase volume rollback (sdUUID=%s srcImg=%s srcVol=%s '
            'dstFormat=%s srcParent=%s)', sdUUID, srcImg, srcVol, dstFormat,
            srcParent)

        imageResourcesNamespace = sd.getNamespace(sc.IMAGE_NAMESPACE, sdUUID)

        with rm.acquireResource(imageResourcesNamespace, srcImg, rm.EXCLUSIVE):
            vol = sdCache.produce(sdUUID).produceVolume(srcImg, srcVol)
            vol.prepare(rw=True, chainrw=True, setrw=True)

            volumePath = vol.getVolumePath()
            backingVolPath = getBackingVolumePath(srcImg, srcParent)

            try:
                qemuimg.rebase(volumePath, backingVolPath,
                               sc.fmt2str(vol.getFormat()),
                               sc.fmt2str(int(dstFormat)),
                               misc.parseBool(unsafe), vars.task.aborting)
                vol.setParent(srcParent)
                vol.recheckIfLeaf()
            except qemuimg.QImgError:
                cls.log.exception(
                    'cannot rollback rebase for volume %s on '
                    '%s', volumePath, backingVolPath)
                raise se.MergeVolumeRollbackError(srcVol)
            finally:
                vol.teardown(sdUUID, srcVol)
예제 #2
0
    def test_chain_after_finalize(self, base_fmt):
        with self.make_env(format=base_fmt, chain_len=3) as env:
            base_vol = env.chain[0]
            # We write data to the base and will read it from the child volume
            # to verify that the chain is valid after qemu-rebase.
            offset = 0
            pattern = 0xf0
            length = 1024
            qemuio.write_pattern(base_vol.volumePath,
                                 sc.fmt2str(base_vol.getFormat()),
                                 offset=offset,
                                 len=length,
                                 pattern=pattern)

            top_vol = env.chain[1]
            child_vol = env.chain[2]

            subchain_info = dict(sd_id=base_vol.sdUUID,
                                 img_id=base_vol.imgUUID,
                                 base_id=base_vol.volUUID,
                                 top_id=top_vol.volUUID,
                                 base_generation=0)
            subchain = merge.SubchainInfo(subchain_info, 0)

            merge.finalize(subchain)

            qemuio.verify_pattern(child_vol.volumePath,
                                  sc.fmt2str(child_vol.getFormat()),
                                  offset=offset,
                                  len=length,
                                  pattern=pattern)
예제 #3
0
 def clone(self, dstPath, volFormat):
     """
     Clone self volume to the specified dst_image_dir/dst_volUUID
     """
     wasleaf = False
     taskName = "parent volume rollback: " + self.volUUID
     vars.task.pushRecovery(
         task.Recovery(taskName, "volume", "Volume",
                       "parentVolumeRollback",
                       [self.sdUUID, self.imgUUID, self.volUUID]))
     if self.isLeaf():
         wasleaf = True
         self.setInternal()
     try:
         self.prepare(rw=False)
         self.log.debug('cloning volume %s to %s', self.volumePath,
                        dstPath)
         parent = getBackingVolumePath(self.imgUUID, self.volUUID)
         domain = sdCache.produce(self.sdUUID)
         qemuimg.create(dstPath, backing=parent,
                        format=sc.fmt2str(volFormat),
                        qcow2Compat=domain.qcow2_compat(),
                        backingFormat=sc.fmt2str(self.getFormat()))
         self.teardown(self.sdUUID, self.volUUID)
     except Exception as e:
         self.log.exception('cannot clone image %s volume %s to %s',
                            self.imgUUID, self.volUUID, dstPath)
         # FIXME: might race with other clones
         if wasleaf:
             self.setLeaf()
         self.teardown(self.sdUUID, self.volUUID)
         raise se.CannotCloneVolume(self.volumePath, dstPath, str(e))
예제 #4
0
    def rebaseVolumeRollback(cls, taskObj, sdUUID, srcImg,
                             srcVol, dstFormat, srcParent, unsafe):
        """
        Rebase volume rollback
        """
        cls.log.info('rebase volume rollback (sdUUID=%s srcImg=%s srcVol=%s '
                     'dstFormat=%s srcParent=%s)', sdUUID, srcImg, srcVol,
                     dstFormat, srcParent)

        imageResourcesNamespace = sd.getNamespace(sc.IMAGE_NAMESPACE, sdUUID)

        with rm.acquireResource(imageResourcesNamespace, srcImg, rm.EXCLUSIVE):
            vol = sdCache.produce(sdUUID).produceVolume(srcImg, srcVol)
            vol.prepare(rw=True, chainrw=True, setrw=True)

            volumePath = vol.getVolumePath()
            backingVolPath = getBackingVolumePath(srcImg, srcParent)

            try:
                qemuimg.rebase(volumePath, backingVolPath,
                               sc.fmt2str(vol.getFormat()),
                               sc.fmt2str(int(dstFormat)),
                               misc.parseBool(unsafe), vars.task.aborting)
                vol.setParent(srcParent)
                vol.recheckIfLeaf()
            except qemuimg.QImgError:
                cls.log.exception('cannot rollback rebase for volume %s on '
                                  '%s', volumePath, backingVolPath)
                raise se.MergeVolumeRollbackError(srcVol)
            finally:
                vol.teardown(sdUUID, srcVol)
예제 #5
0
파일: merge_test.py 프로젝트: oVirt/vdsm
    def test_chain_after_finalize(self, base_fmt):
        with self.make_env(format=base_fmt, chain_len=3) as env:
            base_vol = env.chain[0]
            # We write data to the base and will read it from the child volume
            # to verify that the chain is valid after qemu-rebase.
            offset = 0
            pattern = 0xf0
            length = 1024
            qemuio.write_pattern(
                base_vol.volumePath,
                sc.fmt2str(base_vol.getFormat()),
                offset=offset,
                len=length,
                pattern=pattern)

            top_vol = env.chain[1]
            child_vol = env.chain[2]

            subchain_info = dict(sd_id=base_vol.sdUUID,
                                 img_id=base_vol.imgUUID,
                                 base_id=base_vol.volUUID,
                                 top_id=top_vol.volUUID,
                                 base_generation=0)
            subchain = merge.SubchainInfo(subchain_info, 0)

            merge.finalize(subchain)

            qemuio.verify_pattern(
                child_vol.volumePath,
                sc.fmt2str(child_vol.getFormat()),
                offset=offset,
                len=length,
                pattern=pattern)
예제 #6
0
def verify_qemu_chain(vol_list):
    # Check the integrity of a volume chain by reading the leaf volume
    # and verifying the pattern written by write_chain.  Also, check each
    # volume in the chain to ensure it contains the correct data.
    top_vol = vol_list[-1]
    top_vol_fmt = sc.fmt2str(top_vol.getFormat())
    for i, vol in enumerate(vol_list):
        offset = i * 1024
        pattern = 0xf0 + i

        # Check that the correct pattern can be read through the top volume
        qemu_pattern_verify(top_vol.volumePath,
                            top_vol_fmt,
                            offset=offset,
                            len=1024,
                            pattern=pattern)

        # Check the volume where the pattern was originally written
        vol_fmt = sc.fmt2str(vol.getFormat())
        qemu_pattern_verify(vol.volumePath,
                            vol_fmt,
                            offset=offset,
                            len=1024,
                            pattern=pattern)

        # Check that the next offset contains zeroes.  If we know this layer
        # has zeroes at next_offset we can be sure that data read at the same
        # offset in the next layer belongs to that layer.
        next_offset = (i + 1) * 1024
        qemu_pattern_verify(vol.volumePath,
                            vol_fmt,
                            offset=next_offset,
                            len=1024,
                            pattern=0)
예제 #7
0
    def rebase(self, backingVol, backingVolPath, backingFormat, unsafe,
               rollback):
        """
        Rebase volume on top of new backing volume
        """
        if rollback:
            pvol = self.getParentVolume()
            if not pvol:
                self.log.warn("Can't rebase volume %s, parent missing",
                              self.volUUID)
                return

            name = "Merge volume: " + self.volUUID
            vars.task.pushRecovery(
                task.Recovery(name, "volume", "Volume",
                              "rebaseVolumeRollback",
                              [self.sdUUID, self.getImage(),
                                  self.volUUID, str(pvol.getFormat()),
                                  pvol.volUUID, str(True)]))

        volumePath = self.getVolumePath()

        try:
            qemuimg.rebase(volumePath, backingVolPath,
                           sc.fmt2str(self.getFormat()),
                           sc.fmt2str(backingFormat), unsafe,
                           vars.task.aborting)
        except qemuimg.QImgError:
            self.log.exception('cannot rebase volume %s on %s', volumePath,
                               backingVolPath)
            raise se.MergeSnapshotsError(self.volUUID)

        self.setParent(backingVol)
        self.recheckIfLeaf()
예제 #8
0
def verify_qemu_chain(vol_list):
    # Check the integrity of a volume chain by reading the leaf volume
    # and verifying the pattern written by write_chain.  Also, check each
    # volume in the chain to ensure it contains the correct data.
    top_vol = vol_list[-1]
    top_vol_fmt = sc.fmt2str(top_vol.getFormat())
    for i, vol in enumerate(vol_list):
        offset = i * 1024
        pattern = 0xf0 + i

        # Check that the correct pattern can be read through the top volume
        qemu_pattern_verify(top_vol.volumePath, top_vol_fmt, offset=offset,
                            len=1024, pattern=pattern)

        # Check the volume where the pattern was originally written
        vol_fmt = sc.fmt2str(vol.getFormat())
        qemu_pattern_verify(vol.volumePath, vol_fmt, offset=offset, len=1024,
                            pattern=pattern)

        # Check that the next offset contains zeroes.  If we know this layer
        # has zeroes at next_offset we can be sure that data read at the same
        # offset in the next layer belongs to that layer.
        next_offset = (i + 1) * 1024
        qemu_pattern_verify(vol.volumePath, vol_fmt, offset=next_offset,
                            len=1024, pattern=0)
예제 #9
0
파일: volume.py 프로젝트: bronhaim/vdsm
 def clone(self, dstPath, volFormat):
     """
     Clone self volume to the specified dst_image_dir/dst_volUUID
     """
     wasleaf = False
     taskName = "parent volume rollback: " + self.volUUID
     vars.task.pushRecovery(
         task.Recovery(taskName, "volume", "Volume", "parentVolumeRollback",
                       [self.sdUUID, self.imgUUID, self.volUUID]))
     if self.isLeaf():
         wasleaf = True
         self.setInternal()
     try:
         self.prepare(rw=False)
         self.log.debug('cloning volume %s to %s', self.volumePath, dstPath)
         parent = getBackingVolumePath(self.imgUUID, self.volUUID)
         domain = sdCache.produce(self.sdUUID)
         qemuimg.create(dstPath,
                        backing=parent,
                        format=sc.fmt2str(volFormat),
                        qcow2Compat=domain.qcow2_compat(),
                        backingFormat=sc.fmt2str(self.getFormat()))
         self.teardown(self.sdUUID, self.volUUID)
     except Exception as e:
         self.log.exception('cannot clone image %s volume %s to %s',
                            self.imgUUID, self.volUUID, dstPath)
         # FIXME: might race with other clones
         if wasleaf:
             self.setLeaf()
         self.teardown(self.sdUUID, self.volUUID)
         raise se.CannotCloneVolume(self.volumePath, dstPath, str(e))
예제 #10
0
파일: image.py 프로젝트: chaodongqu/vdsm
    def sparsify(self, tmpSdUUID, tmpImgUUID, tmpVolUUID, dstSdUUID,
                 dstImgUUID, dstVolUUID):
        """
        Reduce sparse image size by converting free space on image to free
        space on storage domain using virt-sparsify.
        """
        self.log.info("tmpSdUUID=%s, tmpImgUUID=%s, tmpVolUUID=%s, "
                      "dstSdUUID=%s, dstImgUUID=%s, dstVolUUID=%s", tmpSdUUID,
                      tmpImgUUID, tmpVolUUID, dstSdUUID, dstImgUUID,
                      dstVolUUID)

        tmpVolume = self._getSparsifyVolume(tmpSdUUID, tmpImgUUID, tmpVolUUID)
        dstVolume = self._getSparsifyVolume(dstSdUUID, dstImgUUID, dstVolUUID)

        if not dstVolume.isSparse():
            raise se.VolumeNotSparse()

        srcVolume = self._getSparsifyVolume(tmpSdUUID, tmpImgUUID,
                                            tmpVolume.getParent())

        tmpVolume.prepare()
        try:
            dstVolume.prepare()
            try:
                # By definition "sparsification" is implemented writing a file
                # with zeroes as large as the entire file-system. So at least
                # tmpVolume needs to be as large as the virtual disk size for
                # the worst case.
                # TODO: Some extra space may be needed for QCOW2 headers
                tmpVolume.extend(tmpVolume.getCapacity())
                # For the dstVolume we may think of an optimization where the
                # extension is as large as the source (and at the end we
                # shrinkToOptimalSize).
                # TODO: Extend the dstVolume only as much as the actual size of
                # srcVolume
                # TODO: Some extra space may be needed for QCOW2 headers
                dstVolume.extend(tmpVolume.getCapacity())

                srcFormat = sc.fmt2str(srcVolume.getFormat())
                dstFormat = sc.fmt2str(dstVolume.getFormat())

                virtsparsify.sparsify(srcVolume.getVolumePath(),
                                      tmpVolume.getVolumePath(),
                                      dstVolume.getVolumePath(),
                                      src_format=srcFormat,
                                      dst_format=dstFormat)
            except Exception:
                self.log.exception('Unexpected error sparsifying %s',
                                   tmpVolUUID)
                raise se.CannotSparsifyVolume(tmpVolUUID)
            finally:
                dstVolume.teardown(sdUUID=dstSdUUID, volUUID=dstVolUUID)
        finally:
            tmpVolume.teardown(sdUUID=tmpSdUUID, volUUID=tmpVolUUID)

        self._shrinkVolumeToOptimalSize(tmpVolume)
        self._shrinkVolumeToOptimalSize(dstVolume)
예제 #11
0
def test_copy_data_collapse(tmpdir, tmp_repo, fake_access, fake_rescan, tmp_db,
                            fake_task, fake_scheduler, monkeypatch,
                            dest_format, sd_version):
    dom = tmp_repo.create_localfs_domain(name="domain", version=sd_version)

    chain_size = 3
    volumes = create_chain(dom, chain_size)
    dest_img_id = str(uuid.uuid4())
    dest_vol_id = str(uuid.uuid4())

    length = MiB

    # Write some data to each layer
    for i, vol in enumerate(volumes):
        qemuio.write_pattern(vol.getVolumePath(),
                             sc.fmt2str(vol.getFormat()),
                             offset=(i * length))

    # The last volume in the chain is the leaf
    source_leaf_vol = volumes[-1]
    dest_vol = create_volume(dom,
                             dest_img_id,
                             dest_vol_id,
                             volFormat=dest_format)

    source = dict(endpoint_type='div',
                  sd_id=source_leaf_vol.sdUUID,
                  img_id=source_leaf_vol.imgUUID,
                  vol_id=source_leaf_vol.volUUID)
    dest = dict(endpoint_type='div',
                sd_id=source_leaf_vol.sdUUID,
                img_id=dest_img_id,
                vol_id=dest_vol_id)

    # Run copy_data from the source chain to dest_vol, essentially
    # executing qemu-img convert
    job = copy_data.Job(str(uuid.uuid4()), 0, source, dest)
    monkeypatch.setattr(guarded, 'context', fake_guarded_context())
    job.run()

    # Source chain and destination image must have the same data but allocation
    # may differ.
    op = qemuimg.compare(source_leaf_vol.getVolumePath(),
                         dest_vol.getVolumePath(),
                         img1_format='qcow2',
                         img2_format=sc.fmt2str(dest_format),
                         strict=False)
    op.run()

    # Destination actual size should be smaller than source chain actual size,
    # since we have only one qcow2 header (qcow2), or no header (raw).
    src_actual_size = sum(
        qemuimg.info(vol.getVolumePath())["actualsize"] for vol in volumes)
    dst_actual_size = qemuimg.info(dest_vol.getVolumePath())["actualsize"]
    assert dst_actual_size < src_actual_size
예제 #12
0
    def estimate_qcow2_size(self, src_vol_params, dst_sd_id):
        """
        Calculate volume allocation size for converting raw/qcow2
        source volume to qcow2 volume on destination storage domain.

        Arguments:
            src_vol_params(dict): Dictionary returned from
                                  `storage.volume.Volume.getVolumeParams()`
            dst_sd_id(str) : Destination volume storage domain id

        Returns:
            Volume allocation in bytes
        """
        # measure required size.
        qemu_measure = qemuimg.measure(image=src_vol_params['path'],
                                       format=sc.fmt2str(
                                           src_vol_params['volFormat']),
                                       output_format=qemuimg.FORMAT.QCOW2)

        # Adds extra room so we don't have to extend this disk immediately
        # when a vm is started.
        chunk_size_mb = config.getint("irs", "volume_utilization_chunk_mb")
        chunk_size = chunk_size_mb * MiB
        required = (qemu_measure["required"] + chunk_size)
        # Limit estimates size by maximum size.
        vol_class = sdCache.produce(dst_sd_id).getVolumeClass()
        max_size = vol_class.max_size(src_vol_params['capacity'],
                                      sc.COW_FORMAT)
        allocation = min(required, max_size)

        # Return estimated size of allocation.
        self.log.debug("Estimated allocation for qcow2 volume:"
                       "%d", allocation)
        return allocation
예제 #13
0
    def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
        src_fmt = sc.name2type(src_fmt)
        dst_fmt = sc.name2type(dst_fmt)
        job_id = make_uuid()

        with self.make_env(env_type, src_fmt, dst_fmt) as env:
            src_vol = env.src_chain[0]
            dst_vol = env.dst_chain[0]
            write_qemu_chain(env.src_chain)
            self.assertRaises(ChainVerificationError,
                              verify_qemu_chain, env.dst_chain)

            source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
                          img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
            dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
                        img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
            job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)

            job.run()
            wait_for_job(job)
            self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)),
                             sorted(guarded.context.locks))

            self.assertEqual(jobs.STATUS.DONE, job.status)
            self.assertEqual(100.0, job.progress)
            self.assertNotIn('error', job.info())
            verify_qemu_chain(env.dst_chain)
            self.assertEqual(sc.fmt2str(dst_fmt),
                             qemuimg.info(dst_vol.volumePath)['format'])
예제 #14
0
파일: merge.py 프로젝트: vjuranek/vdsm
    def _run(self):
        self.log.info("Merging subchain %s", self.subchain)
        with guarded.context(self.subchain.locks):
            self.subchain.validate()
            with self.subchain.prepare(), self.subchain.volume_operation():
                top_vol_path = self.subchain.top_vol.getVolumePath()
                base_vol_path = self.subchain.base_vol.getVolumePath()
                self.log.info("Committing data from %s to %s", top_vol_path,
                              base_vol_path)

                self.operation = qemuimg.commit(
                    top_vol_path,
                    topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
                    base=base_vol_path)
                self.operation.run()

                if (self.subchain.base_vol.getFormat() == sc.COW_FORMAT
                        and self.merge_bitmaps):
                    self.log.info("Merging bitmaps from %s to %s",
                                  top_vol_path, base_vol_path)
                    # Add and merge all the bitmaps from top_vol that don't
                    # exist on the base_vol and not handled by block-commit.
                    base_parent_vol = self.subchain.base_vol.getParentVolume()
                    base_parent_path = (base_parent_vol.getVolumePath()
                                        if base_parent_vol else None)
                    bitmaps.merge_bitmaps(base_vol_path,
                                          top_vol_path,
                                          base_parent_path=base_parent_path)
예제 #15
0
    def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
        src_fmt = sc.name2type(src_fmt)
        dst_fmt = sc.name2type(dst_fmt)
        job_id = make_uuid()

        with self.get_vols(env_type, src_fmt,
                           dst_fmt) as (src_chain, dst_chain):
            src_vol = src_chain[0]
            dst_vol = dst_chain[0]
            write_qemu_chain(src_chain)
            self.assertRaises(ChainVerificationError, verify_qemu_chain,
                              dst_chain)

            source = dict(endpoint_type='div',
                          sd_id=src_vol.sdUUID,
                          img_id=src_vol.imgUUID,
                          vol_id=src_vol.volUUID)
            dest = dict(endpoint_type='div',
                        sd_id=dst_vol.sdUUID,
                        img_id=dst_vol.imgUUID,
                        vol_id=dst_vol.volUUID)
            job = storage.sdm.api.copy_data.Job(job_id, 0, source, dest)

            job.run()
            wait_for_job(job)
            self.assertEqual(sorted(self.expected_locks(src_vol, dst_vol)),
                             sorted(guarded.context.locks))

            self.assertEqual(jobs.STATUS.DONE, job.status)
            self.assertEqual(100.0, job.progress)
            self.assertNotIn('error', job.info())
            verify_qemu_chain(dst_chain)
            self.assertEqual(sc.fmt2str(dst_fmt),
                             qemuimg.info(dst_vol.volumePath)['format'])
예제 #16
0
def test_intra_domain_copy(env_type, src_fmt, dst_fmt):
    src_fmt = sc.name2type(src_fmt)
    dst_fmt = sc.name2type(dst_fmt)
    job_id = make_uuid()

    with make_env(env_type, src_fmt, dst_fmt) as env:
        src_vol = env.src_chain[0]
        dst_vol = env.dst_chain[0]
        write_qemu_chain(env.src_chain)
        with pytest.raises(qemuio.VerificationError):
            verify_qemu_chain(env.dst_chain)

        source = dict(endpoint_type='div',
                      sd_id=src_vol.sdUUID,
                      img_id=src_vol.imgUUID,
                      vol_id=src_vol.volUUID)
        dest = dict(endpoint_type='div',
                    sd_id=dst_vol.sdUUID,
                    img_id=dst_vol.imgUUID,
                    vol_id=dst_vol.volUUID)
        job = copy_data.Job(job_id, 0, source, dest)

        job.run()
        assert (sorted(expected_locks(src_vol, dst_vol)) == sorted(
            guarded.context.locks))

        assert jobs.STATUS.DONE == job.status
        assert 100.0 == job.progress
        assert 'error' not in job.info()
        verify_qemu_chain(env.dst_chain)
        assert (sc.fmt2str(dst_fmt) == qemuimg.info(
            dst_vol.volumePath)['format'])
예제 #17
0
    def _create_cow_volume(
            cls, dom, vol_id, capacity, vol_path, initial_size, vol_parent,
            img_id, src_img_id, src_vol_id):
        """
        specific implementation of _create() for COW volumes.
        All the exceptions are properly handled and logged in volume.create()
        """
        if initial_size:
            cls.log.error("initial size is not supported "
                          "for file-based volumes")
            raise se.InvalidParameterException("initial size", initial_size)

        cls._truncate_volume(vol_path, 0, vol_id, dom)

        if not vol_parent:
            cls.log.info("Request to create COW volume %s with capacity = %s",
                         vol_path, capacity)

            operation = qemuimg.create(vol_path,
                                       size=capacity,
                                       format=sc.fmt2str(sc.COW_FORMAT),
                                       qcow2Compat=dom.qcow2_compat())
            operation.run()
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
                         "with capacity %s",
                         img_id, vol_id, src_img_id, src_vol_id, capacity)
            vol_parent.clone(vol_path, sc.COW_FORMAT, capacity)

        # Forcing the volume permissions in case one of the tools we use
        # (dd, qemu-img, etc.) will mistakenly change the file permissions.
        cls._set_permissions(vol_path, dom)

        return (vol_path,)
예제 #18
0
    def test_intra_domain_copy(self, env_type, src_fmt, dst_fmt):
        src_fmt = sc.name2type(src_fmt)
        dst_fmt = sc.name2type(dst_fmt)
        job_id = str(uuid.uuid4())

        with self.get_vols(env_type, src_fmt, dst_fmt) as (src_chain,
                                                           dst_chain):
            src_vol = src_chain[0]
            dst_vol = dst_chain[0]
            write_qemu_chain(src_chain)
            self.assertRaises(ChainVerificationError,
                              verify_qemu_chain, dst_chain)

            source = dict(endpoint_type='div', sd_id=src_vol.sdUUID,
                          img_id=src_vol.imgUUID, vol_id=src_vol.volUUID)
            dest = dict(endpoint_type='div', sd_id=dst_vol.sdUUID,
                        img_id=dst_vol.imgUUID, vol_id=dst_vol.volUUID)
            job = storage.sdm.api.copy_data.Job(job_id, None, source, dest)
            job.run()
            wait_for_job(job)

            self.assertEqual(jobs.STATUS.DONE, job.status)
            self.assertEqual(100.0, job.progress)
            self.assertNotIn('error', job.info())
            verify_qemu_chain(dst_chain)
            self.assertEqual(sc.fmt2str(dst_fmt),
                             qemuimg.info(dst_vol.volumePath)['format'])
예제 #19
0
 def test_optimal_size_cow_leaf_not_empty(self):
     # verify that optimal size is limited to max size.
     with self.make_volume(size=GIB, format=sc.COW_FORMAT) as vol:
         qemu_pattern_write(path=vol.volumePath,
                            format=sc.fmt2str(vol.getFormat()),
                            len=200 * MEGAB)
         max_size = vol.max_size(GIB, vol.getFormat())
         self.assertEqual(vol.optimal_size(), max_size)
예제 #20
0
 def test_optimal_size_cow_leaf_not_empty(self):
     # verify that optimal size is limited to max size.
     with self.make_volume(size=GIB, format=sc.COW_FORMAT) as vol:
         qemu_pattern_write(path=vol.volumePath,
                            format=sc.fmt2str(vol.getFormat()),
                            len=200 * MEGAB)
         max_size = vol.max_size(GIB, vol.getFormat())
         self.assertEqual(vol.optimal_size(), max_size)
예제 #21
0
    def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath,
                initialSize=None):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """
        if initialSize:
            cls.log.error("initialSize is not supported for file-based "
                          "volumes")
            raise se.InvalidParameterException("initial size",
                                               initialSize)

        sizeBytes = size * BLOCK_SIZE
        truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0

        try:
            oop.getProcessPool(dom.sdUUID).truncateFile(
                volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
                creatExcl=True)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise se.VolumeAlreadyExists(volUUID)
            raise

        if preallocate == sc.PREALLOCATED_VOL:
            try:
                operation = fallocate.allocate(volPath,
                                               sizeBytes)
                with vars.task.abort_callback(operation.abort):
                    with utils.stopwatch("Preallocating volume %s" % volPath):
                        operation.run()
            except exception.ActionStopped:
                raise
            except Exception:
                cls.log.error("Unexpected error", exc_info=True)
                raise se.VolumesZeroingError(volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with size = %s "
                         "sectors", sc.type2name(volFormat), volPath,
                         size)
            if volFormat == sc.COW_FORMAT:
                qemuimg.create(volPath,
                               size=sizeBytes,
                               format=sc.fmt2str(volFormat),
                               qcow2Compat=dom.qcow2_compat())
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID)
            volParent.clone(volPath, volFormat)

        # Forcing the volume permissions in case one of the tools we use
        # (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
        dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)

        return (volPath,)
예제 #22
0
파일: fileVolume.py 프로젝트: EdDev/vdsm
    def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath,
                initialSize=None):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """
        if initialSize:
            cls.log.error("initialSize is not supported for file-based "
                          "volumes")
            raise se.InvalidParameterException("initial size",
                                               initialSize)

        sizeBytes = size * BLOCK_SIZE
        truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0

        try:
            oop.getProcessPool(dom.sdUUID).truncateFile(
                volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
                creatExcl=True)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise se.VolumeAlreadyExists(volUUID)
            raise

        if preallocate == sc.PREALLOCATED_VOL:
            try:
                operation = fallocate.allocate(volPath,
                                               sizeBytes)
                with vars.task.abort_callback(operation.abort):
                    with utils.stopwatch("Preallocating volume %s" % volPath):
                        operation.run()
            except exception.ActionStopped:
                raise
            except Exception:
                cls.log.error("Unexpected error", exc_info=True)
                raise se.VolumesZeroingError(volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with size = %s "
                         "sectors", sc.type2name(volFormat), volPath,
                         size)
            if volFormat == sc.COW_FORMAT:
                qemuimg.create(volPath,
                               size=sizeBytes,
                               format=sc.fmt2str(volFormat),
                               qcow2Compat=dom.qcow2_compat())
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID)
            volParent.clone(volPath, volFormat)

        # Forcing the volume permissions in case one of the tools we use
        # (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
        dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)

        return (volPath,)
예제 #23
0
파일: merge.py 프로젝트: nirs/vdsm
def _rebase_operation(base, child):
    backing = volume.getBackingVolumePath(base.imgUUID, base.volUUID)
    backing_format = sc.fmt2str(base.getFormat())
    operation = qemuimg.rebase(image=child.volumePath,
                               backing=backing,
                               format=qemuimg.FORMAT.QCOW2,
                               backingFormat=backing_format,
                               unsafe=True)
    return operation
예제 #24
0
def _rebase_operation(base, child):
    backing = volume.getBackingVolumePath(base.imgUUID, base.volUUID)
    backing_format = sc.fmt2str(base.getFormat())
    operation = qemuimg.rebase(image=child.volumePath,
                               backing=backing,
                               format=qemuimg.FORMAT.QCOW2,
                               backingFormat=backing_format,
                               unsafe=True)
    return operation
예제 #25
0
def test_copy_data_collapse(tmpdir, tmp_repo, fake_access, fake_rescan, tmp_db,
                            fake_task, fake_scheduler, monkeypatch,
                            dest_format):
    dom = tmp_repo.create_localfs_domain(name="domain", version=5)

    chain_size = 3
    volumes = create_chain(dom, chain_size)
    dest_img_id = str(uuid.uuid4())
    dest_vol_id = str(uuid.uuid4())

    length = MEGAB

    # Write some data to each layer
    for i, vol in enumerate(volumes):
        qemuio.write_pattern(vol.getVolumePath(),
                             sc.fmt2str(vol.getFormat()),
                             offset=(i * length))

    # The last volume in the chain is the leaf
    source_leaf_vol = volumes[-1]
    dest_vol = create_volume(dom, dest_img_id, dest_vol_id, dest_format)

    source = dict(endpoint_type='div',
                  sd_id=source_leaf_vol.sdUUID,
                  img_id=source_leaf_vol.imgUUID,
                  vol_id=source_leaf_vol.volUUID)
    dest = dict(endpoint_type='div',
                sd_id=source_leaf_vol.sdUUID,
                img_id=dest_img_id,
                vol_id=dest_vol_id)

    # Run copy_data from the source chain to dest_vol, essentially
    # executing qemu-img convert
    job = copy_data.Job(str(uuid.uuid4()), 0, source, dest)
    monkeypatch.setattr(guarded, 'context', fake_guarded_context())
    job.run()

    # verify the data written to the source chain is available on the
    # collapsed target volume
    for i in range(chain_size):
        qemuio.verify_pattern(dest_vol.getVolumePath(),
                              sc.fmt2str(dest_vol.getFormat()),
                              offset=(i * length))
예제 #26
0
파일: merge.py 프로젝트: nirs/vdsm
 def _run(self):
     self.log.info("Merging subchain %s", self.subchain)
     with guarded.context(self.subchain.locks):
         self.subchain.validate()
         with self.subchain.prepare(), self.subchain.volume_operation():
             self.operation = qemuimg.commit(
                 self.subchain.top_vol.getVolumePath(),
                 topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
                 base=self.subchain.base_vol.getVolumePath())
             self.operation.run()
예제 #27
0
 def _run(self):
     self.log.info("Merging subchain %s", self.subchain)
     with guarded.context(self.subchain.locks):
         self.subchain.validate()
         with self.subchain.prepare(), self.subchain.volume_operation():
             self.operation = qemuimg.commit(
                 self.subchain.top_vol.getVolumePath(),
                 topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
                 base=self.subchain.base_vol.getVolumePath())
             self.operation.run()
예제 #28
0
 def test_ok(self, vol_fmt):
     with self.fake_volume(vol_fmt) as vol:
         qemu_fmt = sc.fmt2str(vol_fmt)
         op = qemuimg.create(vol.volumePath,
                             size=self.SIZE,
                             format=qemu_fmt)
         op.run()
         h = FakeHSM()
         h.verify_untrusted_volume('sp', vol.sdUUID, vol.imgUUID,
                                   vol.volUUID)
예제 #29
0
파일: volume.py 프로젝트: bronhaim/vdsm
 def updateInvalidatedSize(self):
     # During some complex flows the volume size might have been marked as
     # invalidated (e.g. during a transaction). Here we are checking
     # NOTE: the prerequisite to run this is that the volume is accessible
     # (e.g. lv active) and not in use by another process (e.g. dd, qemu).
     # Going directly to the metadata parameter as we should skip the size
     # validation in getSize.
     if int(self.getMetaParam(sc.SIZE)) < 1:
         volInfo = qemuimg.info(self.getVolumePath(),
                                sc.fmt2str(self.getFormat()))
         # qemu/qemu-img rounds down
         self.setSize(volInfo['virtualsize'] / sc.BLOCK_SIZE)
예제 #30
0
    def _create(cls, dom, imgUUID, volUUID, capacity, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath, initial_size=None,
                add_bitmaps=False):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """

        lv_size = cls.calculate_volume_alloc_size(
            preallocate, volFormat, capacity, initial_size)
        lv_size_mb = utils.round(lv_size, MiB) // MiB

        lvm.createLV(dom.sdUUID, volUUID, lv_size_mb, activate=True,
                     initialTags=(sc.TAG_VOL_UNINIT,))

        fileutils.rm_file(volPath)
        lvPath = lvm.lvPath(dom.sdUUID, volUUID)
        cls.log.info("Creating volume symlink from %r to %r", lvPath, volPath)
        os.symlink(lvPath, volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with capacity = %s",
                         sc.type2name(volFormat), volPath, capacity)
            if volFormat == sc.COW_FORMAT:
                operation = qemuimg.create(volPath,
                                           size=capacity,
                                           format=sc.fmt2str(volFormat),
                                           qcow2Compat=dom.qcow2_compat())
                operation.run()
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
                         "with capacity %s",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID, capacity)
            volParent.clone(volPath, volFormat, capacity)

        with dom.acquireVolumeMetadataSlot(volUUID) as slot:
            mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot),
                      "%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID),
                      "%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)]
            lvm.changeLVsTags(
                dom.sdUUID,
                (volUUID,),
                delTags=[sc.TAG_VOL_UNINIT],
                addTags=mdTags)

        try:
            lvm.deactivateLVs(dom.sdUUID, [volUUID])
        except se.CannotDeactivateLogicalVolume:
            cls.log.warn("Cannot deactivate new created volume %s/%s",
                         dom.sdUUID, volUUID, exc_info=True)

        return (dom.sdUUID, slot)
예제 #31
0
 def updateInvalidatedSize(self):
     # During some complex flows the volume size might have been marked as
     # invalidated (e.g. during a transaction). Here we are checking
     # NOTE: the prerequisite to run this is that the volume is accessible
     # (e.g. lv active) and not in use by another process (e.g. dd, qemu).
     # Going directly to the metadata parameter as we should skip the size
     # validation in getSize.
     if int(self.getMetaParam(sc.SIZE)) < 1:
         volInfo = qemuimg.info(
             self.getVolumePath(), sc.fmt2str(self.getFormat()))
         # qemu/qemu-img rounds down
         self.setSize(volInfo['virtualsize'] / sc.BLOCK_SIZE)
예제 #32
0
파일: blockVolume.py 프로젝트: nirs/vdsm
    def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """

        lvSize = cls.calculate_volume_alloc_size(preallocate,
                                                 size, initialSize)

        lvm.createLV(dom.sdUUID, volUUID, lvSize, activate=True,
                     initialTags=(sc.TAG_VOL_UNINIT,))

        fileutils.rm_file(volPath)
        lvPath = lvm.lvPath(dom.sdUUID, volUUID)
        cls.log.info("Creating volume symlink from %r to %r", lvPath, volPath)
        os.symlink(lvPath, volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with size = %s "
                         "blocks", sc.type2name(volFormat), volPath,
                         size)
            if volFormat == sc.COW_FORMAT:
                operation = qemuimg.create(volPath,
                                           size=size * BLOCK_SIZE,
                                           format=sc.fmt2str(volFormat),
                                           qcow2Compat=dom.qcow2_compat())
                operation.run()
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
                         "with size %s (blocks)",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID, size)
            volParent.clone(volPath, volFormat, size)

        with dom.acquireVolumeMetadataSlot(volUUID) as slot:
            mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot),
                      "%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID),
                      "%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)]
            lvm.changeLVTags(dom.sdUUID, volUUID, delTags=[sc.TAG_VOL_UNINIT],
                             addTags=mdTags)

        try:
            lvm.deactivateLVs(dom.sdUUID, [volUUID])
        except se.CannotDeactivateLogicalVolume:
            cls.log.warn("Cannot deactivate new created volume %s/%s",
                         dom.sdUUID, volUUID, exc_info=True)

        return (dom.sdUUID, slot)
예제 #33
0
파일: volume.py 프로젝트: bronhaim/vdsm
    def rebase(self, backingVol, backingVolPath, backingFormat, unsafe,
               rollback):
        """
        Rebase volume on top of new backing volume
        """
        if rollback:
            pvol = self.getParentVolume()
            if not pvol:
                self.log.warn("Can't rebase volume %s, parent missing",
                              self.volUUID)
                return

            name = "Merge volume: " + self.volUUID
            vars.task.pushRecovery(
                task.Recovery(name, "volume", "Volume", "rebaseVolumeRollback",
                              [
                                  self.sdUUID,
                                  self.getImage(), self.volUUID,
                                  str(pvol.getFormat()), pvol.volUUID,
                                  str(True)
                              ]))

        volumePath = self.getVolumePath()

        try:
            qemuimg.rebase(volumePath, backingVolPath,
                           sc.fmt2str(self.getFormat()),
                           sc.fmt2str(backingFormat), unsafe,
                           vars.task.aborting)
        except qemuimg.QImgError:
            self.log.exception('cannot rebase volume %s on %s', volumePath,
                               backingVolPath)
            raise se.MergeSnapshotsError(self.volUUID)

        self.setParent(backingVol)
        self.recheckIfLeaf()
예제 #34
0
def write_qemu_chain(vol_list):
    # Starting with the base volume in vol_list, write to the chain in a
    # pattern like the following:
    #
    #  logical offset: 0K            1K            2K            3K
    #   Base Volume 0: 0xf0 0xf0 ...
    #               1:               0xf1 0xf1 ...
    #               2:                             0xf2 0xf2 ...
    #   Leaf Volume 3:                                           0xf3 0xf3 ...
    # This allows us to verify the integrity of the whole chain.
    for i, vol in enumerate(vol_list):
        vol_fmt = sc.fmt2str(vol.getFormat())
        offset = i * 1024
        pattern = 0xf0 + i
        qemu_pattern_write(vol.volumePath, vol_fmt, offset=offset,
                           len=1024, pattern=pattern)
예제 #35
0
def write_qemu_chain(vol_list):
    # Starting with the base volume in vol_list, write to the chain in a
    # pattern like the following:
    #
    #  logical offset: 0K            1K            2K            3K
    #   Base Volume 0: 0xf0 0xf0 ...
    #               1:               0xf1 0xf1 ...
    #               2:                             0xf2 0xf2 ...
    #   Leaf Volume 3:                                           0xf3 0xf3 ...
    # This allows us to verify the integrity of the whole chain.
    for i, vol in enumerate(vol_list):
        vol_fmt = sc.fmt2str(vol.getFormat())
        offset = i * 1024
        pattern = 0xf0 + i
        qemu_pattern_write(vol.volumePath, vol_fmt, offset=offset,
                           len=1024, pattern=pattern)
예제 #36
0
    def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath, initialSize=None):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """

        lvSize = cls.calculate_volume_alloc_size(preallocate,
                                                 size, initialSize)

        lvm.createLV(dom.sdUUID, volUUID, "%s" % lvSize, activate=True,
                     initialTags=(sc.TAG_VOL_UNINIT,))

        fileutils.rm_file(volPath)
        os.symlink(lvm.lvPath(dom.sdUUID, volUUID), volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with size = %s "
                         "sectors", sc.type2name(volFormat), volPath,
                         size)
            if volFormat == sc.COW_FORMAT:
                qemuimg.create(volPath,
                               size=size * BLOCK_SIZE,
                               format=sc.fmt2str(volFormat),
                               qcow2Compat=dom.qcow2_compat())
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID)
            volParent.clone(volPath, volFormat)

        with dom.acquireVolumeMetadataSlot(
                volUUID, sc.VOLUME_MDNUMBLKS) as slot:
            mdTags = ["%s%s" % (sc.TAG_PREFIX_MD, slot),
                      "%s%s" % (sc.TAG_PREFIX_PARENT, srcVolUUID),
                      "%s%s" % (sc.TAG_PREFIX_IMAGE, imgUUID)]
            lvm.changeLVTags(dom.sdUUID, volUUID, delTags=[sc.TAG_VOL_UNINIT],
                             addTags=mdTags)

        try:
            lvm.deactivateLVs(dom.sdUUID, [volUUID])
        except se.CannotDeactivateLogicalVolume:
            cls.log.warn("Cannot deactivate new created volume %s/%s",
                         dom.sdUUID, volUUID, exc_info=True)

        return (dom.sdUUID, slot)
예제 #37
0
def test_readonly(nbd_env, format, allocation):
    # Volume served by qemu-nd.
    img_id = str(uuid.uuid4())
    vol_id = str(uuid.uuid4())
    nbd_env.make_volume(nbd_env.virtual_size,
                        img_id,
                        vol_id,
                        vol_format=format,
                        prealloc=allocation)

    # Fill volume with data before starting the server.
    vol = nbd_env.sd_manifest.produceVolume(img_id, vol_id)
    op = qemuimg.convert(nbd_env.src,
                         vol.getVolumePath(),
                         dstFormat=sc.fmt2str(format),
                         preallocation=PREALLOCATION.get(format))
    op.run()

    # Server configuration.
    config = {
        "sd_id": nbd_env.sd_manifest.sdUUID,
        "img_id": img_id,
        "vol_id": vol_id,
        "readonly": True,
    }

    with nbd_server(config) as nbd_url:
        # Writing to NBD server must fail.
        with pytest.raises(cmdutils.Error):
            op = qemuimg.convert(nbd_env.src,
                                 nbd_url,
                                 srcFormat="raw",
                                 create=False)
            op.run()

        # Copy data from NBD server to dst. Both files should match byte
        # for byte after the operation.
        op = qemuimg.convert(nbd_url, nbd_env.dst, dstFormat="raw")
        op.run()

    with io.open(nbd_env.src) as s, io.open(nbd_env.dst) as d:
        assert s.read() == d.read()

    # Now the server should not be accessible.
    with pytest.raises(cmdutils.Error):
        qemuimg.info(nbd_url)
예제 #38
0
파일: nbd_test.py 프로젝트: nirs/vdsm
def test_readonly(nbd_env, format, allocation):
    # Volume served by qemu-nd.
    img_id = str(uuid.uuid4())
    vol_id = str(uuid.uuid4())
    nbd_env.make_volume(
        nbd_env.virtual_size,
        img_id,
        vol_id,
        vol_format=format,
        prealloc=allocation)

    # Fill volume with data before starting the server.
    vol = nbd_env.sd_manifest.produceVolume(img_id, vol_id)
    op = qemuimg.convert(
        nbd_env.src,
        vol.getVolumePath(),
        dstFormat=sc.fmt2str(format),
        preallocation=PREALLOCATION.get(format))
    op.run()

    # Server configuration.
    config = {
        "sd_id": nbd_env.sd_manifest.sdUUID,
        "img_id": img_id,
        "vol_id": vol_id,
        "readonly": True,
    }

    with nbd_server(config) as nbd_url:
        # Writing to NBD server must fail.
        with pytest.raises(cmdutils.Error):
            op = qemuimg.convert(
                nbd_env.src, nbd_url, srcFormat="raw", create=False)
            op.run()

        # Copy data from NBD server to dst. Both files should match byte
        # for byte after the operation.
        op = qemuimg.convert(nbd_url, nbd_env.dst, dstFormat="raw")
        op.run()

    with io.open(nbd_env.src) as s, io.open(nbd_env.dst) as d:
        assert s.read() == d.read()

    # Now the server should not be accessible.
    with pytest.raises(cmdutils.Error):
        qemuimg.info(nbd_url)
예제 #39
0
    def test_merge_subchain(self, sd_type, chain_len, base_index, top_index):
        job_id = make_uuid()
        with self.make_env(sd_type=sd_type, chain_len=chain_len) as env:
            write_qemu_chain(env.chain)
            base_vol = env.chain[base_index]
            top_vol = env.chain[top_index]

            subchain_info = dict(sd_id=base_vol.sdUUID,
                                 img_id=base_vol.imgUUID,
                                 base_id=base_vol.volUUID,
                                 top_id=top_vol.volUUID,
                                 base_generation=0)
            subchain = merge.SubchainInfo(subchain_info, 0)
            job = api_merge.Job(job_id, subchain)
            job.run()
            wait_for_job(job)
            self.assertEqual(job.status, jobs.STATUS.DONE)

            # Verify that the chain data was merged
            for i in range(base_index, top_index + 1):
                offset = i * 1024
                pattern = 0xf0 + i

                # We expect to read all data from top
                verify_pattern(
                    top_vol.volumePath,
                    qemuimg.FORMAT.QCOW2,
                    offset=offset,
                    len=1024,
                    pattern=pattern)

                # And base, since top was merged into base
                verify_pattern(
                    base_vol.volumePath,
                    sc.fmt2str(base_vol.getFormat()),
                    offset=offset,
                    len=1024,
                    pattern=pattern)

            self.assertEqual(sorted(self.expected_locks(base_vol)),
                             sorted(guarded.context.locks))

            self.assertEqual(base_vol.getLegality(), sc.LEGAL_VOL)
            self.assertEqual(base_vol.getMetaParam(sc.GENERATION), 1)
예제 #40
0
    def _run(self):
        self.log.info("Merging subchain %s", self.subchain)
        with guarded.context(self.subchain.locks):
            self.subchain.validate()
            # Base volume must be ILLEGAL. Otherwise, VM could be run while
            # performing cold merge.
            base_legality = self.subchain.base_vol.getLegality()
            if base_legality == sc.LEGAL_VOL:
                raise se.UnexpectedVolumeState(self.subchain.base_id,
                                               sc.ILLEGAL_VOL,
                                               base_legality)

            with self.subchain.prepare(), self.subchain.volume_operation():
                self.operation = qemuimg.commit(
                    self.subchain.top_vol.getVolumePath(),
                    topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
                    base=self.subchain.base_vol.getVolumePath())
                with utils.closing(self.operation):
                    self.operation.wait_for_completion()
예제 #41
0
    def reconcileVolumeChain(self, sdUUID, imgUUID, leafVolUUID):
        """
        Discover and return the actual volume chain of an offline image
        according to the qemu-img info command and synchronize volume metadata.
        """
        # Prepare volumes
        dom = sdCache.produce(sdUUID)
        allVols = dom.getAllVolumes()
        imgVolumes = sd.getVolsOfImage(allVols, imgUUID).keys()
        dom.activateVolumes(imgUUID, imgVolumes)

        # Walk the volume chain using qemu-img.  Not safe for running VMs
        actualVolumes = []
        volUUID = leafVolUUID
        while volUUID is not None:
            actualVolumes.insert(0, volUUID)
            vol = dom.produceVolume(imgUUID, volUUID)
            qemuImgFormat = sc.fmt2str(vol.getFormat())
            imgInfo = qemuimg.info(vol.volumePath, qemuImgFormat)
            backingFile = imgInfo.get('backingfile')
            if backingFile is not None:
                volUUID = os.path.basename(backingFile)
            else:
                volUUID = None

        # A merge of the active layer has copy and pivot phases.
        # During copy, data is copied from the leaf into its parent.  Writes
        # are mirrored to both volumes.  So even after copying is complete the
        # volumes will remain consistent.  Finally, the VM is pivoted from the
        # old leaf to the new leaf and mirroring to the old leaf ceases. During
        # mirroring and before pivoting, we mark the old leaf ILLEGAL so we
        # know it's safe to delete in case the operation is interrupted.
        vol = dom.produceVolume(imgUUID, leafVolUUID)
        if vol.getLegality() == sc.ILLEGAL_VOL:
            actualVolumes.remove(leafVolUUID)

        # Now that we know the correct volume chain, sync the storge metadata
        self.syncVolumeChain(sdUUID, imgUUID, actualVolumes[-1], actualVolumes)

        dom.deactivateImage(imgUUID)
        return actualVolumes
예제 #42
0
파일: fileVolume.py 프로젝트: oVirt/vdsm
    def _create_cow_volume(
            cls, dom, vol_id, size, vol_path, initial_size, vol_parent,
            img_id, src_img_id, src_vol_id):
        """
        specific implementation of _create() for COW volumes.
        All the exceptions are properly handled and logged in volume.create()
        """
        if initial_size:
            cls.log.error("initial size is not supported "
                          "for file-based volumes")
            raise se.InvalidParameterException("initial size", initial_size)

        cls._truncate_volume(vol_path, 0, vol_id, dom)

        if not vol_parent:
            cls.log.info("Request to create COW volume %s with size = %s "
                         "bytes", vol_path, size)

            operation = qemuimg.create(vol_path,
                                       size=size,
                                       format=sc.fmt2str(sc.COW_FORMAT),
                                       qcow2Compat=dom.qcow2_compat())
            operation.run()
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s "
                         "with size %s (bytes)",
                         img_id, vol_id, src_img_id, src_vol_id, size)
            size_blk = size // BLOCK_SIZE
            vol_parent.clone(vol_path, sc.COW_FORMAT, size_blk)

        # Forcing the volume permissions in case one of the tools we use
        # (dd, qemu-img, etc.) will mistakenly change the file permissions.
        cls._set_permissions(vol_path, dom)

        return (vol_path,)
예제 #43
0
 def qemu_format(self):
     return sc.fmt2str(self.volume.getFormat())
예제 #44
0
 def qemu_format(self):
     # TODO: Use Image._detect_format to handle broken VM md images.
     return sc.fmt2str(self._vol.getFormat())
예제 #45
0
    def _interImagesCopy(self, destDom, srcSdUUID, imgUUID, chains):
        srcLeafVol = chains['srcChain'][-1]
        dstLeafVol = chains['dstChain'][-1]
        try:
            # Prepare the whole chains before the copy
            srcLeafVol.prepare(rw=False)
            dstLeafVol.prepare(rw=True, chainrw=True, setrw=True)
        except Exception:
            self.log.error("Unexpected error", exc_info=True)
            # teardown volumes
            self.__cleanupMove(srcLeafVol, dstLeafVol)
            raise

        try:
            for srcVol in chains['srcChain']:
                # Do the actual copy
                try:
                    dstVol = destDom.produceVolume(imgUUID=imgUUID,
                                                   volUUID=srcVol.volUUID)

                    if workarounds.invalid_vm_conf_disk(srcVol):
                        srcFormat = dstFormat = qemuimg.FORMAT.RAW
                    else:
                        srcFormat = sc.fmt2str(srcVol.getFormat())
                        dstFormat = sc.fmt2str(dstVol.getFormat())

                    parentVol = dstVol.getParentVolume()

                    if parentVol is not None:
                        backing = volume.getBackingVolumePath(
                            imgUUID, parentVol.volUUID)
                        backingFormat = sc.fmt2str(parentVol.getFormat())
                    else:
                        backing = None
                        backingFormat = None

                    if (destDom.supportsSparseness
                            and dstVol.getType() == sc.PREALLOCATED_VOL):
                        preallocation = qemuimg.PREALLOCATION.FALLOC
                    else:
                        preallocation = None

                    operation = qemuimg.convert(
                        srcVol.getVolumePath(),
                        dstVol.getVolumePath(),
                        srcFormat=srcFormat,
                        dstFormat=dstFormat,
                        dstQcow2Compat=destDom.qcow2_compat(),
                        backing=backing,
                        backingFormat=backingFormat,
                        preallocation=preallocation,
                        unordered_writes=destDom.recommends_unordered_writes(
                            dstVol.getFormat()),
                        create=not destDom.is_block(),
                    )
                    with utils.stopwatch("Copy volume %s" % srcVol.volUUID):
                        self._run_qemuimg_operation(operation)
                except ActionStopped:
                    raise
                except se.StorageException:
                    self.log.error("Unexpected error", exc_info=True)
                    raise
                except Exception:
                    self.log.error(
                        "Copy image error: image=%s, src domain=%s,"
                        " dst domain=%s",
                        imgUUID,
                        srcSdUUID,
                        destDom.sdUUID,
                        exc_info=True)
                    raise se.CopyImageError()
        finally:
            # teardown volumes
            self.__cleanupMove(srcLeafVol, dstLeafVol)
예제 #46
0
파일: volume.py 프로젝트: bronhaim/vdsm
 def getQemuImageInfo(self):
     """
     Returns volume information as returned by qemu-img info command
     """
     return qemuimg.info(self.getVolumePath(), sc.fmt2str(self.getFormat()))
예제 #47
0
 def _initialize_volume(self, vol_format, size):
     if vol_format == sc.COW_FORMAT:
         qemuimg.create(self.volume_path, size, sc.fmt2str(vol_format))
예제 #48
0
    def copyCollapsed(self, sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID,
                      dstVolUUID, descr, dstSdUUID, volType, volFormat,
                      preallocate, postZero, force, discard):
        """
        Create new template/volume from VM.
        Do it by collapse and copy the whole chain (baseVolUUID->srcVolUUID)
        """
        self.log.info(
            "sdUUID=%s vmUUID=%s srcImgUUID=%s srcVolUUID=%s "
            "dstImgUUID=%s dstVolUUID=%s dstSdUUID=%s volType=%s "
            "volFormat=%s preallocate=%s force=%s postZero=%s "
            "discard=%s", sdUUID, vmUUID, srcImgUUID, srcVolUUID, dstImgUUID,
            dstVolUUID, dstSdUUID, volType, sc.type2name(volFormat),
            sc.type2name(preallocate), str(force), str(postZero), discard)
        try:
            srcVol = dstVol = None

            # Find out dest sdUUID
            if dstSdUUID == sd.BLANK_UUID:
                dstSdUUID = sdUUID
            volclass = sdCache.produce(sdUUID).getVolumeClass()
            destDom = sdCache.produce(dstSdUUID)

            # find src volume
            try:
                srcVol = volclass(self.repoPath, sdUUID, srcImgUUID,
                                  srcVolUUID)
            except se.StorageException:
                raise
            except Exception as e:
                self.log.error(e, exc_info=True)
                raise se.SourceImageActionError(srcImgUUID, sdUUID, str(e))

            # Create dst volume
            try:
                # Before reading source volume parameters from volume metadata,
                # prepare the volume. This ensure that the volume capacity will
                # match the actual virtual size, see
                # https://bugzilla.redhat.com/1700623.
                srcVol.prepare(rw=False)

                volParams = srcVol.getVolumeParams()

                if volFormat in [sc.COW_FORMAT, sc.RAW_FORMAT]:
                    dstVolFormat = volFormat
                else:
                    dstVolFormat = volParams['volFormat']

                # TODO: This is needed only when copying to qcow2-thin volume
                # on block storage. Move into calculate_initial_size.
                dst_vol_allocation = self.calculate_vol_alloc(
                    sdUUID, volParams, dstSdUUID, dstVolFormat)

                # Find out dest volume parameters
                if preallocate in [sc.PREALLOCATED_VOL, sc.SPARSE_VOL]:
                    volParams['prealloc'] = preallocate

                initial_size = self.calculate_initial_size(
                    destDom.supportsSparseness, dstVolFormat,
                    volParams['prealloc'], dst_vol_allocation)

                self.log.info(
                    "Copy source %s:%s:%s to destination %s:%s:%s "
                    "capacity=%s, initial size=%s", sdUUID, srcImgUUID,
                    srcVolUUID, dstSdUUID, dstImgUUID, dstVolUUID,
                    volParams['capacity'], initial_size)

                # If image already exists check whether it illegal/fake,
                # overwrite it
                if not self.isLegal(dstSdUUID, dstImgUUID):
                    force = True

                # We must first remove the previous instance of image (if
                # exists) in destination domain, if we got the overwrite
                # command
                if force:
                    self.log.info(
                        "delete image %s on domain %s before "
                        "overwriting", dstImgUUID, dstSdUUID)
                    _deleteImage(destDom, dstImgUUID, postZero, discard)

                destDom.createVolume(imgUUID=dstImgUUID,
                                     capacity=volParams['capacity'],
                                     volFormat=dstVolFormat,
                                     preallocate=volParams['prealloc'],
                                     diskType=volParams['disktype'],
                                     volUUID=dstVolUUID,
                                     desc=descr,
                                     srcImgUUID=sc.BLANK_UUID,
                                     srcVolUUID=sc.BLANK_UUID,
                                     initial_size=initial_size)

                dstVol = sdCache.produce(dstSdUUID).produceVolume(
                    imgUUID=dstImgUUID, volUUID=dstVolUUID)

            except se.StorageException:
                self.log.error("Unexpected error", exc_info=True)
                raise
            except Exception as e:
                self.log.error("Unexpected error", exc_info=True)
                raise se.CopyImageError("Destination volume %s error: %s" %
                                        (dstVolUUID, str(e)))

            try:
                # Start the actual copy image procedure
                dstVol.prepare(rw=True, setrw=True)

                if (destDom.supportsSparseness
                        and dstVol.getType() == sc.PREALLOCATED_VOL):
                    preallocation = qemuimg.PREALLOCATION.FALLOC
                else:
                    preallocation = None

                try:
                    operation = qemuimg.convert(
                        volParams['path'],
                        dstVol.getVolumePath(),
                        srcFormat=sc.fmt2str(volParams['volFormat']),
                        dstFormat=sc.fmt2str(dstVolFormat),
                        dstQcow2Compat=destDom.qcow2_compat(),
                        preallocation=preallocation,
                        unordered_writes=destDom.recommends_unordered_writes(
                            dstVolFormat),
                        create=not destDom.is_block(),
                    )
                    with utils.stopwatch("Copy volume %s" % srcVol.volUUID):
                        self._run_qemuimg_operation(operation)
                except ActionStopped:
                    raise
                except cmdutils.Error as e:
                    self.log.exception('conversion failure for volume %s',
                                       srcVol.volUUID)
                    raise se.CopyImageError(str(e))

                # Mark volume as SHARED
                if volType == sc.SHARED_VOL:
                    dstVol.setShared()

                dstVol.setLegality(sc.LEGAL_VOL)

                if force:
                    # Now we should re-link all deleted hardlinks, if exists
                    destDom.templateRelink(dstImgUUID, dstVolUUID)
            except se.StorageException:
                self.log.error("Unexpected error", exc_info=True)
                raise
            except Exception as e:
                self.log.error("Unexpected error", exc_info=True)
                raise se.CopyImageError("src image=%s, dst image=%s: msg=%s" %
                                        (srcImgUUID, dstImgUUID, str(e)))

            self.log.info("Finished copying %s:%s -> %s:%s", sdUUID,
                          srcVolUUID, dstSdUUID, dstVolUUID)
            # TODO: handle return status
            return dstVolUUID
        finally:
            self.__cleanupCopy(srcVol=srcVol, dstVol=dstVol)
예제 #49
0
 def backing_qemu_format(self):
     parent_vol = self._vol.getParentVolume()
     if not parent_vol:
         return None
     return sc.fmt2str(parent_vol.getFormat())
예제 #50
0
 def getQemuImageInfo(self):
     """
     Returns volume information as returned by qemu-img info command
     """
     return qemuimg.info(self.getVolumePath(),
                         sc.fmt2str(self.getFormat()))