Example #1
0
    def _allocate_volume(cls, vol_path, size, preallocate):
        try:
            # Always create sparse image, since qemu-img create uses
            # posix_fallocate() which is inefficient and harmful.
            op = qemuimg.create(vol_path, size=size, format=qemuimg.FORMAT.RAW)

            # This is fast but it can get stuck if storage is inaccessible.
            with vars.task.abort_callback(op.abort):
                with utils.stopwatch("Creating image {}".format(vol_path),
                                     level=logging.INFO,
                                     log=cls.log):
                    op.run()

            # If the image is preallocated, allocate the rest of the image
            # using fallocate helper. qemu-img create always writes zeroes to
            # the first block so we should skip it during preallocation.
            if preallocate == sc.PREALLOCATED_VOL:
                op = fallocate.allocate(vol_path, size - 4096, offset=4096)

                # This is fast on NFS 4.2, GlusterFS, XFS and ext4, but can be
                # extremely slow on NFS < 4.2, writing zeroes to entire image.
                with vars.task.abort_callback(op.abort):
                    with utils.stopwatch(
                            "Preallocating volume {}".format(vol_path),
                            level=logging.INFO,
                            log=cls.log):
                        op.run()
        except exception.ActionStopped:
            raise
        except Exception:
            cls.log.error("Unexpected error", exc_info=True)
            raise se.VolumesZeroingError(vol_path)
Example #2
0
def resize_map(name):
    """
    Invoke multipathd to resize a device
    Must run as root

    Raises Error if multipathd failed to resize the map.
    """
    if os.geteuid() != 0:
        return supervdsm.getProxy().multipath_resize_map(name)

    log.debug("Resizing map %r", name)
    cmd = [_MULTIPATHD.cmd, "resize", "map", name]
    with utils.stopwatch("Resized map %r" % name, log=log):
        p = commands.start(cmd,
                           sudo=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
        out, err = commands.communicate(p)

        out = out.decode("utf-8")
        err = err.decode("utf-8")

        # multipathd reports some errors using non-zero exit code and
        # stderr (need to be root), but the command may return 0, and
        # the result is reported using stdout.
        if p.returncode != 0 or out != "ok\n":
            e = cmdutils.Error(cmd, p.returncode, out, err)
            raise Error("Resizing map {!r} failed: {}".format(name, e))
Example #3
0
File: sdc.py Project: vjuranek/vdsm
    def _findUnfetchedDomain(self, sdUUID):
        from vdsm.storage import blockSD
        from vdsm.storage import glusterSD
        from vdsm.storage import localFsSD
        from vdsm.storage import nfsSD

        # The order is somewhat important, it's ordered
        # by how quickly get can find the domain. For instance
        # if an nfs mount is unavailable we will get stuck
        # until it times out, this should affect fetching
        # of block\local domains. If for any case in the future
        # this changes, please update the order.

        self.log.info("Looking up domain %s", sdUUID)
        with utils.stopwatch(
                "Looking up domain {}".format(sdUUID),
                level=logging.INFO,
                log=self.log):
            for mod in (blockSD, glusterSD, localFsSD, nfsSD):
                try:
                    return mod.findDomain(sdUUID)
                except se.StorageDomainDoesNotExist:
                    pass
                except Exception:
                    self.log.error(
                        "Error while looking for domain `%s`",
                        sdUUID, exc_info=True)

        raise se.StorageDomainDoesNotExist(sdUUID)
Example #4
0
def zero(device_path, size=None, task=_NullTask()):
    """
    Zero a block device.

    Arguments:
        device_path (str): Path to block device to wipe
        size (int): Number of bytes to write. If not specified, use the device
            size. Size must be aligned to `vdsm.storage.constants.BLOCK_SIZE`.
        task (`storage.task.Task`): Task running this operation. If specified,
            the zero operation will be aborted if the task is aborted.

    Raises:
        `vdsm.common.exception.ActionStopped` if the wipe was aborted
        `vdsm.storage.exception.VolumesZeroingError` if writing to storage
            failed.
        `vdsm.storage.exception.InvalidParameterException` if size is not
            aligned to `vdsm.storage.constants.BLOCK_SIZE`.
    """
    if size is None:
        # Always aligned to LVM extent size (128MiB).
        size = fsutils.size(device_path)
    elif size % sc.BLOCK_SIZE:
        raise se.InvalidParameterException("size", size)

    log.info("Zeroing device %s (size=%d)", device_path, size)
    with utils.stopwatch("Zero device %s" % device_path,
                         level=logging.INFO, log=log):
        try:
            op = blkdiscard.zeroout_operation(device_path, size)
            with task.abort_callback(op.abort):
                op.run()
        except se.StorageException as e:
            raise se.VolumesZeroingError("Zeroing device %s failed: %s"
                                         % (device_path, e))
Example #5
0
 def umount(self, force=False, lazy=False, freeloop=False, timeout=None):
     umount = supervdsm.getProxy().umount if os.geteuid() != 0 else _umount
     self.log.info("unmounting %s", self.fs_file)
     with utils.stopwatch("%s unmounted" % self.fs_file, log=self.log):
         umount(self.fs_file, force=force, lazy=lazy, freeloop=freeloop,
                timeout=timeout)
     self._wait_for_events()
Example #6
0
def copyToImage(dstImgPath, methodArgs):
    totalSize = getLengthFromArgs(methodArgs)
    fileObj = methodArgs['fileObj']

    # Unlike copyFromImage, we don't use direct I/O when writing because:
    # - Images are small so using host page cache is ok.
    # - Images typically aligned to 512 bytes (tar), may fail on 4k storage.
    cmd = [
        constants.EXT_DD,
        "of=%s" % dstImgPath,
        "bs=%s" % MiB,
        # Ensure that data reach physical storage before returning.
        "conv=fsync",
    ]

    log.info("Copy to image %s", dstImgPath)
    with utils.stopwatch("Copy %s bytes" % totalSize,
                         level=logging.INFO,
                         log=log):
        p = commands.start(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
        with commands.terminating(p):
            _copyData(fileObj, p.stdin, totalSize)
            try:
                _, err = p.communicate(timeout=WAIT_TIMEOUT)
            except subprocess.TimeoutExpired:
                log.error("timeout waiting for dd process")
                raise se.StorageException()

            if p.returncode != 0:
                log.error("dd failed rc=%s err=%r", p.returncode, err)
                raise se.MiscFileWriteException()
Example #7
0
    def _extendSizeRaw(self, newSize):
        volPath = self.getVolumePath()
        curSizeBytes = self.oop.os.stat(volPath).st_size
        newSizeBytes = newSize * BLOCK_SIZE

        # No real sanity checks here, they should be included in the calling
        # function/method. We just validate the sizes to be consistent since
        # they're computed and used in the pre-allocated case.
        if newSizeBytes == curSizeBytes:
            return  # Nothing to do
        elif curSizeBytes <= 0:
            raise se.StorageException("Volume size is impossible: %s" %
                                      curSizeBytes)
        elif newSizeBytes < curSizeBytes:
            raise se.VolumeResizeValueError(newSize)

        if self.getType() == sc.PREALLOCATED_VOL:
            self.log.info("Preallocating volume %s to %s bytes", volPath,
                          newSizeBytes)
            operation = fallocate.allocate(volPath,
                                           newSizeBytes - curSizeBytes,
                                           curSizeBytes)
            with vars.task.abort_callback(operation.abort):
                with utils.stopwatch("Preallocating volume %s" % volPath):
                    operation.run()
        else:
            # for sparse files we can just truncate to the correct size
            # also good fallback for failed preallocation
            self.log.info("Truncating volume %s to %s bytes", volPath,
                          newSizeBytes)
            self.oop.truncateFile(volPath, newSizeBytes)
Example #8
0
 def mount(self, mntOpts=None, vfstype=None, cgroup=None):
     mount = supervdsm.getProxy().mount if os.geteuid() != 0 else _mount
     self.log.info("mounting %s at %s", self.fs_spec, self.fs_file)
     with utils.stopwatch("%s mounted" % self.fs_file, log=self.log):
         mount(self.fs_spec, self.fs_file, mntOpts=mntOpts, vfstype=vfstype,
               cgroup=cgroup)
     self._wait_for_events()
Example #9
0
def zero(device_path, size=None, task=_NullTask()):
    """
    Zero a block device.

    Arguments:
        device_path (str): Path to block device to wipe
        size (int): Number of bytes to write. If not specified, use the device
            size. Size must be aligned to `vdsm.storage.constants.BLOCK_SIZE`.
        task (`storage.task.Task`): Task running this operation. If specified,
            the zero operation will be aborted if the task is aborted.

    Raises:
        `vdsm.common.exception.ActionStopped` if the wipe was aborted
        `vdsm.storage.exception.VolumesZeroingError` if writing to storage
            failed.
        `vdsm.storage.exception.InvalidParameterException` if size is not
            aligned to `vdsm.storage.constants.BLOCK_SIZE`.
    """
    if size is None:
        # Always aligned to LVM extent size (128MiB).
        size = fsutils.size(device_path)
    elif size % sc.BLOCK_SIZE:
        raise se.InvalidParameterException("size", size)

    log.info("Zeroing device %s (size=%d)", device_path, size)
    with utils.stopwatch("Zero device %s" % device_path,
                         level=logging.INFO,
                         log=log):
        try:
            op = blkdiscard.zeroout_operation(device_path, size)
            with task.abort_callback(op.abort):
                op.run()
        except se.StorageException as e:
            raise se.VolumesZeroingError("Zeroing device %s failed: %s" %
                                         (device_path, e))
Example #10
0
    def _run(self):
        with guarded.context(self._source.locks + self._dest.locks):
            with self._source.prepare(), self._dest.prepare():
                # Do not start copying if we have already been aborted
                if self._status == jobs.STATUS.ABORTED:
                    return

                # Workaround for volumes containing VM configuration info that
                # were created with invalid vdsm metadata.
                if self._source.is_invalid_vm_conf_disk():
                    src_format = dst_format = qemuimg.FORMAT.RAW
                else:
                    src_format = self._source.qemu_format
                    dst_format = self._dest.qemu_format

                with self._dest.volume_operation():
                    self._operation = qemuimg.convert(
                        self._source.path,
                        self._dest.path,
                        srcFormat=src_format,
                        dstFormat=dst_format,
                        dstQcow2Compat=self._dest.qcow2_compat,
                        backing=self._dest.backing_path,
                        backingFormat=self._dest.backing_qemu_format,
                        unordered_writes=self._dest.
                        recommends_unordered_writes,
                        create=self._dest.requires_create)
                    with utils.stopwatch("Copy volume {}".format(
                            self._source.path),
                                         level=logging.INFO,
                                         log=self.log):
                        self._operation.run()
Example #11
0
    def _extendSizeRaw(self, newSize):
        volPath = self.getVolumePath()
        curSizeBytes = self.oop.os.stat(volPath).st_size
        newSizeBytes = newSize * BLOCK_SIZE

        # No real sanity checks here, they should be included in the calling
        # function/method. We just validate the sizes to be consistent since
        # they're computed and used in the pre-allocated case.
        if newSizeBytes == curSizeBytes:
            return  # Nothing to do
        elif curSizeBytes <= 0:
            raise se.StorageException(
                "Volume size is impossible: %s" % curSizeBytes)
        elif newSizeBytes < curSizeBytes:
            raise se.VolumeResizeValueError(newSize)

        if self.getType() == sc.PREALLOCATED_VOL:
            self.log.info("Preallocating volume %s to %s bytes",
                          volPath, newSizeBytes)
            operation = fallocate.allocate(volPath,
                                           newSizeBytes - curSizeBytes,
                                           curSizeBytes)
            with vars.task.abort_callback(operation.abort):
                with utils.stopwatch("Preallocating volume %s" % volPath):
                    operation.run()
        else:
            # for sparse files we can just truncate to the correct size
            # also good fallback for failed preallocation
            self.log.info("Truncating volume %s to %s bytes",
                          volPath, newSizeBytes)
            self.oop.truncateFile(volPath, newSizeBytes)
Example #12
0
 def test_mkIsoFs(self, label):
     """
     Tests mkimage.mkIsoFs creating an image and checking its content
     """
     checkSudo(["mount", "-o", "loop", "somefile", "target"])
     checkSudo(["umount", "target"])
     iso_img = mkimage.mkIsoFs("vmId_iso", self.files, label)
     self.assertTrue(os.path.exists(iso_img))
     m = storage.mount.Mount(iso_img, self.workdir)
     m.mount(mntOpts='loop')
     try:
         self._check_content()
         self._check_label(iso_img, label)
         self._check_permissions(iso_img,
                                 ((stat.S_IRUSR, True),
                                  (stat.S_IWUSR, True),
                                  (stat.S_IXUSR, False)))
         self._check_permissions(iso_img,
                                 ((stat.S_IRGRP, True),
                                  (stat.S_IWGRP, False),
                                  (stat.S_IXGRP, False)))
         self._check_permissions(iso_img,
                                 ((stat.S_IROTH, False),
                                  (stat.S_IWOTH, False),
                                  (stat.S_IXOTH, False)))
     finally:
         m.umount(force=True, freeloop=True)
         # TODO: Use libudev to wait for specific event
         with stopwatch("Wait for udev events"):
             udevadm.settle(5)
Example #13
0
 def mount(self, mntOpts=None, vfstype=None, timeout=None, cgroup=None):
     mount = supervdsm.getProxy().mount if os.geteuid() != 0 else _mount
     self.log.info("mounting %s at %s", self.fs_spec, self.fs_file)
     with utils.stopwatch("%s mounted" % self.fs_file, log=self.log):
         mount(self.fs_spec, self.fs_file, mntOpts=mntOpts, vfstype=vfstype,
               timeout=timeout, cgroup=cgroup)
     self._wait_for_events()
Example #14
0
 def umount(self, force=False, lazy=False, freeloop=False, timeout=None):
     umount = supervdsm.getProxy().umount if os.geteuid() != 0 else _umount
     self.log.info("unmounting %s", self.fs_file)
     with utils.stopwatch("%s unmounted" % self.fs_file, log=self.log):
         umount(self.fs_file, force=force, lazy=lazy, freeloop=freeloop,
                timeout=timeout)
     self._wait_for_events()
Example #15
0
    def _extendSizeRaw(self, new_capacity):
        volPath = self.getVolumePath()
        cur_capacity = self.oop.os.stat(volPath).st_size

        # No real sanity checks here, they should be included in the calling
        # function/method. We just validate the sizes to be consistent since
        # they're computed and used in the pre-allocated case.
        if new_capacity == cur_capacity:
            return  # Nothing to do
        elif cur_capacity <= 0:
            raise se.StorageException("Volume capacity is impossible: %s" %
                                      cur_capacity)
        elif new_capacity < cur_capacity:
            raise se.VolumeResizeValueError(new_capacity)

        if self.getType() == sc.PREALLOCATED_VOL:
            self.log.info("Preallocating volume %s to %s", volPath,
                          new_capacity)
            op = fallocate.allocate(volPath,
                                    new_capacity - cur_capacity,
                                    offset=cur_capacity)
            with vars.task.abort_callback(op.abort):
                with utils.stopwatch("Preallocating volume {}".format(volPath),
                                     level=logging.INFO,
                                     log=self.log):
                    op.run()
        else:
            # for sparse files we can just truncate to the correct size
            # also good fallback for failed preallocation
            self.log.info("Truncating volume %s to %s", volPath, new_capacity)
            self.oop.truncateFile(volPath, new_capacity)
Example #16
0
    def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath,
                initialSize=None):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """
        if initialSize:
            cls.log.error("initialSize is not supported for file-based "
                          "volumes")
            raise se.InvalidParameterException("initial size",
                                               initialSize)

        sizeBytes = size * BLOCK_SIZE
        truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0

        try:
            oop.getProcessPool(dom.sdUUID).truncateFile(
                volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
                creatExcl=True)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise se.VolumeAlreadyExists(volUUID)
            raise

        if preallocate == sc.PREALLOCATED_VOL:
            try:
                operation = fallocate.allocate(volPath,
                                               sizeBytes)
                with vars.task.abort_callback(operation.abort):
                    with utils.stopwatch("Preallocating volume %s" % volPath):
                        operation.run()
            except exception.ActionStopped:
                raise
            except Exception:
                cls.log.error("Unexpected error", exc_info=True)
                raise se.VolumesZeroingError(volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with size = %s "
                         "sectors", sc.type2name(volFormat), volPath,
                         size)
            if volFormat == sc.COW_FORMAT:
                qemuimg.create(volPath,
                               size=sizeBytes,
                               format=sc.fmt2str(volFormat),
                               qcow2Compat=dom.qcow2_compat())
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID)
            volParent.clone(volPath, volFormat)

        # Forcing the volume permissions in case one of the tools we use
        # (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
        dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)

        return (volPath,)
Example #17
0
    def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
                volParent, srcImgUUID, srcVolUUID, volPath,
                initialSize=None):
        """
        Class specific implementation of volumeCreate. All the exceptions are
        properly handled and logged in volume.create()
        """
        if initialSize:
            cls.log.error("initialSize is not supported for file-based "
                          "volumes")
            raise se.InvalidParameterException("initial size",
                                               initialSize)

        sizeBytes = size * BLOCK_SIZE
        truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0

        try:
            oop.getProcessPool(dom.sdUUID).truncateFile(
                volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
                creatExcl=True)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise se.VolumeAlreadyExists(volUUID)
            raise

        if preallocate == sc.PREALLOCATED_VOL:
            try:
                operation = fallocate.allocate(volPath,
                                               sizeBytes)
                with vars.task.abort_callback(operation.abort):
                    with utils.stopwatch("Preallocating volume %s" % volPath):
                        operation.run()
            except exception.ActionStopped:
                raise
            except Exception:
                cls.log.error("Unexpected error", exc_info=True)
                raise se.VolumesZeroingError(volPath)

        if not volParent:
            cls.log.info("Request to create %s volume %s with size = %s "
                         "sectors", sc.type2name(volFormat), volPath,
                         size)
            if volFormat == sc.COW_FORMAT:
                qemuimg.create(volPath,
                               size=sizeBytes,
                               format=sc.fmt2str(volFormat),
                               qcow2Compat=dom.qcow2_compat())
        else:
            # Create hardlink to template and its meta file
            cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
                         imgUUID, volUUID, srcImgUUID, srcVolUUID)
            volParent.clone(volPath, volFormat)

        # Forcing the volume permissions in case one of the tools we use
        # (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
        dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)

        return (volPath,)
Example #18
0
 def test_default_level_log(self, level):
     log = FakeLogger(level)
     with utils.stopwatch("message", log=log):
         time.sleep(0.01)
     self.assertNotEqual(log.messages, [])
     level, message, kwargs = log.messages[0]
     print("Logged: %s" % message, end=" ")
     self.assertEqual(level, logging.DEBUG)
     self.assertTrue(message.startswith("message"),
                     "Unexpected message: %s" % message)
Example #19
0
def rescan():
    timeout = config.getint('irs', 'scsi_rescan_maximal_timeout')
    log.info("Scanning iSCSI devices")
    try:
        with utils.stopwatch("Scanning iSCSI devices",
                             level=logging.INFO,
                             log=log):
            iscsiadm.session_rescan(timeout=timeout)
    except iscsiadm.IscsiSessionError as e:
        log.error("Scan failed: %s", e)
Example #20
0
def _sanlock_direct(cmd, args, block_size, alignment):
    options = [
        "-Z",
        str(block_size), "-A",
        str(alignment // sc.ALIGNMENT_1M) + "M"
    ]

    with utils.stopwatch("sanlock direct {} {}".format(cmd, args)):
        commands.run([sanlock_direct.SANLOCK.cmd, "direct", cmd] + args +
                     options)
Example #21
0
 def test_default_level_log(self, level):
     log = FakeLogger(level)
     with utils.stopwatch("message", log=log):
         time.sleep(0.01)
     self.assertNotEqual(log.messages, [])
     level, message, kwargs = log.messages[0]
     print("Logged: %s" % message, end=" ")
     self.assertEqual(level, logging.DEBUG)
     self.assertTrue(message.startswith("message"),
                     "Unexpected message: %s" % message)
Example #22
0
def download_image(image_path, url, headers=None):
    # In case of v2 API, we need to append /file suffix to URL to download
    # image content. In case of v1 API, no URL changes are needed.
    if api_version(url) == "v2":
        url = url + "/file"

    with utils.stopwatch("Downloading {} to {}".format(url, image_path),
                         level=logging.INFO,
                         log=log):
        curlImgWrap.download(url, image_path, headers)
Example #23
0
 def _fallocate_volume(cls, vol_path, size):
     try:
         operation = fallocate.allocate(vol_path, size)
         with vars.task.abort_callback(operation.abort):
             with utils.stopwatch("Preallocating volume %s" % vol_path):
                 operation.run()
     except exception.ActionStopped:
         raise
     except Exception:
         cls.log.error("Unexpected error", exc_info=True)
         raise se.VolumesZeroingError(vol_path)
Example #24
0
 def _fallocate_volume(cls, vol_path, size):
     try:
         operation = fallocate.allocate(vol_path, size)
         with vars.task.abort_callback(operation.abort):
             with utils.stopwatch("Preallocating volume %s" % vol_path):
                 operation.run()
     except exception.ActionStopped:
         raise
     except Exception:
         cls.log.error("Unexpected error", exc_info=True)
         raise se.VolumesZeroingError(vol_path)
Example #25
0
def rescan():
    """
    Rescan HBAs discovering new devices.
    """
    log.info("Scanning FC devices")
    try:
        with utils.stopwatch(
                "Scanning FC devices", level=logging.INFO, log=log):
            supervdsm.getProxy().hbaRescan()
    except Error as e:
        log.error("Scanning FC devices failed: %s", e)
Example #26
0
 def testLoopMount(self):
     with namedTemporaryDir() as mpath:
         # two nested with blocks to be python 2.6 friendly
         with createFloppyImage(FLOPPY_SIZE) as path:
             m = mount.Mount(path, mpath)
             m.mount(mntOpts="loop")
             try:
                 self.assertTrue(m.isMounted())
             finally:
                 m.umount(force=True, freeloop=True)
                 # TODO: Use libudev to wait for specific event
                 with stopwatch("Wait for udev events"):
                     udevadm.settle(5)
Example #27
0
 def testLoopMount(self):
     with namedTemporaryDir() as mpath:
         # two nested with blocks to be python 2.6 friendly
         with createFloppyImage(FLOPPY_SIZE) as path:
             m = mount.Mount(path, mpath)
             m.mount(mntOpts="loop")
             try:
                 self.assertTrue(m.isMounted())
             finally:
                 m.umount(force=True, freeloop=True)
                 # TODO: Use libudev to wait for specific event
                 with stopwatch("Wait for udev events"):
                     udevadm.settle(5)
Example #28
0
    def _wait_for_events(self):
        """
        This is an ugly hack to wait until the udev events generated when
        adding or removing a mount are processed.

        Note that we may wait for unrelated events, or wait too little if the
        system is overloaded.

        TODO: find a way to wait for the specific event.
        """
        with utils.stopwatch("Waiting for udev mount events", log=self.log):
            timeout = config.getint('irs', 'udev_settle_timeout')
            udevadm.settle(timeout)
Example #29
0
    def _wait_for_events(self):
        """
        This is an ugly hack to wait until the udev events generated when
        adding or removing a mount are processed.

        Note that we may wait for unrelated events, or wait too little if the
        system is overloaded.

        TODO: find a way to wait for the specific event.
        """
        with utils.stopwatch("Waiting for udev mount events", log=self.log):
            timeout = config.getint('irs', 'udev_settle_timeout')
            udevadm.settle(timeout)
Example #30
0
 def test_mkFloppyFs(self, label):
     """
     Tests mkimage.mkFloppyFs creating an image and checking its content.
     Requires root permissions for writing into the floppy image.
     """
     floppy = mkimage.mkFloppyFs("vmId_floppy", self.files, label)
     self.assertTrue(os.path.exists(floppy))
     m = mount.Mount(floppy, self.workdir)
     m.mount(mntOpts='loop')
     try:
         self._check_content(checkPerms=False)
         self._check_label(floppy, label)
     finally:
         m.umount(force=True, freeloop=True)
         # TODO: Use libudev to wait for specific event
         with stopwatch("Wait for udev events"):
             udevadm.settle(5)
Example #31
0
def _resize_map(name):
    """
    Invoke multipathd to resize a device
    Must run as root

    Raises Error if multipathd failed to resize the map.
    """
    log.debug("Resizing map %r", name)
    cmd = [_MULTIPATHD.cmd, "resize", "map", name]
    with utils.stopwatch("Resized map %r" % name, log=log):
        rc, out, err = utils.execCmd(cmd, raw=True, execCmdLogger=log)
        # multipathd reports some errors using non-zero exit code and stderr
        # (need to be root), but the command may return 0, and the result is
        # reported using stdout.
        if rc != 0 or out != "ok\n":
            raise Error("Resizing map %r failed: out=%r err=%r"
                        % (name, out, err))
Example #32
0
def discard(device_path):
    """
    Discard a block device.

    Discard is best effort; if the operation fails we don't fail the flow
    calling it.

    Arguments:
        device_path (str): Path to block device to discard
    """
    log.info("Discarding device %s", device_path)
    try:
        with utils.stopwatch("Discarded device %s" % device_path,
                             level=logging.INFO, log=log):
            blkdiscard.discard(device_path)
    except cmdutils.Error as e:
        log.warning("Discarding device %s failed: %s", device_path, e)
Example #33
0
def discard(device_path):
    """
    Discard a block device.

    Discard is best effort; if the operation fails we don't fail the flow
    calling it.

    Arguments:
        device_path (str): Path to block device to discard
    """
    log.info("Discarding device %s", device_path)
    try:
        with utils.stopwatch("Discarded device %s" % device_path,
                             level=logging.INFO, log=log):
            blkdiscard.discard(device_path)
    except cmdutils.Error as e:
        log.warning("Discarding device %s failed: %s", device_path, e)
Example #34
0
def resize_devices():
    """
    This is needed in case a device has been increased on the storage server
    Resize multipath map if the underlying slaves are bigger than
    the map size.
    The slaves can be bigger if the LUN size has been increased on the storage
    server after the initial discovery.
    """
    log.info("Resizing multipath devices")
    with utils.stopwatch("Resizing multipath devices",
                         level=logging.INFO,
                         log=log):
        for dmId, guid in getMPDevsIter():
            try:
                _resize_if_needed(guid)
            except Exception:
                log.exception("Could not resize device %s", guid)
Example #35
0
    def acquireHostId(self, hostId, wait):
        self.log.info("Acquiring host id for domain %s (id=%s, wait=%s)",
                      self._sdUUID, hostId, wait)

        # Ensure that future calls to acquire() will wait until host id is
        # acquired.
        self._ready.valid = True

        with self._lock:
            try:
                with utils.stopwatch("sanlock.add_lockspace"):
                    sanlock.add_lockspace(self._lockspace_name,
                                          hostId,
                                          self._idsPath,
                                          wait=wait)
            except sanlock.SanlockException as e:
                if e.errno == errno.EINPROGRESS:
                    # if the request is not asynchronous wait for the ongoing
                    # lockspace operation to complete else silently continue,
                    # the host id has been acquired or it's in the process of
                    # being acquired (async).
                    if wait:
                        if not sanlock.inq_lockspace(self._lockspace_name,
                                                     hostId,
                                                     self._idsPath,
                                                     wait=True):
                            raise se.AcquireHostIdFailure(self._sdUUID, e)
                        self.log.info(
                            "Host id for domain %s successfully "
                            "acquired (id=%s, wait=%s)", self._sdUUID, hostId,
                            wait)
                        self._ready.set()
                elif e.errno == errno.EEXIST:
                    self.log.info(
                        "Host id for domain %s already acquired "
                        "(id=%s, wait=%s)", self._sdUUID, hostId, wait)
                    self._ready.set()
                else:
                    raise se.AcquireHostIdFailure(self._sdUUID, e)
            else:
                if wait:
                    self.log.info(
                        "Host id for domain %s successfully "
                        "acquired (id=%s, wait=%s)", self._sdUUID, hostId,
                        wait)
                    self._ready.set()
Example #36
0
 def test_mkFloppyFs(self, label):
     """
     Tests mkimage.mkFloppyFs creating an image and checking its content.
     Requires root permissions for writing into the floppy image.
     """
     floppy = mkimage.mkFloppyFs("vmId_floppy", self.files, label)
     self.assertTrue(os.path.exists(floppy))
     m = mount.Mount(floppy, self.workdir)
     m.mount(mntOpts="loop")
     try:
         self._check_content(checkPerms=False)
         self._check_label(floppy, label)
     finally:
         m.umount(force=True, freeloop=True)
         # TODO: Use libudev to wait for specific event
         with stopwatch("Wait for udev events"):
             udevadm.settle(5)
Example #37
0
def _resize_map(name):
    """
    Invoke multipathd to resize a device
    Must run as root

    Raises Error if multipathd failed to resize the map.
    """
    log.debug("Resizing map %r", name)
    cmd = [_MULTIPATHD.cmd, "resize", "map", name]
    with utils.stopwatch("Resized map %r" % name, log=log):
        rc, out, err = commands.execCmd(cmd, raw=True, execCmdLogger=log)
        # multipathd reports some errors using non-zero exit code and stderr
        # (need to be root), but the command may return 0, and the result is
        # reported using stdout.
        if rc != 0 or out != "ok\n":
            raise Error("Resizing map %r failed: out=%r err=%r"
                        % (name, out, err))
Example #38
0
File: sdc.py Project: dong-df/vdsm
    def refreshStorage(self, resize=True):
        self.log.info("Refreshing storage domain cache (resize=%s)", resize)
        with utils.stopwatch("Refreshing storage domain cache",
                             level=logging.INFO,
                             log=self.log):
            self.__staleStatus = self.STORAGE_REFRESHING

            multipath.rescan()
            if resize:
                multipath.resize_devices()
            lvm.invalidateCache()

            # If a new invalidateStorage request came in after the refresh
            # started then we cannot flag the storages as updated (force a
            # new rescan later).
            with self._syncroot:
                if self.__staleStatus == self.STORAGE_REFRESHING:
                    self.__staleStatus = self.STORAGE_UPDATED
Example #39
0
def zero(device_path, size=None, task=None):
    """
    Zero a block device.

    Arguments:
        device_path (str): Path to block device to wipe
        size (int): Number of bytes to write. If not specified, use the device
            size. Size must be aligned to `vdsm.storage.constants.BLOCK_SIZE`.
        task (`storage.task.Task`): Task running this operation. If specified,
            the zero operation will be aborted if the task is aborted.

    Raises:
        `vdsm.common.exception.ActionStopped` if the wipe was aborted
        `vdsm.storage.exception.VolumesZeroingError` if writing to storage
            failed.
        `vdsm.storage.exception.InvalidParameterException` if size is not
            aligned to `vdsm.storage.constants.BLOCK_SIZE`.
    """
    if size is None:
        # Always aligned to LVM extent size (128MiB).
        size = fsutils.size(device_path)
    elif size % sc.BLOCK_SIZE:
        raise se.InvalidParameterException("size", size)

    log.info("Zeroing device %s (size=%d)", device_path, size)
    with utils.stopwatch("Zero device %s" % device_path,
                         level=logging.INFO,
                         log=log):
        try:
            # Write optimal size blocks. Images are always aligned to
            # optimal size blocks, so we typically have only one call.
            blocks = size // OPTIMAL_BLOCK_SIZE
            if blocks > 0:
                _zero(device_path, 0, OPTIMAL_BLOCK_SIZE, blocks, task=task)

            # When zeroing special volumes size may not be aligned to
            # optimal block size, so we need to write the last block.
            rest = size % OPTIMAL_BLOCK_SIZE
            if rest > 0:
                offset = blocks * OPTIMAL_BLOCK_SIZE
                _zero(device_path, offset, rest, 1, task=task)
        except se.StorageException as e:
            raise se.VolumesZeroingError("Zeroing device %s failed: %s" %
                                         (device_path, e))
Example #40
0
 def test_mkIsoFs(self, label):
     """
     Tests mkimage.mkIsoFs creating an image and checking its content
     """
     iso_img = mkimage.mkIsoFs("vmId_iso", self.files, label)
     self.assertTrue(os.path.exists(iso_img))
     m = mount.Mount(iso_img, self.workdir)
     try:
         m.mount(mntOpts='loop')
     except mount.MountError as e:
         raise SkipTest("Error mounting iso image: %s" % e)
     try:
         self._check_content()
         self._check_label(iso_img, label)
     finally:
         m.umount(force=True, freeloop=True)
         # TODO: Use libudev to wait for specific event
         with stopwatch("Wait for udev events"):
             udevadm.settle(5)
Example #41
0
def zero(device_path, size=None, task=None):
    """
    Zero a block device.

    Arguments:
        device_path (str): Path to block device to wipe
        size (int): Number of bytes to write. If not specified, use the device
            size. Size must be aligned to `vdsm.storage.constants.BLOCK_SIZE`.
        task (`storage.task.Task`): Task running this operation. If specified,
            the zero operation will be aborted if the task is aborted.

    Raises:
        `vdsm.common.exception.ActionStopped` if the wipe was aborted
        `vdsm.storage.exception.VolumesZeroingError` if writing to storage
            failed.
        `vdsm.storage.exception.InvalidParameterException` if size is not
            aligned to `vdsm.storage.constants.BLOCK_SIZE`.
    """
    if size is None:
        # Always aligned to LVM extent size (128MiB).
        size = fsutils.size(device_path)
    elif size % sc.BLOCK_SIZE:
        raise se.InvalidParameterException("size", size)

    log.info("Zeroing device %s (size=%d)", device_path, size)
    with utils.stopwatch("Zero device %s" % device_path,
                         level=logging.INFO, log=log):
        try:
            # Write optimal size blocks. Images are always aligned to
            # optimal size blocks, so we typically have only one call.
            blocks = size // OPTIMAL_BLOCK_SIZE
            if blocks > 0:
                _zero(device_path, 0, OPTIMAL_BLOCK_SIZE, blocks, task=task)

            # When zeroing special volumes size may not be aligned to
            # optimal block size, so we need to write the last block.
            rest = size % OPTIMAL_BLOCK_SIZE
            if rest > 0:
                offset = blocks * OPTIMAL_BLOCK_SIZE
                _zero(device_path, offset, rest, 1, task=task)
        except se.StorageException as e:
            raise se.VolumesZeroingError("Zeroing device %s failed: %s"
                                         % (device_path, e))
Example #42
0
 def test_mkIsoFs(self, label):
     """
     Tests mkimage.mkIsoFs creating an image and checking its content
     """
     iso_img = mkimage.mkIsoFs("vmId_iso", self.files, label)
     self.assertTrue(os.path.exists(iso_img))
     m = mount.Mount(iso_img, self.workdir)
     try:
         m.mount(mntOpts="loop")
     except mount.MountError as e:
         raise SkipTest("Error mounting iso image: %s" % e)
     try:
         self._check_content()
         self._check_label(iso_img, label)
     finally:
         m.umount(force=True, freeloop=True)
         # TODO: Use libudev to wait for specific event
         with stopwatch("Wait for udev events"):
             udevadm.settle(5)
Example #43
0
def copyFromImage(dstImgPath, methodArgs):
    fileObj = methodArgs['fileObj']
    bytes_left = total_size = methodArgs['length']

    # Unlike copyToImage, we must use direct I/O to avoid reading stale data
    # from host page cache, in case OVF disk was modified on another host.
    cmd = [
        constants.EXT_DD,
        "if=%s" % dstImgPath,
        "bs=%s" % MiB,
        "count=%s" % (total_size // MiB + 1),
        "iflag=direct",
    ]

    log.info("Copy from image %s", dstImgPath)
    with utils.stopwatch(
            "Copy %s bytes" % total_size, level=logging.INFO, log=log):
        p = commands.start(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        with commands.terminating(p):
            _copyData(p.stdout, fileObj, bytes_left)
Example #44
0
def zero(device_path, size=None, task=_NullTask()):
    """
    Zero a block device.

    Arguments:
        device_path (str): Path to block device to wipe
        size (int): Number of bytes to write. If not specified, use the device
            size. Size must be aligned to `vdsm.storage.constants.BLOCK_SIZE`.
        task (`storage.task.Task`): Task running this operation. If specified,
            the zero operation will be aborted if the task is aborted.

    Raises:
        `vdsm.common.exception.ActionStopped` if the wipe was aborted
        `vdsm.storage.exception.VolumesZeroingError` if writing to storage
            failed.
        `vdsm.storage.exception.InvalidParameterException` if size is not
            aligned to `vdsm.storage.constants.BLOCK_SIZE`.
    """
    if size is None:
        # Always aligned to LVM extent size (128MiB).
        size = fsutils.size(device_path)
    elif size % sc.BLOCK_SIZE:
        raise se.InvalidParameterException("size", size)

    log.info("Zeroing device %s (size=%d)", device_path, size)
    with utils.stopwatch("Zero device %s" % device_path,
                         level=logging.INFO,
                         log=log):
        zero_method = config.get('irs', 'zero_method')
        try:
            if zero_method == "blkdiscard":
                _zero_blkdiscard(device_path, size, task)
            elif zero_method == "dd":
                _zero_dd(device_path, size, task)
            else:
                raise exception.InvalidConfiguration(
                    reason="Unsupported value for irs:zero_method",
                    zero_method=zero_method)
        except se.StorageException as e:
            raise se.VolumesZeroingError("Zeroing device %s failed: %s" %
                                         (device_path, e))
Example #45
0
    def acquireHostId(self, hostId, wait):
        self.log.info("Acquiring host id for domain %s (id=%s, wait=%s)",
                      self._sdUUID, hostId, wait)

        # Ensure that future calls to acquire() will wait until host id is
        # acquired.
        self._ready.valid = True

        with self._lock:
            try:
                with utils.stopwatch("sanlock.add_lockspace"):
                    sanlock.add_lockspace(self._sdUUID, hostId, self._idsPath,
                                          **{'async': not wait})
            except sanlock.SanlockException as e:
                if e.errno == errno.EINPROGRESS:
                    # if the request is not asynchronous wait for the ongoing
                    # lockspace operation to complete else silently continue,
                    # the host id has been acquired or it's in the process of
                    # being acquired (async).
                    if wait:
                        if not sanlock.inq_lockspace(self._sdUUID, hostId,
                                                     self._idsPath, wait=True):
                            raise se.AcquireHostIdFailure(self._sdUUID, e)
                        self.log.info("Host id for domain %s successfully "
                                      "acquired (id=%s, wait=%s)",
                                      self._sdUUID, hostId, wait)
                        self._ready.set()
                elif e.errno == errno.EEXIST:
                    self.log.info("Host id for domain %s already acquired "
                                  "(id=%s, wait=%s)",
                                  self._sdUUID, hostId, wait)
                    self._ready.set()
                else:
                    raise se.AcquireHostIdFailure(self._sdUUID, e)
            else:
                if wait:
                    self.log.info("Host id for domain %s successfully "
                                  "acquired (id=%s, wait=%s)",
                                  self._sdUUID, hostId, wait)
                    self._ready.set()
Example #46
0
def zero(device_path, size=None, task=_NullTask()):
    """
    Zero a block device.

    Arguments:
        device_path (str): Path to block device to wipe
        size (int): Number of bytes to write. If not specified, use the device
            size. Size must be aligned to `vdsm.storage.constants.BLOCK_SIZE`.
        task (`storage.task.Task`): Task running this operation. If specified,
            the zero operation will be aborted if the task is aborted.

    Raises:
        `vdsm.common.exception.ActionStopped` if the wipe was aborted
        `vdsm.storage.exception.VolumesZeroingError` if writing to storage
            failed.
        `vdsm.storage.exception.InvalidParameterException` if size is not
            aligned to `vdsm.storage.constants.BLOCK_SIZE`.
    """
    if size is None:
        # Always aligned to LVM extent size (128MiB).
        size = fsutils.size(device_path)
    elif size % sc.BLOCK_SIZE:
        raise se.InvalidParameterException("size", size)

    log.info("Zeroing device %s (size=%d)", device_path, size)
    with utils.stopwatch("Zero device %s" % device_path,
                         level=logging.INFO, log=log):
        zero_method = config.get('irs', 'zero_method')
        try:
            if zero_method == "blkdiscard":
                _zero_blkdiscard(device_path, size, task)
            elif zero_method == "dd":
                _zero_dd(device_path, size, task)
            else:
                raise exception.InvalidConfiguration(
                    reason="Unsupported value for irs:zero_method",
                    zero_method=zero_method)
        except se.StorageException as e:
            raise se.VolumesZeroingError("Zeroing device %s failed: %s"
                                         % (device_path, e))
Example #47
0
 def testSymlinkMount(self):
     with namedTemporaryDir() as root_dir:
         backing_image = os.path.join(root_dir, 'backing.img')
         link_to_image = os.path.join(root_dir, 'link_to_image')
         mountpoint = os.path.join(root_dir, 'mountpoint')
         with open(backing_image, 'w') as f:
             os.ftruncate(f.fileno(), 1024 ** 3)
         rc, out, err = execCmd(['/sbin/mkfs.ext2', "-F", backing_image],
                                raw=True)
         if rc != 0:
             raise RuntimeError("Error creating filesystem: %s" % err)
         os.symlink(backing_image, link_to_image)
         os.mkdir(mountpoint)
         m = mount.Mount(link_to_image, mountpoint)
         m.mount(mntOpts="loop")
         try:
             self.assertTrue(m.isMounted())
         finally:
             m.umount(force=True, freeloop=True)
             # TODO: Use libudev to wait for specific event
             with stopwatch("Wait for udev events"):
                 udevadm.settle(5)
Example #48
0
 def test_custom_level_log(self):
     log = FakeLogger(logging.INFO)
     with utils.stopwatch("message", level=logging.INFO, log=log):
         pass
     self.assertNotEqual(log.messages, [])
Example #49
0
 def test_default_level_no_log(self):
     log = FakeLogger(logging.INFO)
     with utils.stopwatch("message", log=log):
         pass
     self.assertEqual(log.messages, [])
Example #50
0
 def test_info(self):
     log = FakeLogger(logging.INFO)
     with utils.stopwatch("message", log=log):
         pass
     self.assertEqual(log.messages, [])
Example #51
0
 def test_debug(self):
     log = FakeLogger(logging.DEBUG)
     with utils.stopwatch("message", log=log):
         pass
     self.assertNotEqual(log.messages, [])
Example #52
0
 def test_notset(self):
     log = FakeLogger(logging.NOTSET)
     with utils.stopwatch("message", log=log):
         pass
     self.assertNotEqual(log.messages, [])