示例#1
0
文件: xlease_test.py 项目: oVirt/vdsm
    def test_rebuild_empty(self, fake_sanlock):
        with make_volume() as vol:
            # Add underlying sanlock resources
            for i in [3, 4, 6]:
                resource = "%04d" % i
                offset = xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * i
                fake_sanlock.write_resource(
                    vol.lockspace, resource, [(vol.path, offset)])
            # The index is empty
            assert vol.leases() == {}

            # After rebuilding the index it should contain all the underlying
            # resources.
            file = xlease.DirectFile(vol.path)
            with utils.closing(file):
                xlease.rebuild_index(vol.lockspace, file)
            expected = {
                "0003": {
                    "offset": xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 3,
                    "updating": False,
                },
                "0004": {
                    "offset": xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 4,
                    "updating": False,
                },
                "0006": {
                    "offset": xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 6,
                    "updating": False,
                },
            }
            file = xlease.DirectFile(vol.path)
            with utils.closing(file):
                vol = xlease.LeasesVolume(file)
                with utils.closing(vol):
                    assert vol.leases() == expected
示例#2
0
文件: xlease_test.py 项目: oVirt/vdsm
def write_records(records, file):
    index = xlease.VolumeIndex()
    with utils.closing(index):
        index.load(file)
        for recnum, record in records:
            block = index.copy_record_block(recnum)
            with utils.closing(block):
                block.write_record(recnum, record)
                block.dump(file)
示例#3
0
文件: xlease_test.py 项目: oVirt/vdsm
 def test_pread(self, tmpdir, direct_file, offset, size):
     data = b"a" * 512 + b"b" * 512 + b"c" * 512 + b"d" * 512
     path = tmpdir.join("file")
     path.write(data)
     file = direct_file(str(path))
     with utils.closing(file):
         buf = mmap.mmap(-1, size)
         with utils.closing(buf):
             n = file.pread(offset, buf)
             assert n == size
             assert buf[:] == data[offset:offset + size]
示例#4
0
文件: xlease_test.py 项目: oVirt/vdsm
 def test_remove_write_failure(self):
     record = xlease.Record(make_uuid(), 0, updating=True)
     with make_volume((42, record)) as base:
         file = FailingWriter(base.path)
         with utils.closing(file):
             vol = xlease.LeasesVolume(file)
             with utils.closing(vol):
                 with pytest.raises(WriteError):
                     vol.remove(record.resource)
                 # Must succeed becuase writng to storage failed
                 assert record.resource in vol.leases()
示例#5
0
文件: xlease_test.py 项目: oVirt/vdsm
 def test_pread_short(self, tmpdir, direct_file):
     data = b"a" * 1024
     path = tmpdir.join("file")
     path.write(data)
     file = direct_file(str(path))
     with utils.closing(file):
         buf = mmap.mmap(-1, 1024)
         with utils.closing(buf):
             n = file.pread(512, buf)
             assert n == 512
             assert buf[:n] == data[512:]
示例#6
0
文件: xlease_test.py 项目: oVirt/vdsm
def make_volume(*records):
    with make_leases() as path:
        lockspace = os.path.basename(os.path.dirname(path))
        file = xlease.DirectFile(path)
        with utils.closing(file):
            xlease.format_index(lockspace, file)
            if records:
                write_records(records, file)
            vol = xlease.LeasesVolume(file)
            with utils.closing(vol):
                yield vol
示例#7
0
文件: xlease_test.py 项目: oVirt/vdsm
 def test_add_write_failure(self):
     with make_volume() as base:
         file = FailingWriter(base.path)
         with utils.closing(file):
             vol = xlease.LeasesVolume(file)
             with utils.closing(vol):
                 lease_id = make_uuid()
                 with pytest.raises(WriteError):
                     vol.add(lease_id)
                 # Must succeed becuase writng to storage failed
                 assert lease_id not in vol.leases()
示例#8
0
文件: sd.py 项目: EdDev/vdsm
def _external_leases_volume(path):
    """
    Context manager returning the external leases volume.

    The caller is responsible for holding the external_leases_lock in the
    correct mode.
    """
    backend = xlease.DirectFile(path)
    with utils.closing(backend):
        vol = xlease.LeasesVolume(backend)
        with utils.closing(vol):
            yield vol
示例#9
0
文件: xlease_test.py 项目: oVirt/vdsm
 def test_pwrite(self, tmpdir, direct_file, offset, size):
     # Create a file full of "a"s
     path = tmpdir.join("file")
     path.write(b"a" * 2048)
     buf = mmap.mmap(-1, size)
     with utils.closing(buf):
         # Write "b"s
         buf.write(b"b" * size)
         file = direct_file(str(path))
         with utils.closing(file):
             file.pwrite(offset, buf)
     data = path.read()
     expected = ("a" * offset +
                 "b" * size +
                 "a" * (2048 - offset - size))
     assert data == expected
示例#10
0
def main(*args):
    """
    This tool is used to check and optionally repair broken volume leases.
    """
    parsed_args = _parse_args(args)

    if not parsed_args.repair and not _confirm_check_leases():
        return

    cli = client.connect(parsed_args.host, parsed_args.port,
                         use_tls=parsed_args.use_ssl)
    with utils.closing(cli):
        print()
        print("Checking active storage domains. This can take several "
              "minutes, please wait.")
        broken_leases = _get_leases_to_repair(cli)
        if not broken_leases:
            print()
            print("There are no leases to repair.")
            return

    print()
    _print_broken_leases(broken_leases)
    if not parsed_args.repair and not _confirm_repair_leases():
        return

    _repair(broken_leases)
示例#11
0
文件: client.py 项目: EdDev/vdsm
def main(args=None):
    schema = find_schema()
    namespaces = create_namespaces(schema)
    parser = option_parser(namespaces)
    args = parser.parse_args(args)
    try:
        if args.method_args and args.file is not None:
            raise UsageError("Conflicting command line parameters: %r and "
                             "file option: %r" % (args.method_args, args.file))

        namespace = args.namespace
        method = args.method

        if args.file:
            request_params = parse_file(args.file)
        else:
            request_params = parse_params(args.method_args)

        cli = client.connect(args.host, port=args.port, use_tls=args.use_tls,
                             timeout=args.timeout,
                             gluster_enabled=args.gluster_enabled)

        with utils.closing(cli):
            command = getattr(getattr(cli, namespace), method)
            result = command(**request_params)
            print(json.dumps(result, indent=4))
    except UsageError as e:
        parser.error(str(e))
    except Exception as e:
        fail(e)
示例#12
0
def dump_chains(*args):
    """
    dump-volume-chains
    Query VDSM about the existing structure of image volumes and prints
    them in an ordered fashion with optional additional info per volume.
    Alternatively, dumps the volumes information in json format without
    analysis.
    """
    parsed_args = _parse_args(args)
    cli = client.connect(parsed_args.host, parsed_args.port,
                         use_tls=parsed_args.use_ssl)
    with utils.closing(cli):
        volumes_info = _get_volumes_info(cli, parsed_args.sd_uuid)
        if parsed_args.output == 'text':
            # perform analysis and print in human readable format
            image_chains = _get_volumes_chains(volumes_info)
            _print_volume_chains(image_chains, volumes_info)
        elif parsed_args.output == 'json':
            # no analysis, dump chains in json format
            json.dump(volumes_info, sys.stdout, indent=2)
        elif parsed_args.output == 'sqlite':
            # no analysis, dump chains in sql format
            _dump_sql(volumes_info, parsed_args.sqlite_file)
        else:
            raise ValueError('unknown output format %s' % parsed_args.output)
示例#13
0
文件: utils_test.py 项目: EdDev/vdsm
    def test_write(self, cmd, recv_out, recv_err):
        text = bytes('Hello World')
        received = bytearray()

        def recv_data(buffer):
            # cannot use received += buffer with a variable
            # defined in the parent function.
            operator.iadd(received, buffer)

        c = self._startCommand(cmd)
        p = utils.CommandStream(
            c,
            recv_data if recv_out else self.assertUnexpectedCall,
            recv_data if recv_err else self.assertUnexpectedCall)
        with utils.closing(p):
            c.stdin.write(text)
            c.stdin.flush()
            c.stdin.close()

            while not p.closed:
                p.receive()

        retcode = c.wait()

        self.assertEqual(retcode, 0)
        self.assertEqual(text, str(received))
示例#14
0
    def _run(self):
        with guarded.context(self._source.locks + self._dest.locks):
            with self._source.prepare(), self._dest.prepare():
                # Do not start copying if we have already been aborted
                if self._status == jobs.STATUS.ABORTED:
                    return

                # Workaround for volumes containing VM configuration info that
                # were created with invalid vdsm metadata.
                if self._source.is_invalid_vm_conf_disk():
                    src_format = dst_format = qemuimg.FORMAT.RAW
                else:
                    src_format = self._source.qemu_format
                    dst_format = self._dest.qemu_format

                with self._dest.volume_operation():
                    self._operation = qemuimg.convert(
                        self._source.path,
                        self._dest.path,
                        srcFormat=src_format,
                        dstFormat=dst_format,
                        dstQcow2Compat=self._dest.qcow2_compat,
                        backing=self._dest.backing_path,
                        backingFormat=self._dest.backing_qemu_format)
                    with utils.closing(self._operation):
                        self._operation.wait_for_completion()
示例#15
0
文件: xlease.py 项目: nirs/vdsm
    def pread(self, offset, buf):
        """
        Read len(buf) bytes from storage at offset into mmap buf.

        Returns:
            The number bytes read (int).
        """
        self._file.seek(offset, os.SEEK_SET)
        pos = 0
        if six.PY2:
            # There is no way to create a writable memoryview on mmap object in
            # python 2, so we must read into a temporary buffer and copy into
            # the given buffer.
            rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED)
            with utils.closing(rbuf, log=log.name):
                while pos < len(buf):
                    # TODO: Handle EOF
                    nread = uninterruptible(self._file.readinto, rbuf)
                    if nread == 0:
                        break  # EOF
                    buf.write(rbuf[:nread])
                    pos += nread
        else:
            # In python 3 we can read directly into the underlying buffer
            # without any copies using a memoryview.
            while pos < len(buf):
                rbuf = memoryview(buf)[pos:]
                # TODO: Handle EOF
                nread = uninterruptible(self._file.readinto, rbuf)
                if nread == 0:
                    break  # EOF
                pos += nread
        return pos
示例#16
0
 def test_partial(self, output_list, progress_list):
     p = qemuimg.QemuImgOperation(['true'])
     with utils.closing(p):
         for output, progress in zip(output_list, progress_list):
             p._recvstdout(output)
             self.assertEqual(p.progress, progress)
         p.poll()
         self.assertEqual(p.finished, True)
示例#17
0
def resume_paused_vm(vm_id):
    unpause_file = MARK_FOR_UNPAUSE_PATH % vm_id
    if os.path.isfile(unpause_file):
        use_tls = config.getboolean('vars', 'ssl')
        cli = client.connect('localhost', use_tls=use_tls)
        with utils.closing(cli):
            cli.VM.cont(vmID=vm_id)
        os.remove(unpause_file)
示例#18
0
    def test_progress_simple(self):
        p = qemuimg.QemuImgOperation(['true'])
        with utils.closing(p):
            for progress in self._progress_iterator():
                p._recvstdout(self.PROGRESS_FORMAT % progress)
                self.assertEqual(p.progress, progress)

            p.poll()
            self.assertEqual(p.finished, True)
示例#19
0
def converted_size(filename, compat):
    converted = filename + ".qcow2"
    operation = qemuimg.convert(filename,
                                converted,
                                srcFormat=qemuimg.FORMAT.RAW,
                                dstFormat=qemuimg.FORMAT.QCOW2,
                                dstQcow2Compat=compat)
    with utils.closing(operation):
        operation.wait_for_completion()
    return os.stat(converted).st_size
示例#20
0
    def test_unexpected_output(self):
        p = qemuimg.QemuImgOperation(['true'])
        with utils.closing(p):
            self.assertRaises(ValueError, p._recvstdout, 'Hello World\r')

            p._recvstdout('Hello ')
            self.assertRaises(ValueError, p._recvstdout, 'World\r')

            p.poll()
            self.assertEqual(p.finished, True)
示例#21
0
文件: sd.py 项目: oVirt/vdsm
    def rebuild_external_leases(self):
        """
        Rebuild the external leases volume index from volume contents.

        Must be called only on the SPM.
        """
        with self._manifest.external_leases_lock.exclusive:
            path = self.external_leases_path()
            backend = xlease.DirectFile(path)
            with utils.closing(backend):
                xlease.rebuild_index(self.sdUUID, backend)
示例#22
0
文件: merge.py 项目: EdDev/vdsm
 def _run(self):
     self.log.info("Merging subchain %s", self.subchain)
     with guarded.context(self.subchain.locks):
         self.subchain.validate()
         with self.subchain.prepare(), self.subchain.volume_operation():
             self.operation = qemuimg.commit(
                 self.subchain.top_vol.getVolumePath(),
                 topFormat=sc.fmt2str(self.subchain.top_vol.getFormat()),
                 base=self.subchain.base_vol.getVolumePath())
             with utils.closing(self.operation):
                 self.operation.wait_for_completion()
示例#23
0
    def test_progress_batch(self):
        p = qemuimg.QemuImgOperation(['true'])
        with utils.closing(p):
            p._recvstdout(
                (self.PROGRESS_FORMAT % 10.00) +
                (self.PROGRESS_FORMAT % 25.00) +
                (self.PROGRESS_FORMAT % 33.33))

            self.assertEqual(p.progress, 33.33)

            p.poll()
            self.assertEqual(p.finished, True)
示例#24
0
文件: xlease.py 项目: nirs/vdsm
    def _write_record(self, recnum, record):
        """
        Write record recnum to storage atomically.

        Copy the block where the record is located, modify it and write the
        block to storage. If this succeeds, write the record to the index.
        """
        block = self._index.copy_record_block(recnum)
        with utils.closing(block):
            block.write_record(recnum, record)
            block.dump(self._file)
        self._index.write_record(recnum, record)
示例#25
0
文件: sd.py 项目: oVirt/vdsm
    def external_leases_volume(self):
        """
        Context manager returning the external leases volume.

        The caller is responsible for holding the external_leases_lock in the
        correct mode.
        """
        path = self.external_leases_path()
        with self.external_leases_backend(self.sdUUID, path) as backend:
            vol = xlease.LeasesVolume(backend)
            with utils.closing(vol):
                yield vol
示例#26
0
    def external_leases_backend(cls, lockspace, path):
        """
        Overrides StorageDomainManifest method to use an interruptible direct
        file implementation that will not make the processs uninterruptible if
        the storage becomes non-responsive. See
        https://bugzilla.redhat.com/1518676.

        See StorageDomainManifest.external_leases_backend for more info.
        """
        dom_oop = oop.getProcessPool(lockspace)
        backend = xlease.InterruptibleDirectFile(path, dom_oop)
        with utils.closing(backend):
            yield backend
示例#27
0
 def test_leases(self, tmp_vol, fake_sanlock):
     vol = xlease.LeasesVolume(tmp_vol.backend)
     with utils.closing(vol):
         uuid = make_uuid()
         lease_info = vol.add(uuid)
         leases = vol.leases()
         expected = {
             uuid: {
                 "offset": lease_info.offset,
                 "updating": False,
             }
         }
         assert leases == expected
示例#28
0
    def test_echo(self, size, use_ssl):
        data = dummyTextGenerator(size)

        with constructAcceptor(self.log, use_ssl, _SampleBridge()) as acceptor:
            sslctx = DEAFAULT_SSL_CONTEXT if use_ssl else None

            with utils.closing(
                    StandAloneRpcClient(acceptor._host, acceptor._port,
                                        'jms.topic.vdsm_requests',
                                        str(uuid4()), sslctx,
                                        False)) as client:
                self.assertEqual(
                    client.callMethod('echo', (data, ), str(uuid4())), data)
示例#29
0
    def test_commit_progress(self):
        with namedTemporaryDir() as tmpdir:
            size = 1048576
            base = os.path.join(tmpdir, "base.img")
            make_image(base, size, qemuimg.FORMAT.RAW, 0, "1.1")

            top = os.path.join(tmpdir, "top.img")
            make_image(top, size, qemuimg.FORMAT.QCOW2, 1, "1.1", base)

            op = qemuimg.commit(top, topFormat=qemuimg.FORMAT.QCOW2)
            with utils.closing(op):
                op.wait_for_completion()
                self.assertEqual(100, op.progress)
示例#30
0
文件: utils_test.py 项目: EdDev/vdsm
    def test_timeout(self):
        c = self._startCommand(["sleep", "5"])
        p = utils.CommandStream(c, self.assertUnexpectedCall,
                                self.assertUnexpectedCall)
        with utils.closing(p):
            with self.assertElapsed(2):
                p.receive(2)

            self.assertEqual(p.closed, False)

        c.terminate()

        self.assertEqual(c.wait(), -signal.SIGTERM)
示例#31
0
def dump_chains(*args):
    """
    dump-volume-chains
    Query VDSM about the existing structure of image volumes and prints
    them in an ordered fashion with optional additional info per volume.
    """
    parsed_args = _parse_args(args)
    cli = client.connect(parsed_args.host, parsed_args.port,
                         use_tls=parsed_args.use_ssl)
    with utils.closing(cli):
        image_chains, volumes_info = _get_volumes_chains(
            cli, parsed_args.sd_uuid)
        _print_volume_chains(image_chains, volumes_info)
示例#32
0
文件: fileSD.py 项目: oVirt/vdsm
    def external_leases_backend(cls, lockspace, path):
        """
        Overrides StorageDomainManifest method to use an interruptible direct
        file implementation that will not make the processs uninterruptible if
        the storage becomes non-responsive. See
        https://bugzilla.redhat.com/1518676.

        See StorageDomainManifest.external_leases_backend for more info.
        """
        dom_oop = oop.getProcessPool(lockspace)
        backend = xlease.InterruptibleDirectFile(path, dom_oop)
        with utils.closing(backend):
            yield backend
示例#33
0
    def test_signals(self, method, expected_retcode):
        c = self._startCommand(["sleep", "2"])
        p = utils.CommandStream(c, self.assertUnexpectedCall,
                                self.assertUnexpectedCall)
        with utils.closing(p):
            getattr(c, method)()

            try:
                with self.assertElapsed(0):
                    p.receive(2)
            finally:
                retcode = c.wait()

        self.assertEqual(retcode, expected_retcode)
示例#34
0
 def test_remove_exists(self, tmp_vol, fake_sanlock):
     vol = xlease.LeasesVolume(tmp_vol.backend)
     with utils.closing(vol):
         leases = [make_uuid() for i in range(3)]
         for lease in leases:
             vol.add(lease)
         lease = vol.lookup(leases[1])
         vol.remove(lease.resource)
         assert lease.resource not in vol.leases()
         res = fake_sanlock.read_resource(lease.path, lease.offset)
         # There is no sanlock api for removing a resource, so we mark a
         # removed resource with empty (invalid) lockspace and lease id.
         assert res["lockspace"] == ""
         assert res["resource"] == ""
示例#35
0
文件: sd.py 项目: benipeled/vdsm
    def rebuild_external_leases(self):
        """
        Rebuild the external leases volume index from volume contents.

        Must be called only on the SPM.
        """
        with self._manifest.external_leases_lock.exclusive:
            path = self.external_leases_path()
            backend = xlease.DirectFile(path)
            with utils.closing(backend):
                xlease.rebuild_index(self.sdUUID,
                                     backend,
                                     alignment=self._manifest.alignment,
                                     block_size=self._manifest.block_size)
示例#36
0
文件: sd.py 项目: benipeled/vdsm
    def external_leases_volume(self):
        """
        Context manager returning the external leases volume.

        The caller is responsible for holding the external_leases_lock in the
        correct mode.
        """
        path = self.external_leases_path()
        with self.external_leases_backend(self.sdUUID, path) as backend:
            vol = xlease.LeasesVolume(backend,
                                      alignment=self._alignment,
                                      block_size=self._block_size)
            with utils.closing(vol):
                yield vol
示例#37
0
文件: utils_test.py 项目: EdDev/vdsm
    def test_signals(self, method, expected_retcode):
        c = self._startCommand(["sleep", "2"])
        p = utils.CommandStream(c, self.assertUnexpectedCall,
                                self.assertUnexpectedCall)
        with utils.closing(p):
            getattr(c, method)()

            try:
                with self.assertElapsed(0):
                    p.receive(2)
            finally:
                retcode = c.wait()

        self.assertEqual(retcode, expected_retcode)
示例#38
0
def dump_chains(*args):
    """
    dump-volume-chains
    Query VDSM about the existing structure of image volumes and prints
    them in an ordered fashion with optional additional info per volume.
    """
    parsed_args = _parse_args(args)
    cli = client.connect(parsed_args.host,
                         parsed_args.port,
                         use_tls=parsed_args.use_ssl)
    with utils.closing(cli):
        image_chains, volumes_info = _get_volumes_chains(
            cli, parsed_args.sd_uuid)
        _print_volume_chains(image_chains, volumes_info)
示例#39
0
 def test_add_exists(self, tmp_vol, fake_sanlock):
     vol = xlease.LeasesVolume(tmp_vol.backend,
                               alignment=tmp_vol.alignment,
                               block_size=tmp_vol.block_size)
     with utils.closing(vol):
         lease_id = make_uuid()
         lease = vol.add(lease_id)
         with pytest.raises(xlease.LeaseExists):
             vol.add(lease_id)
         res = fake_sanlock.read_resource(lease.path,
                                          lease.offset,
                                          align=tmp_vol.alignment,
                                          sector=tmp_vol.block_size)
         assert res["lockspace"] == lease.lockspace.encode("utf-8")
         assert res["resource"] == lease.resource.encode("utf-8")
示例#40
0
文件: sd.py 项目: benipeled/vdsm
    def external_leases_backend(cls, lockspace, path):
        """
        Return a context manager for performing I/O to the extenal leases
        volume.

        Arguments:
            lockspace (str): Sanlock lockspace name, storage domain uuid.
            path (str): Path to the external leases volume

        Returns:
            context manager.
        """
        backend = xlease.DirectFile(path)
        with utils.closing(backend):
            yield backend
示例#41
0
    def test_rebuild_empty(self, tmp_vol, fake_sanlock):
        vol = xlease.LeasesVolume(tmp_vol.backend)
        with utils.closing(vol):
            # Add underlying sanlock resources
            for i in [3, 4, 6]:
                resource = "%04d" % i
                offset = xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * i
                fake_sanlock.write_resource(
                    vol.lockspace, resource, [(vol.path, offset)])
            # The index is empty
            assert vol.leases() == {}

            # After rebuilding the index it should contain all the underlying
            # resources.
            file = xlease.DirectFile(vol.path)
            with utils.closing(file):
                xlease.rebuild_index(vol.lockspace, file)
            expected = {
                "0003": {
                    "offset": xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 3,
                    "updating": False,
                },
                "0004": {
                    "offset": xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 4,
                    "updating": False,
                },
                "0006": {
                    "offset": xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 6,
                    "updating": False,
                },
            }
            file = xlease.DirectFile(vol.path)
            with utils.closing(file):
                vol = xlease.LeasesVolume(file)
                with utils.closing(vol):
                    assert vol.leases() == expected
示例#42
0
 def test_add_sanlock_failure(self, tmp_vol, fake_sanlock):
     vol = xlease.LeasesVolume(tmp_vol.backend)
     with utils.closing(vol):
         lease_id = make_uuid()
         # Make sanlock fail to write a resource
         fake_sanlock.errors["write_resource"] = \
             fake_sanlock.SanlockException
         with pytest.raises(fake_sanlock.SanlockException):
             vol.add(lease_id)
         # We should have an updating lease record
         lease = vol.leases()[lease_id]
         assert lease["updating"]
         # There should be no lease on storage
         with pytest.raises(fake_sanlock.SanlockException) as e:
             fake_sanlock.read_resource(vol.path, lease["offset"])
             assert e.exception.errno == fake_sanlock.SANLK_LEADER_MAGIC
示例#43
0
def rebuild_index(
        lockspace, file, alignment=sc.ALIGNMENT_1M,
        block_size=sc.BLOCK_SIZE_512):
    """
    Rebuild xleases volume index from underlying storage.

    This operation synchronizes the index with the actual sanlock resource on
    storage, assuming that existing sanlock resources are the one and only
    truth.

    Like format_index, if the operation fails the index is left in "updating"
    state.

    Raises:
    - OSError if I/O operation failed
    - sanlock.SanlockException if sanlock operation failed
    """
    log.info("Rebuilding index for lockspace %r (version=%d)",
             lockspace, INDEX_VERSION)
    index = VolumeIndex()
    with utils.closing(index):
        with index.updating(lockspace, file):
            # Read resources and write records
            max_offset = file.size() - SLOT_SIZE
            for recnum in range(MAX_RECORDS):
                offset = lease_offset(recnum)
                if offset > max_offset:
                    index.write_record(recnum, EMPTY_RECORD)
                    continue
                try:
                    res = read_resource(
                        file.name,
                        offset,
                        alignment=alignment,
                        block_size=block_size)
                except NoSuchResource:
                    record = EMPTY_RECORD
                else:
                    log.debug("Restoring lease %s", res)
                    record = Record(res.resource, offset)
                index.write_record(recnum, record)

            # Attempt to write index to file
            log.debug("Writing new index")
            index.dump(file)
示例#44
0
 def test_remove_sanlock_failure(self, tmp_vol, fake_sanlock):
     vol = xlease.LeasesVolume(tmp_vol.backend)
     with utils.closing(vol):
         lease_id = make_uuid()
         vol.add(lease_id)
         # Make sanlock fail to remove a resource (currnently removing a
         # resouce by writing invalid lockspace and resoruce name).
         fake_sanlock.errors["write_resource"] = \
             fake_sanlock.SanlockException
         with pytest.raises(fake_sanlock.SanlockException):
             vol.remove(lease_id)
         # We should have an updating lease record
         lease = vol.leases()[lease_id]
         assert lease["updating"]
         # There lease should still be on storage
         res = fake_sanlock.read_resource(vol.path, lease["offset"])
         assert res["lockspace"] == vol.lockspace
         assert res["resource"] == lease_id
示例#45
0
    def format_external_leases(cls, lockspace, path):
        """
        Format the special xleases volume.

        Called when creating a new storage domain, or when upgrading storage
        domain to version 4.

        WARNING: destructive operation, must not be called on active external
        leases volume.

        TODO: should move to storage domain subclasses of each subclass can use
        its own backend.

        Must be called only on the SPM.
        """
        backend = xlease.DirectFile(path)
        with utils.closing(backend):
            xlease.format_index(lockspace, backend)
示例#46
0
 def test_add_first_free_slot(self, tmp_vol, fake_sanlock):
     vol = xlease.LeasesVolume(tmp_vol.backend)
     with utils.closing(vol):
         uuids = [make_uuid() for i in range(4)]
         for uuid in uuids[:3]:
             vol.add(uuid)
         vol.remove(uuids[1])
         vol.add(uuids[3])
         leases = vol.leases()
         # The first lease in the first slot
         assert leases[uuids[0]]["offset"] == xlease.USER_RESOURCE_BASE
         # The forth lease was added in the second slot after the second
         # lease was removed.
         assert (leases[uuids[3]]["offset"] ==
                 xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE)
         # The third lease in the third slot
         assert (leases[uuids[2]]["offset"] ==
                 xlease.USER_RESOURCE_BASE + xlease.SLOT_SIZE * 2)
示例#47
0
def format_xleases(*args):
    """
    format-xleases sd_id path

    WARNING:

        This is a destructive operation - you must put the storage
        domain into maintenance before running this tool.

    Format the xleases volume index, dropping all leases from the index.
    This does not delete sanlock resources on the volume. If you want to
    restore existing sanlock resources, use rebuild-index. If you want
    to delete also sanlock resources on this volume you can wipe the
    entire volume using dd before formatting it.

    Notes:

    - With iSCSI based storage you may need to connect to the traget
      using iscsiadm.

    - With file based storage, you may need to mount the storage domain.

    - With block based stoage, you need to activate the xleases logical
      volume before the operation, and deactivate after the operation.

    If formatting fails, the volume will not be usable (it will be
    marked as "updating"), but the operation can be tried again.

    Creating xleases volume on file storage:

        PATH=/rhev/data-center/mnt/server:_path/sd-id/dom_md/xleases
        truncate -s 1G $PATH
        vdsm-tool format-xleases sd-id $PATH

    Creating the xleases volume on block storage:

        lvcreate --name xleases --size 1g sd-id
        vdsm-tool format-xleases sd-id /dev/sd-id/xleases
        lvchange -an sd-id/xleases
    """
    args = parse_args(args)
    backend = xlease.DirectFile(args.path)
    with utils.closing(backend):
        xlease.format_index(args.sd_id, backend)
示例#48
0
def rebuild_xleases(*args):
    """
    rebuild-xleases sd_id path

    WARNING:

        This is a destructive operation - you must put the storage
        domain into maintenance before running this tool.
        The xleases volume index is the source of truth so rebuilding
        from storage can break it badly.

    Rebuild the xleases volume index, restoring all sanlock resource on
    the xleases volume. If you want to drop all leases in the index, use
    format-xleases.

    Notes:

    - With iSCSI based storage you may need to connect to the traget
      using iscsiadm.

    - With file based storage, you may need to mount the storage domain.

    - With block based stoage, you need to activate the xleases logical
      volume before the operation, and deactivate after the operation.

    If rebuilding fails, the volume will not be usable (it will be
    marked as "updating"), but the operation can be tried again.

    Rebuilding xleases volume on file storage:

        PATH=/rhev/data-center/mnt/server:_path/sd-id/dom_md/xleases
        vdsm-tool rebuild-xleases sd-id $PATH

    Rebuilding the xleases volume on block storage:

        lvchange -ay sd-id/xleases
        vdsm-tool rebuild-xleases sd-id /dev/sd-id/xleases
        lvchange -an sd-id/xleases
    """
    args = parse_args(args)
    backend = xlease.DirectFile(args.path)
    with utils.closing(backend):
        xlease.rebuild_index(args.sd_id, backend)
示例#49
0
 def readinto(self, buf):
     pos = 0
     if six.PY2:
         # There is no way to create a writable memoryview on mmap object in
         # python 2, so we must read into a temporary buffer and copy into
         # the given buffer.
         rbuf = mmap.mmap(-1, len(buf), mmap.MAP_SHARED)
         with utils.closing(rbuf, log=log.name):
             while pos < len(buf):
                 nread = uninterruptible(self._file.readinto, rbuf)
                 buf.write(rbuf[:nread])
                 pos += nread
     else:
         # In python 3 we can read directly into the underlying buffer
         # without any copies using a memoryview.
         while pos < len(buf):
             rbuf = memoryview(buf)[pos:]
             pos += uninterruptible(self._file.readinto, rbuf)
     return pos
示例#50
0
    def test_commit(self, qcow2_compat, base, top, use_base):
        size = 1048576
        with namedTemporaryDir() as tmpdir:
            chain = []
            parent = None
            # Create a chain of 4 volumes.
            for i in range(4):
                vol = os.path.join(tmpdir, "vol%d.img" % i)
                format = (qemuimg.FORMAT.RAW
                          if i == 0 else qemuimg.FORMAT.QCOW2)
                make_image(vol, size, format, i, qcow2_compat, parent)
                orig_offset = qemuimg.check(vol)["offset"] if i > 0 else None
                chain.append((vol, orig_offset))
                parent = vol

            base_vol = chain[base][0]
            top_vol = chain[top][0]
            op = qemuimg.commit(top_vol,
                                topFormat=qemuimg.FORMAT.QCOW2,
                                base=base_vol if use_base else None)
            with utils.closing(op):
                op.wait_for_completion()

            base_fmt = (qemuimg.FORMAT.RAW
                        if base == 0 else qemuimg.FORMAT.QCOW2)
            for i in range(base, top + 1):
                offset = i * 1024
                pattern = 0xf0 + i
                # The base volume must have the data from all the volumes
                # merged into it.
                qemu_pattern_verify(base_vol,
                                    base_fmt,
                                    offset=offset,
                                    len=1024,
                                    pattern=pattern)
                if i > base:
                    # internal and top volumes should keep the data, we
                    # may want to wipe this data when deleting the volumes
                    # later.
                    vol, orig_offset = chain[i]
                    actual_offset = qemuimg.check(vol)["offset"]
                    self.assertEqual(actual_offset, orig_offset)
示例#51
0
def format_index(lockspace, file):
    """
    Format xleases volume index, deleting all existing records.

    Should be used only when creating a new leases volume, or if the volume
    should be repaired. Afterr formatting the index, the index can be rebuilt
    from storage contents.

    Raises:
    - OSError if I/O operation failed
    """
    log.info("Formatting index for lockspace %r (version=%d)",
             lockspace, INDEX_VERSION)
    index = VolumeIndex()
    with utils.closing(index):
        with index.updating(lockspace, file):
            # Write empty records
            for recnum in range(MAX_RECORDS):
                index.write_record(recnum, EMPTY_RECORD)
            # Attempt to write index to file
            index.dump(file)
示例#52
0
    def test_event(self, use_ssl):
        done = threading.Event()

        with constructAcceptor(self.log, use_ssl, _SampleBridge(),
                               'jms.queue.events') as acceptor:
            sslctx = DEAFAULT_SSL_CONTEXT if use_ssl else None
            with utils.closing(StandAloneRpcClient(acceptor._host,
                                                   acceptor._port,
                                                   'jms.topic.vdsm_requests',
                                                   'jms.queue.events',
                                                   sslctx, False)) as client:

                def callback(client, event, params):
                    self.assertEqual(event, 'vdsm.event')
                    self.assertEqual(params['content'], True)
                    done.set()

                client.registerEventCallback(callback)
                client.callMethod("event", [], str(uuid4()))
                done.wait(timeout=CALL_TIMEOUT)
                self.assertTrue(done.is_set())
示例#53
0
    def updating(self, lockspace, file):
        """
        Context manager for index updates.

        Before entering the context, mark the index as updating. When exiting
        cleanly from the context, clear the updating flag. If the user code
        fails, the index will be left in updating state.
        """
        # Mark as updating
        metadata = IndexMetadata(INDEX_VERSION, lockspace, updating=True)
        self.write_metadata(metadata)

        # Call withotu try-finally intentionally, so failure in the caller code
        # will leave the index mark as "updating".
        yield

        # Clear updating flag
        metadata = IndexMetadata(INDEX_VERSION, lockspace)
        block = ChangeBlock(metadata.bytes(), 0)
        with utils.closing(block):
            block.dump(file)
示例#54
0
def main(args=None):
    preliminary_parser = argparse.ArgumentParser(add_help=False)
    preliminary_parser.add_argument('--gluster-enabled',
                                    dest="gluster_enabled",
                                    action="store_true",
                                    help="gluster enabled")
    preliminary_parser.set_defaults(gluster_enabled=False)
    known_args, extra = preliminary_parser.parse_known_args()
    schema = find_schema(known_args.gluster_enabled)
    namespaces = create_namespaces(schema)

    parser = option_parser(namespaces)
    args = parser.parse_args(extra)
    try:
        if args.method_args and args.file is not None:
            raise UsageError("Conflicting command line parameters: %r and "
                             "file option: %r" % (args.method_args, args.file))

        namespace = args.namespace
        method = args.method

        if args.file:
            request_params = parse_file(args.file)
        else:
            request_params = parse_params(args.method_args)

        cli = client.connect(args.host,
                             port=args.port,
                             use_tls=args.use_tls,
                             timeout=args.timeout,
                             gluster_enabled=known_args.gluster_enabled)

        with utils.closing(cli):
            command = getattr(getattr(cli, namespace), method)
            result = command(**request_params)
            print(json.dumps(result, indent=4))
    except UsageError as e:
        parser.error(str(e))
    except Exception as e:
        fail(e)
示例#55
0
    def test_event(self, use_ssl):
        with constructAcceptor(self.log, use_ssl, _SampleBridge(),
                               'jms.queue.events') as acceptor:
            sslctx = DEAFAULT_SSL_CONTEXT if use_ssl else None
            with utils.closing(
                    StandAloneRpcClient(acceptor._host, acceptor._port,
                                        'jms.topic.vdsm_requests',
                                        'jms.queue.events', sslctx,
                                        False)) as client:

                event_queue = queue.Queue()
                custom_topic = 'jms.queue.events'
                client.subscribe(custom_topic, event_queue)

                client.callMethod("event", [], str(uuid4()))

                try:
                    event, event_params = event_queue.get(timeout=CALL_TIMEOUT)
                except queue.Empty:
                    self.fail("Event queue timed out.")
                self.assertEqual(event, 'vdsm.event')
                self.assertEqual(event_params['content'], True)
示例#56
0
    def test_receive(self, cmd, recv_out, recv_err):
        text = bytes('Hello World')
        received = bytearray()

        def recv_data(buffer):
            # cannot use received += buffer with a variable
            # defined in the parent function.
            operator.iadd(received, buffer)

        cmd[-1] = cmd[-1] % text

        c = self._startCommand(cmd)
        p = utils.CommandStream(
            c, recv_data if recv_out else self.assertUnexpectedCall,
            recv_data if recv_err else self.assertUnexpectedCall)
        with utils.closing(p):
            while not p.closed:
                p.receive()

        retcode = c.wait()

        self.assertEqual(retcode, 0)
        self.assertEqual(text, received)
示例#57
0
    def test_add_first_free_slot(self, tmp_vol, fake_sanlock):
        vol = xlease.LeasesVolume(tmp_vol.backend,
                                  alignment=tmp_vol.alignment,
                                  block_size=tmp_vol.block_size)
        with utils.closing(vol):
            uuids = [make_uuid() for i in range(4)]
            for uuid in uuids[:3]:
                vol.add(uuid)
            vol.remove(uuids[1])
            vol.add(uuids[3])
            leases = vol.leases()

            # The first lease in the first slot
            offset = xlease.lease_offset(0, tmp_vol.alignment)
            assert leases[uuids[0]]["offset"] == offset

            # The forth lease was added in the second slot after the second
            # lease was removed.
            offset = xlease.lease_offset(1, tmp_vol.alignment)
            assert leases[uuids[3]]["offset"] == offset

            # The third lease in the third slot
            offset = xlease.lease_offset(2, tmp_vol.alignment)
            assert leases[uuids[2]]["offset"] == offset
示例#58
0
 def test_error_while_closing(self):
     c = Closer()
     with self.assertRaises(CloseError):
         with utils.closing(c):
             pass
示例#59
0
 def test_error_before_close(self):
     c = Closer()
     with self.assertRaises(UserError):
         with utils.closing(c):
             raise UserError
         self.assertTrue(c.was_closed)
示例#60
0
 def test_size(self, direct_file):
     with make_leases() as path:
         file = direct_file(path)
         with utils.closing(file):
             assert file.size() == constants.GIB