Ejemplo n.º 1
0
    def setUp(self):
        # monkey-patch swiftclient to use out mock up
        import swiftnbd.swift as swift
        swift.client = MockConnection
        from swiftnbd.swift import SwiftStorage

        # create a disk doubling the actual size of the mock up so we
        # have some uninitialised space to run tests
        self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
        self.store.flush()
    def setUp(self):
        # monkey-patch swiftclient to use out mock up
        import swiftnbd.swift as swift
        swift.client = MockConnection
        from swiftnbd.swift import SwiftStorage

        # create a disk doubling the actual size of the mock up so we
        # have some uninitialised space to run tests
        self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
        self.store.flush()
Ejemplo n.º 3
0
    def do_download(self):

        self.log.debug("downloading %s" % self.args.container)

        cli, meta = self._setup_client()
        if cli is None:
            return 1
        elif 'client' in meta:
            self.log.error("%s is locked, downloading a container in use is unreliable" % self.args.container)
            return 1

        object_size = int(meta['object-size'])
        objects = int(meta['objects'])

        store = SwiftStorage(self.authurl,
                             self.username,
                             self.password,
                             self.args.container,
                             object_size,
                             objects,
                             )
        try:
            store.lock("ctl-download")
        except StorageError as ex:
            self.log.error(ex)
            return 1

        size = 0
        fdo = None
        try:
            fdo = open(self.args.image, "w")

            while True:
                data = store.read(object_size)
                if data == '':
                    break
                fdo.write(data)
                size += len(data)
                if not self.args.quiet:
                    sys.stdout.write("\rDownloading %s [%.2d%%]" % (self.args.container, 100*size/(objects*object_size)))
                    sys.stdout.flush()
        except IOError as ex:
            self.log.error(ex)
            return 1
        except KeyboardInterrupt:
            self.log.warning("user interrupt")
            return 1
        finally:
            if fdo:
                fdo.close()

            try:
                store.unlock()
            except StorageError as ex:
                self.log.warning("Failed to unlock %s: %s" % (self.args.container, ex))

        if not self.args.quiet:
            sys.stdout.write("\r")
            sys.stdout.flush()

        self.log.info("Done, %s bytes written" % size)

        return 0
Ejemplo n.º 4
0
class SwiftStorageTestCase(unittest.TestCase):
    """Test the object-split file class."""
    def setUp(self):
        # monkey-patch swiftclient to use out mock up
        import swiftnbd.swift as swift
        swift.client = MockConnection
        from swiftnbd.swift import SwiftStorage

        # create a disk doubling the actual size of the mock up so we
        # have some uninitialised space to run tests
        self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
        self.store.flush()

    def tearDown(self):
        pass

    def test_read_full_object_content(self):
        self.store.seek(0)
        data = self.store.read(512)
        self.assertEqual(data, b'\xff' * 512)

    def test_read_full_object_no_content(self):
        self.store.seek(8 * 512)
        data = self.store.read(512)
        self.assertEqual(data, b'\0' * 512)

    def test_write_full_object(self):
        self.store.seek(0)
        self.store.write(b'X' * 512)
        self.assertEqual(MockConnection.object(0), b'X' * 512)

        self.store.seek(8 * 512)
        self.store.write(b'X' * 512)
        self.assertEqual(MockConnection.object(8), b'X' * 512)

    def test_read_partial_object_content(self):
        self.store.seek(0)
        data = self.store.read(256)
        self.assertEqual(data, b'\xff' * 256)

    def test_read_partial_object_no_content(self):
        self.store.seek(8 * 512)
        data = self.store.read(256)
        self.assertEqual(data, b'\0' * 256)

    def test_write_partial_object_content(self):
        self.store.seek(0)
        self.store.write(b'X' * 256)
        self.assertEqual(MockConnection.object(0), b'X' * 256 + b'\xff' * 256)

    def test_write_partial_object_no_content(self):
        self.store.seek(8 * 512)
        self.store.write(b'X' * 256)
        self.assertEqual(MockConnection.object(8), b'X' * 256 + b'\0' * 256)

    def test_read_inter_object_content(self):
        self.store.seek(256)
        data = self.store.read(512)
        self.assertEqual(data, b'\xff' * 512)

    def test_read_inter_object_no_content(self):
        self.store.seek(8 * 512 + 256)
        data = self.store.read(512)
        self.assertEqual(data, b'\0' * 512)

    def test_read_inter_object_content_and_no_content(self):
        self.store.seek(8 * 512 - 256)
        data = self.store.read(512)
        self.assertEqual(data, b'\xff' * 256 + b'\0' * 256)

    def test_write_inter_object_content(self):
        self.store.seek(256)
        self.store.write(b'X' * 512)
        self.assertEqual(MockConnection.object(0), b'\xff' * 256 + b'X' * 256)
        self.assertEqual(MockConnection.object(1), b'X' * 256 + b'\xff' * 256)

    def test_write_inter_object_no_content(self):
        self.store.seek(8 * 512 + 256)
        self.store.write(b'X' * 512)
        self.assertEqual(MockConnection.object(8), b'\0' * 256 + b'X' * 256)
        self.assertEqual(MockConnection.object(9), b'X' * 256 + b'\0' * 256)

    def test_write_inter_object_content_and_no_content(self):
        self.store.seek(8 * 512 - 256)
        self.store.write(b'X' * 512)
        self.assertEqual(MockConnection.object(7), b'\xff' * 256 + b'X' * 256)
        self.assertEqual(MockConnection.object(8), b'X' * 256 + b'\0' * 256)

    def test_seek_bad_offset(self):
        self.assertRaises(IOError, self.store.seek, -1)
        self.assertRaises(IOError, self.store.seek, 10000000000000)
        try:
            self.store.seek(-1)
        except IOError as ex:
            self.assertEqual(ex.errno, errno.ESPIPE)
        else:
            self.fail("didn't raise IOError")

    def test_tell(self):
        self.store.seek(0)
        self.assertEqual(self.store.tell(), 0)
        self.store.seek(1024)
        self.assertEqual(self.store.tell(), 1024)

    def test_read_end_of_disk(self):
        self.store.seek(15 * 512)
        data = self.store.read(1024)
        self.assertEqual(len(data), 512)

    def test_wite_end_of_disk(self):
        self.store.seek(15 * 512)
        self.assertRaises(IOError, self.store.write, b'X' * 1024)
class SwiftStorageTestCase(unittest.TestCase):
    """Test the object-split file class."""
    def setUp(self):
        # monkey-patch swiftclient to use out mock up
        import swiftnbd.swift as swift
        swift.client = MockConnection
        from swiftnbd.swift import SwiftStorage

        # create a disk doubling the actual size of the mock up so we
        # have some uninitialised space to run tests
        self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
        self.store.flush()

    def tearDown(self):
        pass

    def test_read_full_object_content(self):
        self.store.seek(0)
        data = self.store.read(512)
        self.assertEqual(data, b'\xff'*512)

    def test_read_full_object_no_content(self):
        self.store.seek(8*512)
        data = self.store.read(512)
        self.assertEqual(data, b'\0'*512)

    def test_write_full_object(self):
        self.store.seek(0)
        self.store.write(b'X'*512)
        self.assertEqual(MockConnection.object(0), b'X'*512)

        self.store.seek(8*512)
        self.store.write(b'X'*512)
        self.assertEqual(MockConnection.object(8), b'X'*512)

    def test_read_partial_object_content(self):
        self.store.seek(0)
        data = self.store.read(256)
        self.assertEqual(data, b'\xff'*256)

    def test_read_partial_object_no_content(self):
        self.store.seek(8*512)
        data = self.store.read(256)
        self.assertEqual(data, b'\0'*256)

    def test_write_partial_object_content(self):
        self.store.seek(0)
        self.store.write(b'X'*256)
        self.assertEqual(MockConnection.object(0), b'X'*256 + b'\xff'*256)

    def test_write_partial_object_no_content(self):
        self.store.seek(8*512)
        self.store.write(b'X'*256)
        self.assertEqual(MockConnection.object(8), b'X'*256 + b'\0'*256)

    def test_read_inter_object_content(self):
        self.store.seek(256)
        data = self.store.read(512)
        self.assertEqual(data, b'\xff'*512)

    def test_read_inter_object_no_content(self):
        self.store.seek(8*512 + 256)
        data = self.store.read(512)
        self.assertEqual(data, b'\0'*512)

    def test_read_inter_object_content_and_no_content(self):
        self.store.seek(8*512 - 256)
        data = self.store.read(512)
        self.assertEqual(data, b'\xff'*256 +  b'\0'*256)

    def test_write_inter_object_content(self):
        self.store.seek(256)
        self.store.write(b'X'*512)
        self.assertEqual(MockConnection.object(0), b'\xff'*256 + b'X'*256)
        self.assertEqual(MockConnection.object(1), b'X'*256 + b'\xff'*256)

    def test_write_inter_object_no_content(self):
        self.store.seek(8*512 + 256)
        self.store.write(b'X'*512)
        self.assertEqual(MockConnection.object(8), b'\0'*256 + b'X'*256)
        self.assertEqual(MockConnection.object(9), b'X'*256 + b'\0'*256)

    def test_write_inter_object_content_and_no_content(self):
        self.store.seek(8*512 - 256)
        self.store.write(b'X'*512)
        self.assertEqual(MockConnection.object(7), b'\xff'*256 + b'X'*256)
        self.assertEqual(MockConnection.object(8), b'X'*256 + b'\0'*256)

    def test_seek_bad_offset(self):
        self.assertRaises(IOError, self.store.seek, -1)
        self.assertRaises(IOError, self.store.seek, 10000000000000)
        try:
            self.store.seek(-1)
        except IOError as ex:
            self.assertEqual(ex.errno, errno.ESPIPE)
        else:
            self.fail("didn't raise IOError")

    def test_tell(self):
        self.store.seek(0)
        self.assertEqual(self.store.tell(), 0)
        self.store.seek(1024)
        self.assertEqual(self.store.tell(), 1024)

    def test_read_end_of_disk(self):
        self.store.seek(15*512)
        data = self.store.read(1024)
        self.assertEqual(len(data), 512)

    def test_wite_end_of_disk(self):
        self.store.seek(15*512)
        self.assertRaises(IOError, self.store.write, b'X'*1024)
Ejemplo n.º 6
0
    def do_download(self):

        self.log.debug("downloading %s" % self.args.container)

        cli, meta = self._setup_client()
        if cli is None:
            return 1
        elif 'client' in meta:
            self.log.error(
                "%s is locked, downloading a container in use is unreliable" %
                self.args.container)
            return 1

        object_size = int(meta['object-size'])
        objects = int(meta['objects'])

        store = SwiftStorage(
            self.auth,
            self.args.container,
            object_size,
            objects,
        )
        try:
            store.lock("ctl-download")
        except StorageError as ex:
            self.log.error(ex)
            return 1

        size = 0
        fdo = None
        try:
            fdo = open(self.args.image, "wb")

            while True:
                data = store.read(object_size)
                if data == b'':
                    break
                fdo.write(data)
                size += len(data)
                if not self.args.quiet:
                    sys.stdout.write("\rDownloading %s [%.2d%%]" %
                                     (self.args.container, 100 * size /
                                      (objects * object_size)))
                    sys.stdout.flush()
        except IOError as ex:
            self.log.error(ex)
            return 1
        except KeyboardInterrupt:
            self.log.warning("user interrupt")
            return 1
        finally:
            if fdo:
                fdo.close()

            try:
                store.unlock()
            except StorageError as ex:
                self.log.warning("Failed to unlock %s: %s" %
                                 (self.args.container, ex))

        if not self.args.quiet:
            sys.stdout.write("\r")
            sys.stdout.flush()

        self.log.info("Done, %s bytes written" % size)

        return 0
Ejemplo n.º 7
0
    def run(self):

        if os.path.isfile(self.args.pidfile):
            self.log.error("%s found: is the server already running?" %
                           self.args.pidfile)
            return 1

        stores = dict()
        for container, values in self.conf.items():
            auth = dict(
                authurl=self.args.authurl,
                user=values['username'],
                key=values['password'],
            )

            if self.args.keystone:
                try:
                    from keystoneclient.v2_0 import client as _check_for_ksclient
                except ImportError:
                    sys.exit(
                        "auth 2.0 (keystone) requires python-keystoneclient")
                else:
                    self.log.debug("using auth 2.0 (keystone)")

                if self.args.keystone_separator not in values['username']:
                    self.log.error("%s: separator not found in %r, skipping" %
                                   (container, values['username']))
                    continue

                keystone_auth = values['username'].split(
                    self.args.keystone_separator, 1)
                auth['tenant_name'], auth['user'] = keystone_auth
                auth['auth_version'] = '2.0'
                auth['os_options'] = dict(
                    service_type=self.args.keystone_service,
                    endpoint_type=self.args.keystone_endpoint,
                    region_name=self.args.keystone_region,
                )
                self.log.debug("os_options: %r" % auth['os_options'])

            cli = client.Connection(**auth)

            try:
                headers, _ = cli.get_container(container)
            except (socket.error, client.ClientException) as ex:
                if getattr(ex, 'http_status', None) == 404:
                    self.log.warning("%s doesn't exist, skipping" % container)
                    continue
                else:
                    self.log.error("%s: %r, skipping" % (container, ex.msg))
                    continue

            self.log.debug(headers)

            meta = getMeta(headers)
            if not meta:
                self.log.warning("%s doesn't appear to be setup, skipping" %
                                 container)
                continue

            self.log.debug("Meta: %s" % meta)

            try:
                object_size = int(meta['object-size'])
                objects = int(meta['objects'])
            except ValueError as ex:
                self.log.error("%s doesn't appear to be correct: %s" %
                               (container, ex))
                return 1

            if meta['version'] != disk_version:
                self.log.warning("Version mismatch %s != %s in %s" %
                                 (meta['version'], disk_version, container))

            stores[container] = SwiftStorage(
                auth,
                container,
                object_size,
                objects,
                Cache(int(self.args.cache_limit * 1024**2 / object_size)),
                values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
            )

        addr = (self.args.bind_address, self.args.bind_port)
        server = Server(addr, stores)

        if not self.args.foreground:
            try:
                if os.fork() != 0:
                    os._exit(0)
            except OSError as ex:
                self.log.error("Failed to daemonize: %s" % ex)
                return 1

            os.setsid()
            fd = os.open(os.devnull, os.O_RDWR)
            os.dup2(fd, sys.stdin.fileno())
            os.dup2(fd, sys.stdout.fileno())
            os.dup2(fd, sys.stderr.fileno())

        self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))

        try:
            fd = os.open(self.args.pidfile,
                         (os.O_CREAT | os.O_EXCL | os.O_WRONLY), 0o644)
        except OSError as ex:
            self.log.error("Failed to create the pidfile: %s" % ex)
            return 1

        with os.fdopen(fd, "w") as pidfile_handle:
            pidfile_handle.write("%s\n" % os.getpid())
            pidfile_handle.flush()

            server.serve_forever()

        os.remove(self.args.pidfile)

        # unlock the storages before exit
        server.unlock_all()

        self.log.info("Exiting...")
        return 0
Ejemplo n.º 8
0
    def run(self):

        if os.path.isfile(self.args.pidfile):
            self.log.error("%s found: is the server already running?" %
                           self.args.pidfile)
            return 1

        stores = dict()
        for container, values in self.conf.items():
            cli = client.Connection(values['authurl'], values['username'],
                                    values['password'])

            try:
                headers, _ = cli.get_container(container)
            except (socket.error, client.ClientException) as ex:
                if getattr(ex, 'http_status', None) == 404:
                    self.log.warning("%s doesn't exist, skipping" % container)
                    continue
                else:
                    self.log.error(ex)
                    return 1

            self.log.debug(headers)

            meta = getMeta(headers)
            if not meta:
                self.log.warning("%s doesn't appear to be setup, skipping" %
                                 container)
                continue

            self.log.debug("Meta: %s" % meta)

            try:
                object_size = int(meta['object-size'])
                objects = int(meta['objects'])
            except ValueError as ex:
                self.log.error("%s doesn't appear to be correct: %s" %
                               (container, ex))
                return 1

            if meta['version'] != disk_version:
                self.log.warning("Version mismatch %s != %s in %s" %
                                 (meta['version'], disk_version, container))

            stores[container] = SwiftStorage(
                values['authurl'],
                values['username'],
                values['password'],
                container,
                object_size,
                objects,
                Cache(int(self.args.cache_limit * 1024**2 / object_size)),
                values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
            )

        addr = (self.args.bind_address, self.args.bind_port)
        server = Server(addr, stores)

        if not self.args.foreground:
            try:
                if os.fork() != 0:
                    os._exit(0)
            except OSError as ex:
                self.log.error("Failed to daemonize: %s" % ex)
                return 1

            os.setsid()
            fd = os.open(os.devnull, os.O_RDWR)
            os.dup2(fd, sys.stdin.fileno())
            os.dup2(fd, sys.stdout.fileno())
            os.dup2(fd, sys.stderr.fileno())

        self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))

        try:
            fd = os.open(self.args.pidfile,
                         (os.O_CREAT | os.O_EXCL | os.O_WRONLY), 0o644)
        except OSError as ex:
            self.log.error("Failed to create the pidfile: %s" % ex)
            return 1

        with os.fdopen(fd, "w") as pidfile_handle:
            pidfile_handle.write("%s\n" % os.getpid())
            pidfile_handle.flush()

            server.serve_forever()

        os.remove(self.args.pidfile)

        # unlock the storages before exit
        server.unlock_all()

        self.log.info("Exiting...")
        return 0