コード例 #1
0
ファイル: control.py プロジェクト: thurloat/swift-nbd-server
class Main(object):

    def __init__(self):
        parser = ArgumentParser(description="swiftnbd control tool.",
                                epilog="Contact and support: %s" % project_url
                                )

        subp = parser.add_subparsers(title="commands")

        p = subp.add_parser('list', help='list all containers and their information')
        p.add_argument("-s", "--simple-output", dest="simple",
                       action="store_true",
                       help="write simplified output to stdout")
        p.set_defaults(func=self.do_list)

        p = subp.add_parser('setup', help='setup a container to be used by the server')
        p.add_argument("container", help="container to setup")
        p.add_argument("objects", help="number of objects")

        p.add_argument("--object-size", dest="object_size",
                       default=object_size,
                       help="object size (default: %s)" % object_size)

        p.add_argument("-f", "--force", dest="force",
                       action="store_true",
                       help="force operation")

        p.set_defaults(func=self.do_setup)

        p = subp.add_parser('unlock', help='unlock a container')
        p.add_argument("container", help="container to unlock")
        p.set_defaults(func=self.do_unlock)

        p = subp.add_parser('lock', help='lock a container')
        p.add_argument("container", help="container to lock")
        p.set_defaults(func=self.do_lock)

        p = subp.add_parser('download', help='download a container as a raw image')
        p.add_argument("container", help="container to download")
        p.add_argument("image", help="local file to store the image")
        p.add_argument("-q", "--quiet", dest="quiet",
                       action="store_true",
                       help="don't show the process bar")
        p.set_defaults(func=self.do_download)

        p = subp.add_parser('delete', help='delete a container')
        p.add_argument("container", help="container to delete")
        p.set_defaults(func=self.do_delete)

        parser.add_argument("--version", action="version", version="%(prog)s "  + version)

        parser.add_argument("--secrets", dest="secrets_file",
                            default=secrets_file,
                            help="filename containing user/password (default: %s)" % secrets_file)

        parser.add_argument("-a", "--auth-url", dest="authurl",
                            default=auth_url,
                            help="default authentication URL (default: %s)" % auth_url)

        parser.add_argument("-v", "--verbose", dest="verbose",
                            action="store_true",
                            help="enable verbose logging"
                            )

        self.args = parser.parse_args()

        self.log = setLog(debug=self.args.verbose)

        try:
            self.conf = Config(self.args.secrets_file, self.args.authurl)
        except OSError as ex:
            parser.error("Failed to load secrets: %s" % ex)

        # setup by _setup_client()
        self.authurl = None
        self.username = None
        self.password = None

    def run(self):
        return self.args.func()

    def do_list(self):

        self.log.debug("listing containers")

        if self.args.simple:
            out = print
        else:
            out = self.log.info

        prev_authurl = None
        for container, values in self.conf.iteritems():
            if prev_authurl or prev_authurl != values['authurl']:
                cli = client.Connection(
                    values['authurl'], values['username'], values['password'],
                    auth_version=values['auth_version'],
                    tenant_name=values['tenant_name'])

            try:
                headers, _ = cli.get_container(container)
            except (socket.error, client.ClientException) as ex:
                if getattr(ex, 'http_status', None) == 404:
                    self.log.error("%s doesn't exist (auth-url: %s)" % (container, values['authurl']))
                else:
                    self.log.error(ex)
            else:

                meta = getMeta(headers)
                self.log.debug("Meta: %s" % meta)

                if meta:
                    lock = "unlocked" if not 'client' in meta else "locked by %s" % meta['client']
                    out("%s objects=%s size=%s (version=%s, %s)" % (container,
                                                                    meta['objects'],
                                                                    meta['object-size'],
                                                                    meta['version'],
                                                                    lock,
                                                                    ))
                else:
                    out("%s is not a swiftnbd container" % container)

            prev_authurl = values['authurl']

        return 0

    def _setup_client(self, create=False):
        """
        Setup a client connection.
        If create is True and the container doesn't exist, it is created.

        Sets username, password and authurl.

        Returns a client connection, metadata tuple or (None, None) on error.
        """

        try:
            values = self.conf.get_container(self.args.container)
        except ValueError as ex:
            self.log.error(ex)
            return (None, None)

        self.authurl = values['authurl']
        self.username = values['username']
        self.password = values['password']

        cli = client.Connection(
            values['authurl'], values['username'], values['password'],
            auth_version=values['auth_version'],
            tenant_name=values['tenant_name'])

        try:
            headers, _ = cli.get_container(self.args.container)
        except (socket.error, client.ClientException) as ex:
            if getattr(ex, 'http_status', None) == 404:
                if create:
                    self.log.warning("%s doesn't exist, will be created" % self.args.container)
                    return (cli, dict())
                else:
                    self.log.error("%s doesn't exist" % self.args.container)
            else:
                self.log.error(ex)
            return (None, None)

        self.log.debug(headers)

        meta = getMeta(headers)
        self.log.debug("Meta: %s" % meta)

        if not meta:
            self.log.error("%s hasn't been setup to be used with swiftnbd" % self.args.container)
            return (None, None)

        return (cli, meta)

    def do_unlock(self):

        self.log.debug("unlocking %s" % self.args.container)

        cli, meta = self._setup_client()
        if cli is None:
            return 1
        elif 'client' not in meta:
            self.log.warning("%s is not locked, nothing to do" % self.args.container)
            return 1

        self.log.info("%s lock is: %s" % (self.args.container, meta['client']))

        meta.update(client='', last=meta['client'])
        hdrs = setMeta(meta)
        self.log.debug("Meta headers: %s" % hdrs)

        try:
            cli.put_container(self.args.container, headers=hdrs)
        except client.ClientException as ex:
            self.log.error(ex)
            return 1

        self.log.info("Done, %s is unlocked" % self.args.container)

        return 0

    def do_lock(self):

        self.log.debug("locking %s" % self.args.container)

        cli, meta = self._setup_client()
        if cli is None:
            return 1
        elif 'client' in meta:
            self.log.warning("%s is already locked: %s" % (self.args.container, meta['client']))
            return 1

        meta.update(client='ctl@%i' % time())
        hdrs = setMeta(meta)
        self.log.debug("Meta headers: %s" % hdrs)

        try:
            cli.put_container(self.args.container, headers=hdrs)
        except client.ClientException as ex:
            self.log.error(ex)
            return 1

        self.log.info("Done, %s is locked" % self.args.container)

        return 0

    def do_download(self):

        self.log.debug("downloading %s" % self.args.container)

        cli, meta = self._setup_client()
        if cli is None:
            return 1
        elif 'client' in meta:
            self.log.error("%s is locked, downloading a container in use is unreliable" % self.args.container)
            return 1

        object_size = int(meta['object-size'])
        objects = int(meta['objects'])

        store = SwiftStorage(self.authurl,
                             self.username,
                             self.password,
                             self.args.container,
                             object_size,
                             objects,
                             )
        try:
            store.lock("ctl-download")
        except StorageError as ex:
            self.log.error(ex)
            return 1

        size = 0
        fdo = None
        try:
            fdo = open(self.args.image, "w")

            while True:
                data = store.read(object_size)
                if data == '':
                    break
                fdo.write(data)
                size += len(data)
                if not self.args.quiet:
                    sys.stdout.write("\rDownloading %s [%.2d%%]" % (self.args.container, 100*size/(objects*object_size)))
                    sys.stdout.flush()
        except IOError as ex:
            self.log.error(ex)
            return 1
        except KeyboardInterrupt:
            self.log.warning("user interrupt")
            return 1
        finally:
            if fdo:
                fdo.close()

            try:
                store.unlock()
            except StorageError as ex:
                self.log.warning("Failed to unlock %s: %s" % (self.args.container, ex))

        if not self.args.quiet:
            sys.stdout.write("\r")
            sys.stdout.flush()

        self.log.info("Done, %s bytes written" % size)

        return 0

    def do_delete(self):

        self.log.debug("deleting %s" % self.args.container)

        cli, meta = self._setup_client()
        if cli is None:
            return 1
        elif 'client' in meta:
            self.log.error("%s is locked" % self.args.container)
            return 1

        # this is the default limit for swift
        limit = 10000
        marker = None
        while True:
            try:
                _, objs = cli.get_container(self.args.container, limit=limit, marker=marker)
            except client.ClientException as ex:
                self.log.error(ex)
                return 1

            for obj in objs:
                if 'name' in obj:
                    try:
                        cli.delete_object(self.args.container, obj['name'])
                    except client.ClientException as ex:
                        self.log.error("Failed to delete %s: %s" % (obj['name'], ex))
                        return 1

            if len(objs) < limit:
                break
            else:
                marker = objs[-1]
                self.log.debug("More than %s files, marker=%s" % (limit, marker))

        try:
            cli.delete_container(self.args.container)
        except client.ClientException as ex:
            self.log.error("Failed to delete %s: %s" % (self.args.container, ex))
            return 1

        self.log.info("Done, %s has been deleted" % self.args.container)

        return 0

    def do_setup(self):

        self.log.debug("setting up %s" % self.args.container)

        cli, meta = self._setup_client(create=True)
        if cli is None:
            return 1
        elif meta and not self.args.force:
            self.log.error("%s has already been setup" % self.args.container)
            return 1

        hdrs = setMeta(dict(version=disk_version, objects=self.args.objects, object_size=self.args.object_size, client='', last=''))
        self.log.debug("Meta headers: %s" % hdrs)

        try:
            cli.put_container(self.args.container, headers=hdrs)
        except client.ClientException as ex:
            self.log.error(ex)
            return 1

        self.log.info("Done, %s" % self.args.container)

        return 0
コード例 #2
0
ファイル: main.py プロジェクト: thurloat/swift-nbd-server
class Main(object):
    def __init__(self):

        self.meta = None
        self.object_size = 0
        self.objects = 0
        self.export_size = 0

        parser = ArgumentParser(description=description,
                                epilog="Contact and support: %s" % project_url
                                )

        parser.add_argument("--version", action="version", version="%(prog)s "  + version)

        parser.add_argument("--secrets", dest="secrets_file",
                            default=secrets_file,
                            help="filename containing user/password (default: %s)" % secrets_file)

        parser.add_argument("-a", "--auth-url", dest="authurl",
                            default=auth_url,
                            help="default authentication URL (default: %s)" % auth_url)

        parser.add_argument("-b", "--bind-address", dest="bind_address",
                            default="127.0.0.1",
                            help="bind address (default: 127.0.0.1)")

        parser.add_argument("-p", "--bind-port", dest="bind_port",
                            type=int,
                            default=10809,
                            help="bind address (default: 10809)")

        parser.add_argument("-c", "--cache-limit", dest="cache_limit",
                            type=int,
                            default=64,
                            help="cache memory limit in MB (default: 64)")

        parser.add_argument("-l", "--log-file", dest="log_file",
                            default=None,
                            help="log into the provided file"
                            )

        parser.add_argument("--syslog", dest="syslog",
                            action="store_true",
                            help="log to system logger (local0)"
                            )

        parser.add_argument("-f", "--foreground", dest="foreground",
                            action="store_true",
                            help="don't detach from terminal (foreground mode)"
                            )

        default_pidfile = os.path.join(tempfile.gettempdir(), "%s.pid" % __package__)
        parser.add_argument("--pid-file", dest="pidfile",
                            default=default_pidfile,
                            help="filename to store the PID (default: %s)" % default_pidfile
                            )

        parser.add_argument("-v", "--verbose", dest="verbose",
                            action="store_true",
                            help="enable verbose logging"
                            )

        self.args = parser.parse_args()

        if self.args.cache_limit < 1:
            parser.error("Cache limit can't be less than 1MB")

        self.log = setLog(debug=self.args.verbose, use_syslog=self.args.syslog, use_file=self.args.log_file)

        try:
            self.conf = Config(self.args.secrets_file, self.args.authurl)
        except OSError as ex:
            parser.error("Failed to load secrets: %s" % ex)

    def run(self):

        if os.path.isfile(self.args.pidfile):
            self.log.error("%s found: is the server already running?" % self.args.pidfile)
            return 1

        stores = dict()
        for container, values in self.conf.iteritems():
            cli = client.Connection(
                values['authurl'], values['username'], values['password'],
                auth_version=values['auth_version'],
                tenant_name=values['tenant_name'])

            try:
                headers, _ = cli.get_container(container)
            except (socket.error, client.ClientException) as ex:
                if getattr(ex, 'http_status', None) == 404:
                    self.log.warning("%s doesn't exist, skipping" % container)
                    continue
                else:
                    self.log.error(ex)
                    return 1

            self.log.debug(headers)

            meta = getMeta(headers)
            if not meta:
                self.log.warning("%s doesn't appear to be setup, skipping" % container)
                continue

            self.log.debug("Meta: %s" % meta)

            try:
                object_size = int(meta['object-size'])
                objects = int(meta['objects'])
            except ValueError as ex:
                self.log.error("%s doesn't appear to be correct: %s" % (container, ex))
                return 1

            if meta['version'] != disk_version:
                self.log.warning("Version mismatch %s != %s in %s" % (meta['version'], disk_version, container))

            stores[container] = SwiftStorage(values['authurl'],
                                             values['username'],
                                             values['password'],
                                             container,
                                             object_size,
                                             objects,
                                             Cache(int(self.args.cache_limit*1024**2 / object_size)),
                                             values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
                                             auth_version=values['auth_version'],
                                             tenant_name=values['tenant_name']
                                            )

        addr = (self.args.bind_address, self.args.bind_port)
        server = Server(addr, stores)

        gevent.signal(signal.SIGTERM, server.stop)
        gevent.signal(signal.SIGINT, server.stop)

        if not self.args.foreground:
            try:
                if gevent.fork() != 0:
                    os._exit(0)
            except OSError as ex:
                self.log.error("Failed to daemonize: %s" % ex)
                return 1

            os.setsid()
            fd = os.open(os.devnull, os.O_RDWR)
            os.dup2(fd, sys.stdin.fileno())
            os.dup2(fd, sys.stdout.fileno())
            os.dup2(fd, sys.stderr.fileno())

            gevent.reinit()

        self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))

        try:
            fd = os.open(self.args.pidfile, (os.O_CREAT|os.O_EXCL|os.O_WRONLY), 0644)
        except OSError as ex:
            self.log.error("Failed to create the pidfile: %s" % ex)
            return 1

        with os.fdopen(fd, "w") as pidfile_handle:
            pidfile_handle.write("%s\n" % os.getpid())
            pidfile_handle.flush()

            server.serve_forever()

        os.remove(self.args.pidfile)

        # unlock the storages before exit
        server.unlock_all()

        self.log.info("Exiting...")
        return 0