Exemple #1
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to the timezone in which
    # the server first starts running in locations that periodically change
    # timezones.
    os.environ["TZ"] = time.strftime("%z", time.gmtime())

    wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    wsgi.HttpProtocol.log_request = lambda *a: None
    # Redirect logging other messages by the underlying WSGI software.
    wsgi.HttpProtocol.log_message = lambda s, f, *a: logger.error("ERROR WSGI: " + f % a)
    wsgi.WRITE_TIMEOUT = int(conf.get("client_timeout") or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet.patcher.monkey_patch(all=False, socket=True)
    eventlet_debug = config_true_value(conf.get("eventlet_debug", "no"))
    eventlet.debug.hub_exceptions(eventlet_debug)
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, "server"):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {"log_name": log_name}
    app = loadapp(conf["__file__"], global_conf=global_conf)
    max_clients = int(conf.get("max_clients", "1024"))
    pool = RestrictedGreenPool(size=max_clients)
    try:
        wsgi.server(sock, app, NullLogger(), custom_pool=pool)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #2
0
def run_server(conf, logger, sock):
    wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    wsgi.HttpProtocol.log_request = lambda *a: None
    # Redirect logging other messages by the underlying WSGI software.
    wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet.patcher.monkey_patch(all=False, socket=True)
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if hasattr(logger, 'server'):
        log_name = logger.server
    else:
        log_name = logger.name
    app = loadapp(conf['__file__'], global_conf={'log_name': log_name})
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)
    try:
        wsgi.server(sock, app, NullLogger(), custom_pool=pool)
    except socket.error, err:
        if err[0] != errno.EINVAL:
            raise
Exemple #3
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to the timezone in which
    # the server first starts running in locations that periodically change
    # timezones.
    os.environ['TZ'] = time.strftime("%z", time.gmtime())

    wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    wsgi.HttpProtocol.log_request = lambda *a: None
    # Redirect logging other messages by the underlying WSGI software.
    wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet.patcher.monkey_patch(all=False, socket=True)
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)
    try:
        wsgi.server(sock, app, NullLogger(), custom_pool=pool)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #4
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to UTC.
    os.environ['TZ'] = 'UTC+0'
    time.tzset()

    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    wsgi_logger = NullLogger()
    if eventlet_debug:
        # let eventlet.wsgi.server log to stderr
        wsgi_logger = None
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}

    # loadapp方法从proxy-server.conf中加载其中的app,
    # 调用其中的app_factory方法实例化功能类(Application)
    # conf = {"user": "******", "allow_account_management": "true",
    #         "bind_port": "8080", "__name__": "proxy-server",
    #         "__file__": "/etc/swift/proxy-server.conf",
    #         "account_autocreate": "true", "here": "/etc/swift"}
    # global_conf = {"log_name": "proxy-server"}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024')) # dict.get(key, default=None),key不在,则返回默认值
    pool = RestrictedGreenPool(size=max_clients)

    # Select which protocol class to use (normal or one expecting PROXY
    # protocol)
    if config_true_value(conf.get('require_proxy_protocol', 'no')):
        protocol_class = SwiftHttpProxiedProtocol
    else:
        protocol_class = SwiftHttpProtocol

    server_kwargs = {
        'custom_pool': pool,
        'protocol': protocol_class,
        # Disable capitalizing headers in Eventlet. This is necessary for
        # the AWS SDK to work with s3api middleware (it needs an "ETag"
        # header; "Etag" just won't do).
        'capitalize_response_headers': False,
    }
    # server:从提供的服务器套接字启动一个的WSGI服务器处理请求;这个方法将会一直循环;
    # 当服务退出之后,sock对象将会关闭;但是底层描述符将会继续开放;
    try:
        wsgi.server(sock, app, wsgi_logger, **server_kwargs)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    # 等待池子中的所有 greenthreads 完成工作
    pool.waitall()
Exemple #5
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to the timezone in which
    # the server first starts running in locations that periodically change
    # timezones.
    os.environ['TZ'] = time.strftime("%z", time.gmtime())

    wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    wsgi.HttpProtocol.log_request = lambda *a: None
    # Redirect logging other messages by the underlying WSGI software.
    wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    # NOTE(sileht):
    #     monkey-patching thread is required by python-keystoneclient;
    #     monkey-patching select is required by oslo.messaging pika driver
    #         if thread is monkey-patched.
    eventlet.patcher.monkey_patch(all=False,
                                  socket=True,
                                  select=True,
                                  thread=True)
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    wsgi_logger = NullLogger()
    if eventlet_debug:
        # let eventlet.wsgi.server log to stderr
        wsgi_logger = None
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)
    try:
        # Disable capitalizing headers in Eventlet if possible.  This is
        # necessary for the AWS SDK to work with swift3 middleware.
        argspec = inspect.getargspec(wsgi.server)
        if 'capitalize_response_headers' in argspec.args:
            wsgi.server(sock,
                        app,
                        wsgi_logger,
                        custom_pool=pool,
                        capitalize_response_headers=False)
        else:
            wsgi.server(sock, app, wsgi_logger, custom_pool=pool)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #6
0
    def test_run_server_debug(self):
        config = """
        [DEFAULT]
        eventlet_debug = yes
        client_timeout = 30
        max_clients = 1000
        swift_dir = TEMPDIR

        [pipeline:main]
        pipeline = proxy-server

        [app:proxy-server]
        use = egg:swift#proxy
        # while "set" values normally override default
        set client_timeout = 20
        # this section is not in conf during run_server
        set max_clients = 10
        """

        contents = dedent(config)
        with temptree(['proxy-server.conf']) as t:
            conf_file = os.path.join(t, 'proxy-server.conf')
            with open(conf_file, 'w') as f:
                f.write(contents.replace('TEMPDIR', t))
            _fake_rings(t)
            with mock.patch('swift.proxy.server.Application.'
                            'modify_wsgi_pipeline'):
                with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
                    mock_server = _wsgi.server
                    _wsgi.server = lambda *args, **kwargs: mock_server(
                        *args, **kwargs)
                    with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
                        conf = wsgi.appconfig(conf_file)
                        logger = logging.getLogger('test')
                        sock = listen(('localhost', 0))
                        wsgi.run_server(conf, logger, sock)
        self.assertEquals('HTTP/1.0',
                          _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False,
                                                          socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(True)
        self.assertTrue(mock_server.called)
        args, kwargs = mock_server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assertEquals(20, server_app.client_timeout)
        self.assertEqual(server_logger, None)
        self.assert_('custom_pool' in kwargs)
        self.assertEquals(1000, kwargs['custom_pool'].size)
Exemple #7
0
    def test_run_server_debug(self):
        config = """
        [DEFAULT]
        eventlet_debug = yes
        client_timeout = 30
        max_clients = 1000
        swift_dir = TEMPDIR

        [pipeline:main]
        pipeline = proxy-server

        [app:proxy-server]
        use = egg:swift#proxy
        # while "set" values normally override default
        set client_timeout = 20
        # this section is not in conf during run_server
        set max_clients = 10
        """

        contents = dedent(config)
        with temptree(['proxy-server.conf']) as t:
            conf_file = os.path.join(t, 'proxy-server.conf')
            with open(conf_file, 'w') as f:
                f.write(contents.replace('TEMPDIR', t))
            _fake_rings(t)
            with mock.patch('swift.proxy.server.Application.'
                            'modify_wsgi_pipeline'):
                with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
                    mock_server = _wsgi.server
                    _wsgi.server = lambda *args, **kwargs: mock_server(
                        *args, **kwargs)
                    with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
                        conf = wsgi.appconfig(conf_file)
                        logger = logging.getLogger('test')
                        sock = listen(('localhost', 0))
                        wsgi.run_server(conf, logger, sock)
        self.assertEquals('HTTP/1.0',
                          _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False,
                                                          socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(True)
        self.assertTrue(mock_server.called)
        args, kwargs = mock_server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assertEquals(20, server_app.client_timeout)
        self.assertEqual(server_logger, None)
        self.assert_('custom_pool' in kwargs)
        self.assertEquals(1000, kwargs['custom_pool'].size)
Exemple #8
0
    def test_run_server_conf_dir(self):
        config_dir = {
            'proxy-server.conf.d/pipeline.conf':
            """
            [pipeline:main]
            pipeline = proxy-server
            """,
            'proxy-server.conf.d/app.conf':
            """
            [app:proxy-server]
            use = egg:swift#proxy
            """,
            'proxy-server.conf.d/default.conf':
            """
            [DEFAULT]
            eventlet_debug = yes
            client_timeout = 30
            """
        }
        # strip indent from test config contents
        config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
        with temptree(*zip(*config_dir.items())) as conf_root:
            conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
            with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
                f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
            _fake_rings(conf_root)
            with mock.patch('swift.proxy.server.Application.'
                            'modify_wsgi_pipeline'):
                with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
                    with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
                        with mock.patch.dict('os.environ', {'TZ': ''}):
                            with mock.patch('swift.common.wsgi.inspect'):
                                conf = wsgi.appconfig(conf_dir)
                                logger = logging.getLogger('test')
                                sock = listen(('localhost', 0))
                                wsgi.run_server(conf, logger, sock)
                                self.assert_(os.environ['TZ'] is not '')

        self.assertEquals('HTTP/1.0',
                          _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False,
                                                          socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(True)
        _wsgi.server.assert_called()
        args, kwargs = _wsgi.server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assert_(isinstance(server_logger, wsgi.NullLogger))
        self.assert_('custom_pool' in kwargs)
Exemple #9
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to the timezone in which
    # the server first starts running in locations that periodically change
    # timezones.
    os.environ['TZ'] = time.strftime("%z", time.gmtime())

    wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    wsgi.HttpProtocol.log_request = lambda *a: None
    # Redirect logging other messages by the underlying WSGI software.
    wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    # NOTE(sileht):
    #     monkey-patching thread is required by python-keystoneclient;
    #     monkey-patching select is required by oslo.messaging pika driver
    #         if thread is monkey-patched.
    eventlet.patcher.monkey_patch(all=False, socket=True, select=True,
                                  thread=True)
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    wsgi_logger = NullLogger()
    if eventlet_debug:
        # let eventlet.wsgi.server log to stderr
        wsgi_logger = None
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)
    try:
        # Disable capitalizing headers in Eventlet if possible.  This is
        # necessary for the AWS SDK to work with swift3 middleware.
        argspec = inspect.getargspec(wsgi.server)
        if 'capitalize_response_headers' in argspec.args:
            wsgi.server(sock, app, wsgi_logger, custom_pool=pool,
                        capitalize_response_headers=False)
        else:
            wsgi.server(sock, app, wsgi_logger, custom_pool=pool)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #10
0
    def test_run_server_conf_dir(self):
        config_dir = {
            'proxy-server.conf.d/pipeline.conf': """
            [pipeline:main]
            pipeline = proxy-server
            """,
            'proxy-server.conf.d/app.conf': """
            [app:proxy-server]
            use = egg:swift#proxy
            """,
            'proxy-server.conf.d/default.conf': """
            [DEFAULT]
            eventlet_debug = yes
            client_timeout = 30
            """
        }
        # strip indent from test config contents
        config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
        with temptree(*zip(*config_dir.items())) as conf_root:
            conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
            with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
                f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
            _fake_rings(conf_root)
            with mock.patch('swift.proxy.server.Application.'
                            'modify_wsgi_pipeline'):
                with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
                    with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
                        with mock.patch.dict('os.environ', {'TZ': ''}):
                            with mock.patch('swift.common.wsgi.inspect'):
                                conf = wsgi.appconfig(conf_dir)
                                logger = logging.getLogger('test')
                                sock = listen(('localhost', 0))
                                wsgi.run_server(conf, logger, sock)
                                self.assert_(os.environ['TZ'] is not '')

        self.assertEquals('HTTP/1.0',
                          _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False,
                                                          socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(True)
        _wsgi.server.assert_called()
        args, kwargs = _wsgi.server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assert_(isinstance(server_logger, wsgi.NullLogger))
        self.assert_('custom_pool' in kwargs)
Exemple #11
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to UTC.
    os.environ['TZ'] = 'UTC+0'
    time.tzset()

    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    wsgi_logger = NullLogger()
    if eventlet_debug:
        # let eventlet.wsgi.server log to stderr
        wsgi_logger = None
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)

    # Select which protocol class to use (normal or one expecting PROXY
    # protocol)
    if config_true_value(conf.get('require_proxy_protocol', 'no')):
        protocol_class = SwiftHttpProxiedProtocol
    else:
        protocol_class = SwiftHttpProtocol

    server_kwargs = {
        'custom_pool': pool,
        'protocol': protocol_class,
        # Disable capitalizing headers in Eventlet. This is necessary for
        # the AWS SDK to work with s3api middleware (it needs an "ETag"
        # header; "Etag" just won't do).
        'capitalize_response_headers': False,
    }
    try:
        wsgi.server(sock, app, wsgi_logger, **server_kwargs)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #12
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to UTC.
    os.environ['TZ'] = 'UTC+0'
    time.tzset()

    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    wsgi_logger = NullLogger()
    if eventlet_debug:
        # let eventlet.wsgi.server log to stderr
        wsgi_logger = None
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)

    # Select which protocol class to use (normal or one expecting PROXY
    # protocol)
    if config_true_value(conf.get('require_proxy_protocol', 'no')):
        protocol_class = SwiftHttpProxiedProtocol
    else:
        protocol_class = SwiftHttpProtocol

    server_kwargs = {
        'custom_pool': pool,
        'protocol': protocol_class,
        # Disable capitalizing headers in Eventlet. This is necessary for
        # the AWS SDK to work with s3api middleware (it needs an "ETag"
        # header; "Etag" just won't do).
        'capitalize_response_headers': False,
    }
    try:
        wsgi.server(sock, app, wsgi_logger, **server_kwargs)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #13
0
def run_server(conf, logger, sock, global_conf=None):
    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to UTC.
    os.environ['TZ'] = 'UTC+0'
    time.tzset()

    wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    wsgi.HttpProtocol.log_request = lambda *a: None
    # Redirect logging other messages by the underlying WSGI software.
    wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

    eventlet.hubs.use_hub(get_hub())
    eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)
    wsgi_logger = NullLogger()
    if eventlet_debug:
        # let eventlet.wsgi.server log to stderr
        wsgi_logger = None
    # utils.LogAdapter stashes name in server; fallback on unadapted loggers
    if not global_conf:
        if hasattr(logger, 'server'):
            log_name = logger.server
        else:
            log_name = logger.name
        global_conf = {'log_name': log_name}
    app = loadapp(conf['__file__'], global_conf=global_conf)
    max_clients = int(conf.get('max_clients', '1024'))
    pool = RestrictedGreenPool(size=max_clients)
    try:
        # Disable capitalizing headers in Eventlet if possible.  This is
        # necessary for the AWS SDK to work with swift3 middleware.
        argspec = inspect.getargspec(wsgi.server)
        if 'capitalize_response_headers' in argspec.args:
            wsgi.server(sock,
                        app,
                        wsgi_logger,
                        custom_pool=pool,
                        capitalize_response_headers=False)
        else:
            wsgi.server(sock, app, wsgi_logger, custom_pool=pool)
    except socket.error as err:
        if err[0] != errno.EINVAL:
            raise
    pool.waitall()
Exemple #14
0
    def test_run_server(self):
        config = """
        [DEFAULT]
        client_timeout = 30
        max_clients = 1000
        swift_dir = TEMPDIR

        [pipeline:main]
        pipeline = proxy-server

        [app:proxy-server]
        use = egg:swift#proxy
        # while "set" values normally override default
        set client_timeout = 20
        # this section is not in conf during run_server
        set max_clients = 10
        """

        contents = dedent(config)
        with temptree(["proxy-server.conf"]) as t:
            conf_file = os.path.join(t, "proxy-server.conf")
            with open(conf_file, "w") as f:
                f.write(contents.replace("TEMPDIR", t))
            _fake_rings(t)
            with mock.patch("swift.proxy.server.Application." "modify_wsgi_pipeline"):
                with mock.patch("swift.common.wsgi.wsgi") as _wsgi:
                    with mock.patch("swift.common.wsgi.eventlet") as _eventlet:
                        with mock.patch("swift.common.wsgi.inspect"):
                            conf = wsgi.appconfig(conf_file)
                            logger = logging.getLogger("test")
                            sock = listen(("localhost", 0))
                            wsgi.run_server(conf, logger, sock)
        self.assertEquals("HTTP/1.0", _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False, socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(False)
        _wsgi.server.assert_called()
        args, kwargs = _wsgi.server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assertEquals(20, server_app.client_timeout)
        self.assert_(isinstance(server_logger, wsgi.NullLogger))
        self.assert_("custom_pool" in kwargs)
        self.assertEquals(1000, kwargs["custom_pool"].size)
Exemple #15
0
    def test_run_server_conf_dir(self):
        config_dir = {
            "proxy-server.conf.d/pipeline.conf": """
            [pipeline:main]
            pipeline = proxy-server
            """,
            "proxy-server.conf.d/app.conf": """
            [app:proxy-server]
            use = egg:swift#proxy
            """,
            "proxy-server.conf.d/default.conf": """
            [DEFAULT]
            client_timeout = 30
            """,
        }
        # strip indent from test config contents
        config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
        with temptree(*zip(*config_dir.items())) as conf_root:
            conf_dir = os.path.join(conf_root, "proxy-server.conf.d")
            with open(os.path.join(conf_dir, "swift.conf"), "w") as f:
                f.write("[DEFAULT]\nswift_dir = %s" % conf_root)
            _fake_rings(conf_root)
            with mock.patch("swift.proxy.server.Application." "modify_wsgi_pipeline"):
                with mock.patch("swift.common.wsgi.wsgi") as _wsgi:
                    with mock.patch("swift.common.wsgi.eventlet") as _eventlet:
                        with mock.patch.dict("os.environ", {"TZ": ""}):
                            with mock.patch("swift.common.wsgi.inspect"):
                                conf = wsgi.appconfig(conf_dir)
                                logger = logging.getLogger("test")
                                sock = listen(("localhost", 0))
                                wsgi.run_server(conf, logger, sock)
                                self.assert_(os.environ["TZ"] is not "")

        self.assertEquals("HTTP/1.0", _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False, socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(False)
        _wsgi.server.assert_called()
        args, kwargs = _wsgi.server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assert_(isinstance(server_logger, wsgi.NullLogger))
        self.assert_("custom_pool" in kwargs)
Exemple #16
0
    def test_run_daemon(self):
        sample_conf = "[my-daemon]\nuser = %s\n" % getuser()
        with tmpfile(sample_conf) as conf_file, \
                mock.patch('swift.common.daemon.use_hub') as mock_use_hub:
            with mock.patch.dict('os.environ', {'TZ': ''}), \
                    mock.patch('time.tzset') as mock_tzset:
                daemon.run_daemon(MyDaemon, conf_file)
                self.assertTrue(MyDaemon.forever_called)
                self.assertEqual(os.environ['TZ'], 'UTC+0')
                self.assertEqual(mock_tzset.mock_calls, [mock.call()])
                self.assertEqual(mock_use_hub.mock_calls,
                                 [mock.call(utils.get_hub())])
            daemon.run_daemon(MyDaemon, conf_file, once=True)
            self.assertEqual(MyDaemon.once_called, True)

            # test raise in daemon code
            with mock.patch.object(MyDaemon, 'run_once', MyDaemon.run_raise):
                self.assertRaises(OSError,
                                  daemon.run_daemon,
                                  MyDaemon,
                                  conf_file,
                                  once=True)

            # test user quit
            sio = StringIO()
            logger = logging.getLogger('server')
            logger.addHandler(logging.StreamHandler(sio))
            logger = utils.get_logger(None, 'server', log_route='server')
            with mock.patch.object(MyDaemon, 'run_forever', MyDaemon.run_quit):
                daemon.run_daemon(MyDaemon, conf_file, logger=logger)
            self.assertTrue('user quit' in sio.getvalue().lower())

            # test missing section
            sample_conf = "[default]\nuser = %s\n" % getuser()
            with tmpfile(sample_conf) as conf_file:
                self.assertRaisesRegexp(SystemExit, 'Unable to find my-daemon '
                                        'config section in.*',
                                        daemon.run_daemon,
                                        MyDaemon,
                                        conf_file,
                                        once=True)
Exemple #17
0
    def run_server(max_clients):
        wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
        # Turn off logging requests by the underlying WSGI software.
        wsgi.HttpProtocol.log_request = lambda *a: None
        # Redirect logging other messages by the underlying WSGI software.
        wsgi.HttpProtocol.log_message = \
            lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
        wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

        eventlet.hubs.use_hub(get_hub())
        eventlet.patcher.monkey_patch(all=False, socket=True)
        eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
        eventlet.debug.hub_exceptions(eventlet_debug)
        app = loadapp('config:%s' % conf_file,
                      global_conf={'log_name': log_name})
        pool = RestrictedGreenPool(size=max_clients)
        try:
            wsgi.server(sock, app, NullLogger(), custom_pool=pool)
        except socket.error, err:
            if err[0] != errno.EINVAL:
                raise
Exemple #18
0
    def run_server():
        wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
        # Turn off logging requests by the underlying WSGI software.
        wsgi.HttpProtocol.log_request = lambda *a: None
        # Redirect logging other messages by the underlying WSGI software.
        wsgi.HttpProtocol.log_message = \
            lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
        wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)

        eventlet.hubs.use_hub(get_hub())
        eventlet.patcher.monkey_patch(all=False, socket=True)
        eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
        eventlet.debug.hub_exceptions(eventlet_debug)
        app = loadapp('config:%s' % conf_file,
                      global_conf={'log_name': log_name})
        pool = GreenPool(size=1024)
        try:
            wsgi.server(sock, app, NullLogger(), custom_pool=pool)
        except socket.error, err:
            if err[0] != errno.EINVAL:
                raise
Exemple #19
0
    def test_run_server(self):
        config = """
        [DEFAULT]
        eventlet_debug = yes
        client_timeout = 30
        swift_dir = TEMPDIR

        [pipeline:main]
        pipeline = proxy-server

        [app:proxy-server]
        use = egg:swift#proxy
        """

        contents = dedent(config)
        with temptree(['proxy-server.conf']) as t:
            conf_file = os.path.join(t, 'proxy-server.conf')
            with open(conf_file, 'w') as f:
                f.write(contents.replace('TEMPDIR', t))
            _fake_rings(t)
            with patch('swift.common.wsgi.wsgi') as _wsgi:
                with patch('swift.common.wsgi.eventlet') as _eventlet:
                    conf = wsgi.appconfig(conf_file)
                    logger = logging.getLogger('test')
                    sock = listen(('localhost', 0))
                    wsgi.run_server(conf, logger, sock)
        self.assertEquals('HTTP/1.0',
                          _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False,
                                                          socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(True)
        _wsgi.server.assert_called()
        args, kwargs = _wsgi.server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assert_(isinstance(server_logger, wsgi.NullLogger))
        self.assert_('custom_pool' in kwargs)
Exemple #20
0
    def test_run_server(self):
        config = """
        [DEFAULT]
        eventlet_debug = yes
        client_timeout = 30
        swift_dir = TEMPDIR

        [pipeline:main]
        pipeline = proxy-server

        [app:proxy-server]
        use = egg:swift#proxy
        """

        contents = dedent(config)
        with temptree(['proxy-server.conf']) as t:
            conf_file = os.path.join(t, 'proxy-server.conf')
            with open(conf_file, 'w') as f:
                f.write(contents.replace('TEMPDIR', t))
            _fake_rings(t)
            with patch('swift.common.wsgi.wsgi') as _wsgi:
                with patch('swift.common.wsgi.eventlet') as _eventlet:
                    conf = wsgi.appconfig(conf_file)
                    logger = logging.getLogger('test')
                    sock = listen(('localhost', 0))
                    wsgi.run_server(conf, logger, sock)
        self.assertEquals('HTTP/1.0',
                          _wsgi.HttpProtocol.default_request_version)
        self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
        _eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
        _eventlet.patcher.monkey_patch.assert_called_with(all=False,
                                                          socket=True)
        _eventlet.debug.hub_exceptions.assert_called_with(True)
        _wsgi.server.assert_called()
        args, kwargs = _wsgi.server.call_args
        server_sock, server_app, server_logger = args
        self.assertEquals(sock, server_sock)
        self.assert_(isinstance(server_app, swift.proxy.server.Application))
        self.assert_(isinstance(server_logger, wsgi.NullLogger))
        self.assert_('custom_pool' in kwargs)
Exemple #21
0
    def test_run_daemon(self):
        sample_conf = "[my-daemon]\nuser = %s\n" % getuser()
        with tmpfile(sample_conf) as conf_file, \
                mock.patch('swift.common.daemon.use_hub') as mock_use_hub:
            with mock.patch.dict('os.environ', {'TZ': ''}), \
                    mock.patch('time.tzset') as mock_tzset:
                daemon.run_daemon(MyDaemon, conf_file)
                self.assertTrue(MyDaemon.forever_called)
                self.assertEqual(os.environ['TZ'], 'UTC+0')
                self.assertEqual(mock_tzset.mock_calls, [mock.call()])
                self.assertEqual(mock_use_hub.mock_calls,
                                 [mock.call(utils.get_hub())])
            daemon.run_daemon(MyDaemon, conf_file, once=True)
            self.assertEqual(MyDaemon.once_called, True)

            # test raise in daemon code
            with mock.patch.object(MyDaemon, 'run_once', MyDaemon.run_raise):
                self.assertRaises(OSError, daemon.run_daemon, MyDaemon,
                                  conf_file, once=True)

            # test user quit
            sio = StringIO()
            logger = logging.getLogger('server')
            logger.addHandler(logging.StreamHandler(sio))
            logger = utils.get_logger(None, 'server', log_route='server')
            with mock.patch.object(MyDaemon, 'run_forever', MyDaemon.run_quit):
                daemon.run_daemon(MyDaemon, conf_file, logger=logger)
            self.assertTrue('user quit' in sio.getvalue().lower())

            # test missing section
            sample_conf = "[default]\nuser = %s\n" % getuser()
            with tmpfile(sample_conf) as conf_file:
                self.assertRaisesRegexp(SystemExit,
                                        'Unable to find my-daemon '
                                        'config section in.*',
                                        daemon.run_daemon, MyDaemon,
                                        conf_file, once=True)
Exemple #22
0
def main(args):
    parser = argparse.ArgumentParser(
        description='Relink and cleanup objects to increase partition power')
    parser.add_argument('action', choices=['relink', 'cleanup'])
    parser.add_argument(
        'conf_file',
        nargs='?',
        help=('Path to config file with [object-relinker] section'))
    parser.add_argument('--swift-dir',
                        default=None,
                        dest='swift_dir',
                        help='Path to swift directory')
    parser.add_argument(
        '--policy',
        default=[],
        dest='policies',
        action='append',
        type=policy,
        help='Policy to relink; may specify multiple (default: all)')
    parser.add_argument('--devices',
                        default=None,
                        dest='devices',
                        help='Path to swift device directory')
    parser.add_argument('--user',
                        default=None,
                        dest='user',
                        help='Drop privileges to this user before relinking')
    parser.add_argument('--device',
                        default=[],
                        dest='device_list',
                        action='append',
                        help='Device name to relink (default: all)')
    parser.add_argument('--partition',
                        '-p',
                        default=[],
                        dest='partitions',
                        type=non_negative_int,
                        action='append',
                        help='Partition to relink (default: all)')
    parser.add_argument('--skip-mount-check',
                        default=False,
                        help='Don\'t test if disk is mounted',
                        action="store_true",
                        dest='skip_mount_check')
    parser.add_argument('--files-per-second',
                        default=None,
                        type=non_negative_float,
                        dest='files_per_second',
                        help='Used to limit I/O. Zero implies no limit '
                        '(default: no limit).')
    parser.add_argument('--stats-interval',
                        default=None,
                        type=non_negative_float,
                        dest='stats_interval',
                        help='Emit stats to recon roughly every N seconds. '
                        '(default: %d).' % DEFAULT_STATS_INTERVAL)
    parser.add_argument('--workers',
                        default=None,
                        type=auto_or_int,
                        help=('Process devices across N workers '
                              '(default: one worker per device)'))
    parser.add_argument('--logfile',
                        default=None,
                        dest='logfile',
                        help='Set log file name. Ignored if using conf_file.')
    parser.add_argument('--debug',
                        default=False,
                        action='store_true',
                        help='Enable debug mode')

    args = parser.parse_args(args)
    hubs.use_hub(get_hub())
    if args.conf_file:
        conf = readconf(args.conf_file, 'object-relinker')
        if args.debug:
            conf['log_level'] = 'DEBUG'
        user = args.user or conf.get('user')
        if user:
            drop_privileges(user)
        logger = get_logger(conf)
    else:
        conf = {'log_level': 'DEBUG' if args.debug else 'INFO'}
        if args.user:
            # Drop privs before creating log file
            drop_privileges(args.user)
            conf['user'] = args.user
        logging.basicConfig(
            format='%(message)s',
            level=logging.DEBUG if args.debug else logging.INFO,
            filename=args.logfile)
        logger = logging.getLogger()

    conf.update({
        'swift_dir':
        args.swift_dir or conf.get('swift_dir', '/etc/swift'),
        'devices':
        args.devices or conf.get('devices', '/srv/node'),
        'mount_check': (config_true_value(conf.get('mount_check', 'true'))
                        and not args.skip_mount_check),
        'files_per_second':
        (args.files_per_second if args.files_per_second is not None else
         non_negative_float(conf.get('files_per_second', '0'))),
        'policies':
        set(args.policies) or POLICIES,
        'partitions':
        set(args.partitions),
        'workers':
        config_auto_int_value(
            conf.get('workers') if args.workers is None else args.workers,
            'auto'),
        'recon_cache_path':
        conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH),
        'stats_interval':
        non_negative_float(
            args.stats_interval
            or conf.get('stats_interval', DEFAULT_STATS_INTERVAL)),
    })
    return parallel_process(args.action == 'cleanup', conf, logger,
                            args.device_list)
Exemple #23
0
from swift.common.ring.utils import is_local_device
from swift.common.utils import whataremyips, unlink_older_than, \
    compute_eta, get_logger, dump_recon_cache, ismount, \
    rsync_module_interpolation, mkdirs, config_true_value, list_from_csv, \
    get_hub, tpool_reraise, config_auto_int_value, storage_directory
from swift.common.bufferedhttp import http_connect
from swift.common.daemon import Daemon
from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
from swift.obj import ssync_sender
from swift.obj.diskfile import DiskFileManager, get_data_dir, get_tmp_dir
from swift.common.storage_policy import POLICIES, REPL_POLICY

DEFAULT_RSYNC_TIMEOUT = 900

hubs.use_hub(get_hub())


def _do_listdir(partition, replication_cycle):
    return (((partition + replication_cycle) % 10) == 0)


class ObjectReplicator(Daemon):
    """
    Replicate objects.

    Encapsulates most logic and data needed by the object replication process.
    Each call to .replicate() performs one replication pass.  It's up to the
    caller to do this in a loop.
    """
    def __init__(self, conf, logger=None):
Exemple #24
0
from swift.common.utils import config_true_value, split_path
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server, mem_server as mem_object_server
import swift.proxy.controllers.obj

http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT
DEBUG = True

# In order to get the proper blocking behavior of sockets without using
# threads, where we can set an arbitrary timeout for some piece of code under
# test, we use eventlet with the standard socket library patched. We have to
# perform this setup at module import time, since all the socket module
# bindings in the swiftclient code will have been made by the time nose
# invokes the package or class setup methods.
eventlet.hubs.use_hub(utils.get_hub())
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet.debug.hub_exceptions(False)

from swiftclient import get_auth, http_connection

has_insecure = False
try:
    from swiftclient import __version__ as client_version
    # Prevent a ValueError in StrictVersion with '2.0.3.68.ga99c2ff'
    client_version = '.'.join(client_version.split('.')[:3])
except ImportError:
    # Pre-PBR we had version, not __version__. Anyhow...
    client_version = '1.2'
from distutils.version import StrictVersion
if StrictVersion(client_version) >= StrictVersion('2.0'):
Exemple #25
0
from eventlet.support.greenlets import GreenletExit

from swift.common.ring.utils import is_local_device
from swift.common.utils import whataremyips, unlink_older_than, \
    compute_eta, get_logger, dump_recon_cache, ismount, \
    rsync_module_interpolation, mkdirs, config_true_value, list_from_csv, \
    get_hub, tpool_reraise, config_auto_int_value, storage_directory
from swift.common.bufferedhttp import http_connect
from swift.common.daemon import Daemon
from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
from swift.obj import ssync_sender
from swift.obj.diskfile import DiskFileManager, get_data_dir, get_tmp_dir
from swift.common.storage_policy import POLICIES, REPL_POLICY


hubs.use_hub(get_hub())


class ObjectReplicator(Daemon):
    """
    Replicate objects.

    Encapsulates most logic and data needed by the object replication process.
    Each call to .replicate() performs one replication pass.  It's up to the
    caller to do this in a loop.
    """

    def __init__(self, conf, logger=None):
        """
        :param conf: configuration object obtained from ConfigParser
        :param logger: logging object
Exemple #26
0
def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
    """
    Loads settings from conf, then instantiates daemon ``klass`` and runs the
    daemon with the specified ``once`` kwarg.  The section_name will be derived
    from the daemon ``klass`` if not provided (e.g. ObjectReplicator =>
    object-replicator).

    :param klass: Class to instantiate, subclass of :class:`Daemon`
    :param conf_file: Path to configuration file
    :param section_name: Section name from conf file to load config from
    :param once: Passed to daemon :meth:`Daemon.run` method
    """
    # very often the config section_name is based on the class name
    # the None singleton will be passed through to readconf as is
    if section_name == '':
        section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
                           klass.__name__).lower()
    try:
        conf = utils.readconf(conf_file, section_name,
                              log_name=kwargs.get('log_name'))
    except (ValueError, IOError) as e:
        # The message will be printed to stderr
        # and results in an exit code of 1.
        sys.exit(e)

    use_hub(utils.get_hub())

    # once on command line (i.e. daemonize=false) will over-ride config
    once = once or not utils.config_true_value(conf.get('daemonize', 'true'))

    # pre-configure logger
    if 'logger' in kwargs:
        logger = kwargs.pop('logger')
    else:
        logger = utils.get_logger(conf, conf.get('log_name', section_name),
                                  log_to_console=kwargs.pop('verbose', False),
                                  log_route=section_name)

    # optional nice/ionice priority scheduling
    utils.modify_priority(conf, logger)

    # disable fallocate if desired
    if utils.config_true_value(conf.get('disable_fallocate', 'no')):
        utils.disable_fallocate()
    # set utils.FALLOCATE_RESERVE if desired
    utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
        utils.config_fallocate_value(conf.get('fallocate_reserve', '1%'))

    # By default, disable eventlet printing stacktraces
    eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)

    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to UTC.
    os.environ['TZ'] = 'UTC+0'
    time.tzset()

    logger.notice('Starting %s', os.getpid())
    try:
        DaemonStrategy(klass(conf), logger).run(once=once, **kwargs)
    except KeyboardInterrupt:
        logger.info('User quit')
    logger.notice('Exited %s', os.getpid())
Exemple #27
0
from swift.common.utils import config_true_value, split_path
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server, mem_server as mem_object_server
import swift.proxy.controllers.obj

http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT
DEBUG = True

# In order to get the proper blocking behavior of sockets without using
# threads, where we can set an arbitrary timeout for some piece of code under
# test, we use eventlet with the standard socket library patched. We have to
# perform this setup at module import time, since all the socket module
# bindings in the swiftclient code will have been made by the time nose
# invokes the package or class setup methods.
eventlet.hubs.use_hub(utils.get_hub())
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet.debug.hub_exceptions(False)

from swiftclient import get_auth, http_connection

has_insecure = False
try:
    from swiftclient import __version__ as client_version
    # Prevent a ValueError in StrictVersion with '2.0.3.68.ga99c2ff'
    client_version = '.'.join(client_version.split('.')[:3])
except ImportError:
    # Pre-PBR we had version, not __version__. Anyhow...
    client_version = '1.2'
from distutils.version import StrictVersion
if StrictVersion(client_version) >= StrictVersion('2.0'):
Exemple #28
0
def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
    """
    Loads settings from conf, then instantiates daemon ``klass`` and runs the
    daemon with the specified ``once`` kwarg.  The section_name will be derived
    from the daemon ``klass`` if not provided (e.g. ObjectReplicator =>
    object-replicator).

    :param klass: Class to instantiate, subclass of :class:`Daemon`
    :param conf_file: Path to configuration file
    :param section_name: Section name from conf file to load config from
    :param once: Passed to daemon :meth:`Daemon.run` method
    """
    # very often the config section_name is based on the class name
    # the None singleton will be passed through to readconf as is
    if section_name is '':
        section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
                           klass.__name__).lower()
    try:
        conf = utils.readconf(conf_file, section_name,
                              log_name=kwargs.get('log_name'))
    except (ValueError, IOError) as e:
        # The message will be printed to stderr
        # and results in an exit code of 1.
        sys.exit(e)

    use_hub(utils.get_hub())

    # once on command line (i.e. daemonize=false) will over-ride config
    once = once or not utils.config_true_value(conf.get('daemonize', 'true'))

    # pre-configure logger
    if 'logger' in kwargs:
        logger = kwargs.pop('logger')
    else:
        logger = utils.get_logger(conf, conf.get('log_name', section_name),
                                  log_to_console=kwargs.pop('verbose', False),
                                  log_route=section_name)

    # optional nice/ionice priority scheduling
    utils.modify_priority(conf, logger)

    # disable fallocate if desired
    if utils.config_true_value(conf.get('disable_fallocate', 'no')):
        utils.disable_fallocate()
    # set utils.FALLOCATE_RESERVE if desired
    utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
        utils.config_fallocate_value(conf.get('fallocate_reserve', '1%'))

    # By default, disable eventlet printing stacktraces
    eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no'))
    eventlet.debug.hub_exceptions(eventlet_debug)

    # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on
    # some platforms. This locks in reported times to UTC.
    os.environ['TZ'] = 'UTC+0'
    time.tzset()

    logger.notice('Starting %s', os.getpid())
    try:
        DaemonStrategy(klass(conf), logger).run(once=once, **kwargs)
    except KeyboardInterrupt:
        logger.info('User quit')
    logger.notice('Exited %s', os.getpid())