def test_load_app(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:healthcheck] use = egg:swift#healthcheck """ def modify_func(app, pipe): new = pipe.create_filter('catch_errors') pipe.insert_filter(new) contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch( 'swift.proxy.server.Application.modify_wsgi_pipeline', modify_func): app = wsgi.loadapp(conf_file, global_conf={}) exp = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assertTrue(isinstance(app, exp), app) exp = swift.common.middleware.healthcheck.HealthCheckMiddleware self.assertTrue(isinstance(app.app, exp), app.app) exp = swift.proxy.server.Application self.assertTrue(isinstance(app.app.app, exp), app.app.app) # make sure you can turn off the pipeline modification if you want def blow_up(*_, **__): raise self.fail("needs more struts") with mock.patch( 'swift.proxy.server.Application.modify_wsgi_pipeline', blow_up): app = wsgi.loadapp(conf_file, global_conf={}, allow_modify_pipeline=False) # the pipeline was untouched exp = swift.common.middleware.healthcheck.HealthCheckMiddleware self.assertTrue(isinstance(app, exp), app) exp = swift.proxy.server.Application self.assertTrue(isinstance(app.app, exp), app.app)
def test_proxy_modify_wsgi_pipeline(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.healthcheck', 'swift.proxy.server'])
def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters { 'name': 'catch_errors' }, # already in pipeline { 'name': 'proxy_logging', 'after_fn': lambda _: ['catch_errors'] }, # not in pipeline, comes after more than one thing { 'name': 'container_quotas', 'after_fn': lambda _: ['catch_errors', 'bulk'] } ] contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, 'required_filters', new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.healthcheck', 'swift.common.middleware.proxy_logging', 'swift.common.middleware.bulk', 'swift.common.middleware.container_quotas', 'swift.common.middleware.tempurl', 'swift.proxy.server' ])
def test_loadapp_from_string(self): conf_body = """ [app:main] use = egg:swift#object """ app = wsgi.loadapp(wsgi.ConfigString(conf_body)) self.assertTrue(isinstance(app, obj_server.ObjectController))
def test_proxy_unmodified_wsgi_pipeline(self): # Make sure things are sane even when we modify nothing config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = catch_errors gatekeeper proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:gatekeeper] use = egg:swift#gatekeeper """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.proxy.server'])
def _proxy_modify_wsgi_pipeline(self, pipe): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = %s [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:catch_errors] use = egg:swift#catch_errors [filter:gatekeeper] use = egg:swift#gatekeeper """ config = config % (pipe,) contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) return app
def __init__(self, conf_path, user_agent, request_tries, allow_modify_pipeline=False): if request_tries < 1: raise ValueError('request_tries must be positive') self.app = loadapp(conf_path, allow_modify_pipeline=allow_modify_pipeline) self.user_agent = user_agent self.request_tries = request_tries
def __init__(self, conf_path, user_agent, request_tries, allow_modify_pipeline=False): self.app = loadapp(conf_path, allow_modify_pipeline=allow_modify_pipeline) self.user_agent = user_agent self.request_tries = request_tries
def __init__(self, conf, cutoff=None): super(LogDelivery, self).__init__(conf) self.logger = utils.get_logger(conf, log_route='log-delivery') self.log_dir = conf.get('log_dir', '/var/log/swift/') conf_path = conf.get('__file__') or '/etc/swift/proxy-server.conf' self.swift = wsgi.loadapp(conf_path, conf) self.swift_user = conf.get('log_user', '.log_delivery') self.interval = int(conf.get('interval', '3600')) self.new_log_cutoff = int(cutoff or conf.get('new_log_cutoff', '7200')) self.owners = {}
def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='swift3-object-expirer') self.interval = int(conf.get('interval') or 300) conf_path = conf.get('__file__') or \ '/etc/swift/swift3-object-expirer.conf' self.swift = wsgi.loadapp(conf_path, conf) self.report_interval = int(conf.get('report_interval') or 300) self.report_first_time = self.report_last_time = time() self.report_objects = 0
def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters {"name": "catch_errors"}, # already in pipeline {"name": "proxy_logging", "after_fn": lambda _: ["catch_errors"]}, # not in pipeline, comes after more than one thing {"name": "container_quotas", "after_fn": lambda _: ["catch_errors", "bulk"]}, ] contents = dedent(config) with temptree(["proxy-server.conf"]) as t: conf_file = os.path.join(t, "proxy-server.conf") with open(conf_file, "w") as f: f.write(contents.replace("TEMPDIR", t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, "required_filters", new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual( self.pipeline_modules(app), [ "swift.common.middleware.catch_errors", "swift.common.middleware.healthcheck", "swift.common.middleware.proxy_logging", "swift.common.middleware.bulk", "swift.common.middleware.container_quotas", "swift.common.middleware.tempurl", "swift.proxy.server", ], )
def test_loadapp_from_file(self, tempdir): conf_path = os.path.join(tempdir, 'object-server.conf') conf_body = """ [app:main] use = egg:swift#object """ contents = dedent(conf_body) with open(conf_path, 'w') as f: f.write(contents) app = wsgi.loadapp(conf_path) self.assertTrue(isinstance(app, obj_server.ObjectController))
def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters {'name': 'catch_errors'}, # already in pipeline {'name': 'proxy_logging', 'after': ['catch_errors']}, # not in pipeline, comes after more than one thing {'name': 'container_quotas', 'after': ['catch_errors', 'bulk']}] contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, 'required_filters', new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.healthcheck', 'swift.common.middleware.proxy_logging', 'swift.common.middleware.bulk', 'swift.common.middleware.container_quotas', 'swift.common.middleware.tempurl', 'swift.proxy.server'])
def _loadapp(self, proxy_config_path): """ Load a proxy from an app.conf to get the memcache_ring :returns: the memcache_ring of the memcache middleware filter """ with mock.patch('swift.proxy.server.Ring'): app = loadapp(proxy_config_path) memcache_ring = None while True: memcache_ring = getattr(app, 'memcache', None) if memcache_ring: break app = app.app return memcache_ring
def test_loadapp_storage(self, tempdir): expectations = { 'object': obj_server.ObjectController, 'container': container_server.ContainerController, 'account': account_server.AccountController, } for server_type, controller in expectations.items(): conf_path = os.path.join(tempdir, '%s-server.conf' % server_type) conf_body = """ [DEFAULT] swift_dir = %s [app:main] use = egg:swift#%s """ % (tempdir, server_type) with open(conf_path, 'w') as f: f.write(dedent(conf_body)) app = wsgi.loadapp(conf_path) self.assertTrue(isinstance(app, controller))
def test_loadapp_storage(self, tempdir): expectations = { 'object': obj_server.ObjectController, 'container': container_server.ContainerController, 'account': account_server.AccountController, } for server_type, controller in expectations.items(): conf_path = os.path.join( tempdir, '%s-server.conf' % server_type) conf_body = """ [DEFAULT] swift_dir = %s [app:main] use = egg:swift#%s """ % (tempdir, server_type) with open(conf_path, 'w') as f: f.write(dedent(conf_body)) app = wsgi.loadapp(conf_path) self.assertTrue(isinstance(app, controller))
def test_loadapp_proxy(self, tempdir): conf_path = os.path.join(tempdir, 'proxy-server.conf') conf_body = """ [DEFAULT] swift_dir = %s [pipeline:main] pipeline = catch_errors cache proxy-server [app:proxy-server] use = egg:swift#proxy [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ % tempdir with open(conf_path, 'w') as f: f.write(dedent(conf_body)) account_ring_path = os.path.join(tempdir, 'account.ring.gz') write_fake_ring(account_ring_path) container_ring_path = os.path.join(tempdir, 'container.ring.gz') write_fake_ring(container_ring_path) object_ring_path = os.path.join(tempdir, 'object.ring.gz') write_fake_ring(object_ring_path) object_1_ring_path = os.path.join(tempdir, 'object-1.ring.gz') write_fake_ring(object_1_ring_path) app = wsgi.loadapp(conf_path) proxy_app = app.app.app.app.app self.assertEqual(proxy_app.account_ring.serialized_path, account_ring_path) self.assertEqual(proxy_app.container_ring.serialized_path, container_ring_path) self.assertEqual(proxy_app.get_object_ring(0).serialized_path, object_ring_path) self.assertEqual(proxy_app.get_object_ring(1).serialized_path, object_1_ring_path)
def test_loadapp_proxy(self, tempdir): conf_path = os.path.join(tempdir, 'proxy-server.conf') conf_body = """ [DEFAULT] swift_dir = %s [pipeline:main] pipeline = catch_errors cache proxy-server [app:proxy-server] use = egg:swift#proxy [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ % tempdir with open(conf_path, 'w') as f: f.write(dedent(conf_body)) _fake_rings(tempdir) account_ring_path = os.path.join(tempdir, 'account.ring.gz') container_ring_path = os.path.join(tempdir, 'container.ring.gz') object_ring_paths = {} for policy in POLICIES: object_ring_paths[int(policy)] = os.path.join( tempdir, policy.ring_name + '.ring.gz') app = wsgi.loadapp(conf_path) proxy_app = app.app.app.app.app self.assertEqual(proxy_app.account_ring.serialized_path, account_ring_path) self.assertEqual(proxy_app.container_ring.serialized_path, container_ring_path) for policy_index, expected_path in object_ring_paths.items(): object_ring = proxy_app.get_object_ring(policy_index) self.assertEqual(expected_path, object_ring.serialized_path)
def in_process_setup(the_object_server=object_server): print >> sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS' print >> sys.stderr, 'Using object_server: %s' % the_object_server.__name__ _dir = os.path.normpath( os.path.join(os.path.abspath(__file__), os.pardir, os.pardir, os.pardir)) proxy_conf = os.path.join(_dir, 'etc', 'proxy-server.conf-sample') if os.path.exists(proxy_conf): print >> sys.stderr, 'Using proxy-server config from %s' % proxy_conf else: print >> sys.stderr, 'Failed to find conf file %s' % proxy_conf return monkey_patch_mimetools() global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) swift_conf = os.path.join(_testdir, "swift.conf") with open(swift_conf, "w") as scfp: scfp.write(functests_swift_conf) global orig_swift_conf_name orig_swift_conf_name = utils.SWIFT_CONF_FILE utils.SWIFT_CONF_FILE = swift_conf constraints.reload_constraints() storage_policy.SWIFT_CONF_FILE = swift_conf storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict( (k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global orig_hash_path_suff_pref orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX utils.validate_hash_conf() # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = eventlet.listen(('localhost', 0)) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_host': '127.0.0.1', 'auth_port': str(prolis.getsockname()[1]), 'auth_ssl': 'no', 'auth_prefix': '/auth/', # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', # Service user and prefix (emulates glance, cinder, etc. user) 'account5': 'test5', 'username5': 'tester5', 'password5': 'testing5', 'service_prefix': 'SERVICE', # For tempauth middleware. Update reseller_prefix 'reseller_prefix': 'AUTH, SERVICE', 'SERVICE_require_group': 'service' }) acc1lis = eventlet.listen(('localhost', 0)) acc2lis = eventlet.listen(('localhost', 0)) con1lis = eventlet.listen(('localhost', 0)) con2lis = eventlet.listen(('localhost', 0)) obj1lis = eventlet.listen(('localhost', 0)) obj2lis = eventlet.listen(('localhost', 0)) global _test_sockets _test_sockets = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump( ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{ 'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1] }, { 'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1] }], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump( ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{ 'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1] }, { 'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1] }], 30), f) object_ring_path = os.path.join(_testdir, 'object.ring.gz') with closing(GzipFile(object_ring_path, 'wb')) as f: pickle.dump( ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{ 'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1] }, { 'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1] }], 30), f) eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_request = lambda *a: None logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') # Redirect logging other messages by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_message = \ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 acc1srv = account_server.AccountController(config, logger=debug_logger('acct1')) acc2srv = account_server.AccountController(config, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( config, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( config, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController(config, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController(config, logger=debug_logger('obj2')) logger = debug_logger('proxy') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): app = loadapp(proxy_conf, global_conf=config) nl = utils.NullLogger() prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) obj1spa = eventlet.spawn(eventlet.wsgi.server, obj1lis, obj1srv, nl) obj2spa = eventlet.spawn(eventlet.wsgi.server, obj2lis, obj2srv, nl) global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, { 'X-Timestamp': ts, 'x-trans-id': act }) resp = conn.getresponse() assert (resp.status == 201) create_account('AUTH_test') create_account('AUTH_test2')
def in_process_setup(the_object_server=object_server): _info("IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS") _info("Using object_server class: %s" % the_object_server.__name__) conf_src_dir = os.environ.get("SWIFT_TEST_IN_PROCESS_CONF_DIR") show_debug_logs = os.environ.get("SWIFT_TEST_DEBUG_LOGS") if conf_src_dir is not None: if not os.path.isdir(conf_src_dir): msg = "Config source %s is not a dir" % conf_src_dir raise InProcessException(msg) _info("Using config source dir: %s" % conf_src_dir) # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then # prefer config files from there, otherwise read config from source tree # sample files. A mixture of files from the two sources is allowed. proxy_conf = _in_process_find_conf_file(conf_src_dir, "proxy-server.conf") _info("Using proxy config from %s" % proxy_conf) swift_conf_src = _in_process_find_conf_file(conf_src_dir, "swift.conf") _info("Using swift config from %s" % swift_conf_src) monkey_patch_mimetools() global _testdir _testdir = os.path.join(mkdtemp(), "tmp_functional") utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, "sda1")) utils.mkdirs(os.path.join(_testdir, "sda1", "tmp")) utils.mkdirs(os.path.join(_testdir, "sdb1")) utils.mkdirs(os.path.join(_testdir, "sdb1", "tmp")) swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) global orig_swift_conf_name orig_swift_conf_name = utils.SWIFT_CONF_FILE utils.SWIFT_CONF_FILE = swift_conf constraints.reload_constraints() storage_policy.SWIFT_CONF_FILE = swift_conf storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global orig_hash_path_suff_pref orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX utils.validate_hash_conf() global _test_socks _test_socks = [] # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = eventlet.listen(("localhost", 0)) _test_socks.append(prolis) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update( { # Values needed by the various in-process swift servers "devices": _testdir, "swift_dir": _testdir, "mount_check": "false", "client_timeout": "4", "allow_account_management": "true", "account_autocreate": "true", "allow_versions": "True", # Below are values used by the functional test framework, as well as # by the various in-process swift servers "auth_host": "127.0.0.1", "auth_port": str(prolis.getsockname()[1]), "auth_ssl": "no", "auth_prefix": "/auth/", # Primary functional test account (needs admin access to the # account) "account": "test", "username": "******", "password": "******", # User on a second account (needs admin access to the account) "account2": "test2", "username2": "tester2", "password2": "testing2", # User on same account as first, but without admin access "username3": "tester3", "password3": "testing3", # Service user and prefix (emulates glance, cinder, etc. user) "account5": "test5", "username5": "tester5", "password5": "testing5", "service_prefix": "SERVICE", # For tempauth middleware. Update reseller_prefix "reseller_prefix": "AUTH, SERVICE", "SERVICE_require_group": "service", } ) acc1lis = eventlet.listen(("localhost", 0)) acc2lis = eventlet.listen(("localhost", 0)) con1lis = eventlet.listen(("localhost", 0)) con2lis = eventlet.listen(("localhost", 0)) _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets account_ring_path = os.path.join(_testdir, "account.ring.gz") with closing(GzipFile(account_ring_path, "wb")) as f: pickle.dump( ring.RingData( [[0, 1, 0, 1], [1, 0, 1, 0]], [ {"id": 0, "zone": 0, "device": "sda1", "ip": "127.0.0.1", "port": acc1lis.getsockname()[1]}, {"id": 1, "zone": 1, "device": "sdb1", "ip": "127.0.0.1", "port": acc2lis.getsockname()[1]}, ], 30, ), f, ) container_ring_path = os.path.join(_testdir, "container.ring.gz") with closing(GzipFile(container_ring_path, "wb")) as f: pickle.dump( ring.RingData( [[0, 1, 0, 1], [1, 0, 1, 0]], [ {"id": 0, "zone": 0, "device": "sda1", "ip": "127.0.0.1", "port": con1lis.getsockname()[1]}, {"id": 1, "zone": 1, "device": "sdb1", "ip": "127.0.0.1", "port": con2lis.getsockname()[1]}, ], 30, ), f, ) eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_request = lambda *a: None logger = utils.get_logger(config, "wsgi-server", log_route="wsgi") # Redirect logging other messages by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_message = lambda s, f, *a: logger.error("ERROR WSGI: " + f % a) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 def get_logger_name(name): if show_debug_logs: return debug_logger(name) else: return None acc1srv = account_server.AccountController(config, logger=get_logger_name("acct1")) acc2srv = account_server.AccountController(config, logger=get_logger_name("acct2")) con1srv = container_server.ContainerController(config, logger=get_logger_name("cont1")) con2srv = container_server.ContainerController(config, logger=get_logger_name("cont2")) objsrvs = [ (obj_sockets[index], the_object_server.ObjectController(config, logger=get_logger_name("obj%d" % (index + 1)))) for index in range(len(obj_sockets)) ] if show_debug_logs: logger = debug_logger("proxy") def get_logger(name, *args, **kwargs): return logger with mock.patch("swift.common.utils.get_logger", get_logger): with mock.patch("swift.common.middleware.memcache.MemcacheMiddleware", FakeMemcacheMiddleware): try: app = loadapp(proxy_conf, global_conf=config) except Exception as e: raise InProcessException(e) nl = utils.NullLogger() global proxy_srv proxy_srv = prolis prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl) for objsrv in objsrvs] global _test_coros _test_coros = (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name="account") partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node["ip"], node["port"], node["device"], partition, "PUT", "/" + act, {"X-Timestamp": ts, "x-trans-id": act}, ) resp = conn.getresponse() assert resp.status == 201 create_account("AUTH_test") create_account("AUTH_test2")
def in_process_setup(the_object_server=object_server): print >>sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS' print >>sys.stderr, 'Using object_server: %s' % the_object_server.__name__ _dir = os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir, os.pardir, os.pardir)) proxy_conf = os.path.join(_dir, 'etc', 'proxy-server.conf-sample') if os.path.exists(proxy_conf): print >>sys.stderr, 'Using proxy-server config from %s' % proxy_conf else: print >>sys.stderr, 'Failed to find conf file %s' % proxy_conf return monkey_patch_mimetools() global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) swift_conf = os.path.join(_testdir, "swift.conf") with open(swift_conf, "w") as scfp: scfp.write(functests_swift_conf) global orig_swift_conf_name orig_swift_conf_name = utils.SWIFT_CONF_FILE utils.SWIFT_CONF_FILE = swift_conf constraints.reload_constraints() storage_policy.SWIFT_CONF_FILE = swift_conf storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global orig_hash_path_suff_pref orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX utils.validate_hash_conf() # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = eventlet.listen(('localhost', 0)) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_host': '127.0.0.1', 'auth_port': str(prolis.getsockname()[1]), 'auth_ssl': 'no', 'auth_prefix': '/auth/', # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', # For tempauth middleware 'user_admin_admin': 'admin .admin .reseller_admin', 'user_test_tester': 'testing .admin', 'user_test2_tester2': 'testing2 .admin', 'user_test_tester3': 'testing3' }) acc1lis = eventlet.listen(('localhost', 0)) acc2lis = eventlet.listen(('localhost', 0)) con1lis = eventlet.listen(('localhost', 0)) con2lis = eventlet.listen(('localhost', 0)) obj1lis = eventlet.listen(('localhost', 0)) obj2lis = eventlet.listen(('localhost', 0)) global _test_sockets _test_sockets = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) object_ring_path = os.path.join(_testdir, 'object.ring.gz') with closing(GzipFile(object_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}], 30), f) eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_request = lambda *a: None logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') # Redirect logging other messages by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_message = \ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 acc1srv = account_server.AccountController( config, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( config, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( config, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( config, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController( config, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( config, logger=debug_logger('obj2')) logger = debug_logger('proxy') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): app = loadapp(proxy_conf, global_conf=config) nl = utils.NullLogger() prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) obj1spa = eventlet.spawn(eventlet.wsgi.server, obj1lis, obj1srv, nl) obj2spa = eventlet.spawn(eventlet.wsgi.server, obj2lis, obj2srv, nl) global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) resp = conn.getresponse() assert(resp.status == 201) create_account('AUTH_test') create_account('AUTH_test2')
def in_process_setup(the_object_server=object_server): _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS') _info('Using object_server class: %s' % the_object_server.__name__) conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR') if conf_src_dir is not None: if not os.path.isdir(conf_src_dir): msg = 'Config source %s is not a dir' % conf_src_dir raise InProcessException(msg) _info('Using config source dir: %s' % conf_src_dir) # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then # prefer config files from there, otherwise read config from source tree # sample files. A mixture of files from the two sources is allowed. proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf') _info('Using proxy config from %s' % proxy_conf) swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf') _info('Using swift config from %s' % swift_conf_src) monkey_patch_mimetools() global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) global orig_swift_conf_name orig_swift_conf_name = utils.SWIFT_CONF_FILE utils.SWIFT_CONF_FILE = swift_conf constraints.reload_constraints() storage_policy.SWIFT_CONF_FILE = swift_conf storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global orig_hash_path_suff_pref orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX utils.validate_hash_conf() # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = eventlet.listen(('localhost', 0)) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_host': '127.0.0.1', 'auth_port': str(prolis.getsockname()[1]), 'auth_ssl': 'no', 'auth_prefix': '/auth/', # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', # Service user and prefix (emulates glance, cinder, etc. user) 'account5': 'test5', 'username5': 'tester5', 'password5': 'testing5', 'service_prefix': 'SERVICE', # For tempauth middleware. Update reseller_prefix 'reseller_prefix': 'AUTH, SERVICE', 'SERVICE_require_group': 'service' }) acc1lis = eventlet.listen(('localhost', 0)) acc2lis = eventlet.listen(('localhost', 0)) con1lis = eventlet.listen(('localhost', 0)) con2lis = eventlet.listen(('localhost', 0)) account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_request = lambda *a: None logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') # Redirect logging other messages by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_message = \ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 acc1srv = account_server.AccountController( config, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( config, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( config, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( config, logger=debug_logger('cont2')) objsrvs = [ (obj_sockets[index], the_object_server.ObjectController( config, logger=debug_logger('obj%d' % (index + 1)))) for index in range(len(obj_sockets)) ] logger = debug_logger('proxy') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): try: app = loadapp(proxy_conf, global_conf=config) except Exception as e: raise InProcessException(e) nl = utils.NullLogger() prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl) for objsrv in objsrvs] global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) resp = conn.getresponse() assert(resp.status == 201) create_account('AUTH_test') create_account('AUTH_test2')
def in_process_setup(the_object_server=object_server): _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS') _info('Using object_server class: %s' % the_object_server.__name__) conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR') show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS') if conf_src_dir is not None: if not os.path.isdir(conf_src_dir): msg = 'Config source %s is not a dir' % conf_src_dir raise InProcessException(msg) _info('Using config source dir: %s' % conf_src_dir) # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then # prefer config files from there, otherwise read config from source tree # sample files. A mixture of files from the two sources is allowed. proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf') _info('Using proxy config from %s' % proxy_conf) swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf') _info('Using swift config from %s' % swift_conf_src) global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdc1')) utils.mkdirs(os.path.join(_testdir, 'sdc1', 'tmp')) swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) _info('prepared swift.conf: %s' % swift_conf) # Call the associated method for the value of # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists conf_loader_label = os.environ.get( 'SWIFT_TEST_IN_PROCESS_CONF_LOADER') if conf_loader_label is not None: try: conf_loader = conf_loaders[conf_loader_label] _debug('Calling method %s mapped to conf loader %s' % (conf_loader.__name__, conf_loader_label)) except KeyError as missing_key: raise InProcessException('No function mapped for conf loader %s' % missing_key) try: # Pass-in proxy_conf, swift_conf files proxy_conf, swift_conf = conf_loader(proxy_conf, swift_conf) _debug('Now using proxy conf %s' % proxy_conf) _debug('Now using swift conf %s' % swift_conf) except Exception as err: # noqa raise InProcessException(err) obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) # load new swift.conf file if set_swift_dir(os.path.dirname(swift_conf)): constraints.reload_constraints() storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global _test_socks _test_socks = [] # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = listen_zero() _test_socks.append(prolis) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', 'allow_versioned_writes': 'True', # TODO: move this into s3api config loader because they are # required by only s3api 'allowed_headers': "Content-Disposition, Content-Encoding, X-Delete-At, " "X-Object-Manifest, X-Static-Large-Object, Cache-Control, " "Content-Language, Expires, X-Robots-Tag", # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_uri': 'http://127.0.0.1:%d/auth/v1.0/' % prolis.getsockname()[1], # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', 's3_access_key': 'test:tester', 's3_secret_key': 'testing', # Secondary user of the primary test account (needs admin access # to the account) for s3api 's3_access_key2': 'test:tester2', 's3_secret_key2': 'testing2', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', 's3_access_key3': 'test:tester3', 's3_secret_key3': 'testing3', # Service user and prefix (emulates glance, cinder, etc. user) 'account5': 'test5', 'username5': 'tester5', 'password5': 'testing5', 'service_prefix': 'SERVICE', # For tempauth middleware. Update reseller_prefix 'reseller_prefix': 'AUTH, SERVICE', 'SERVICE_require_group': 'service', # Reseller admin user (needs reseller_admin_role) 'account6': 'test6', 'username6': 'tester6', 'password6': 'testing6' }) acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 def get_logger_name(name): if show_debug_logs: return debug_logger(name) else: return None acc1srv = account_server.AccountController( config, logger=get_logger_name('acct1')) acc2srv = account_server.AccountController( config, logger=get_logger_name('acct2')) con1srv = container_server.ContainerController( config, logger=get_logger_name('cont1')) con2srv = container_server.ContainerController( config, logger=get_logger_name('cont2')) objsrvs = [ (obj_sockets[index], the_object_server.ObjectController( config, logger=get_logger_name('obj%d' % (index + 1)))) for index in range(len(obj_sockets)) ] if show_debug_logs: logger = get_logger_name('proxy') else: logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): try: app = loadapp(proxy_conf, global_conf=config) except Exception as e: raise InProcessException(e) nl = utils.NullLogger() global proxy_srv proxy_srv = prolis prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl, protocol=SwiftHttpProtocol) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl, protocol=SwiftHttpProtocol) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl, protocol=SwiftHttpProtocol) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl, protocol=SwiftHttpProtocol) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl, protocol=SwiftHttpProtocol) objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl, protocol=SwiftHttpProtocol) for objsrv in objsrvs] global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) resp = conn.getresponse() assert resp.status == 201, 'Unable to create account: %s\n%s' % ( resp.status, resp.read()) create_account('AUTH_test') create_account('AUTH_test2')
def in_process_setup(the_object_server=object_server): _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS') _info('Using object_server class: %s' % the_object_server.__name__) conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR') show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS') if conf_src_dir is not None: if not os.path.isdir(conf_src_dir): msg = 'Config source %s is not a dir' % conf_src_dir raise InProcessException(msg) _info('Using config source dir: %s' % conf_src_dir) # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then # prefer config files from there, otherwise read config from source tree # sample files. A mixture of files from the two sources is allowed. proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf') _info('Using proxy config from %s' % proxy_conf) swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf') _info('Using swift config from %s' % swift_conf_src) monkey_patch_mimetools() global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdc1')) utils.mkdirs(os.path.join(_testdir, 'sdc1', 'tmp')) swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) _info('prepared swift.conf: %s' % swift_conf) # Call the associated method for the value of # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists conf_loader_label = os.environ.get( 'SWIFT_TEST_IN_PROCESS_CONF_LOADER') if conf_loader_label is not None: try: conf_loader = conf_loaders[conf_loader_label] _debug('Calling method %s mapped to conf loader %s' % (conf_loader.__name__, conf_loader_label)) except KeyError as missing_key: raise InProcessException('No function mapped for conf loader %s' % missing_key) try: # Pass-in proxy_conf, swift_conf files proxy_conf, swift_conf = conf_loader(proxy_conf, swift_conf) _debug('Now using proxy conf %s' % proxy_conf) _debug('Now using swift conf %s' % swift_conf) except Exception as err: # noqa raise InProcessException(err) obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) # load new swift.conf file if set_swift_dir(os.path.dirname(swift_conf)): constraints.reload_constraints() storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global _test_socks _test_socks = [] # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = listen_zero() _test_socks.append(prolis) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', 'allow_versioned_writes': 'True', # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_host': '127.0.0.1', 'auth_port': str(prolis.getsockname()[1]), 'auth_ssl': 'no', 'auth_prefix': '/auth/', # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', # Service user and prefix (emulates glance, cinder, etc. user) 'account5': 'test5', 'username5': 'tester5', 'password5': 'testing5', 'service_prefix': 'SERVICE', # For tempauth middleware. Update reseller_prefix 'reseller_prefix': 'AUTH, SERVICE', 'SERVICE_require_group': 'service', # Reseller admin user (needs reseller_admin_role) 'account6': 'test6', 'username6': 'tester6', 'password6': 'testing6' }) # If an env var explicitly specifies the proxy-server object_post_as_copy # option then use its value, otherwise leave default config unchanged. object_post_as_copy = os.environ.get( 'SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY') if object_post_as_copy is not None: object_post_as_copy = config_true_value(object_post_as_copy) config['object_post_as_copy'] = str(object_post_as_copy) _debug('Setting object_post_as_copy to %r' % object_post_as_copy) acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_request = lambda *a: None logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') # Redirect logging other messages by the underlying WSGI software. eventlet.wsgi.HttpProtocol.log_message = \ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 def get_logger_name(name): if show_debug_logs: return debug_logger(name) else: return None acc1srv = account_server.AccountController( config, logger=get_logger_name('acct1')) acc2srv = account_server.AccountController( config, logger=get_logger_name('acct2')) con1srv = container_server.ContainerController( config, logger=get_logger_name('cont1')) con2srv = container_server.ContainerController( config, logger=get_logger_name('cont2')) objsrvs = [ (obj_sockets[index], the_object_server.ObjectController( config, logger=get_logger_name('obj%d' % (index + 1)))) for index in range(len(obj_sockets)) ] if show_debug_logs: logger = debug_logger('proxy') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): try: app = loadapp(proxy_conf, global_conf=config) except Exception as e: raise InProcessException(e) nl = utils.NullLogger() global proxy_srv proxy_srv = prolis prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl) objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl) for objsrv in objsrvs] global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) resp = conn.getresponse() assert resp.status == 201, 'Unable to create account: %s\n%s' % ( resp.status, resp.body) create_account('AUTH_test') create_account('AUTH_test2')