예제 #1
0
 def test_reload_invalid_storage_policies(self):
     conf = self._conf("""
     [storage-policy:0]
     name = zero
     [storage-policy:00]
     name = double-zero
     """)
     with NamedTemporaryFile() as f:
         conf.write(f)
         f.flush()
         with mock.patch('swift.common.storage_policy.SWIFT_CONF_FILE',
                         new=f.name):
             try:
                 reload_storage_policies()
             except SystemExit as e:
                 err_msg = str(e)
             else:
                 self.fail('SystemExit not raised')
     parts = [
         'Invalid Storage Policy Configuration',
         'Duplicate index',
     ]
     for expected in parts:
         self.assert_(expected in err_msg,
                      '%s was not in %s' % (expected, err_msg))
예제 #2
0
 def test_reload_invalid_storage_policies(self):
     conf = self._conf("""
     [storage-policy:0]
     name = zero
     [storage-policy:00]
     name = double-zero
     """)
     with NamedTemporaryFile() as f:
         conf.write(f)
         f.flush()
         with mock.patch('swift.common.storage_policy.SWIFT_CONF_FILE',
                         new=f.name):
             try:
                 reload_storage_policies()
             except SystemExit as e:
                 err_msg = str(e)
             else:
                 self.fail('SystemExit not raised')
     parts = [
         'Invalid Storage Policy Configuration',
         'Duplicate index',
     ]
     for expected in parts:
         self.assertTrue(
             expected in err_msg, '%s was not in %s' % (expected,
                                                        err_msg))
예제 #3
0
 def tearDown(self):
     shutil.rmtree(self.testdir, ignore_errors=1)
     storage_policy.reload_storage_policies()
예제 #4
0
파일: __init__.py 프로젝트: mahak/swift
def in_process_setup(the_object_server=object_server):
    _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS')
    _info('Using object_server class: %s' % the_object_server.__name__)
    conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR')
    show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS')

    if conf_src_dir is not None:
        if not os.path.isdir(conf_src_dir):
            msg = 'Config source %s is not a dir' % conf_src_dir
            raise InProcessException(msg)
        _info('Using config source dir: %s' % conf_src_dir)

    # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then
    # prefer config files from there, otherwise read config from source tree
    # sample files. A mixture of files from the two sources is allowed.
    proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf')
    _info('Using proxy config from %s' % proxy_conf)
    swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
    _info('Using swift config from %s' % swift_conf_src)

    global _testdir
    _testdir = os.path.join(mkdtemp(), 'tmp_functional')
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, 'sda1'))
    utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdc1'))
    utils.mkdirs(os.path.join(_testdir, 'sdc1', 'tmp'))

    swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
    _info('prepared swift.conf: %s' % swift_conf)

    # Call the associated method for the value of
    # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists
    conf_loader_label = os.environ.get(
        'SWIFT_TEST_IN_PROCESS_CONF_LOADER')
    if conf_loader_label is not None:
        try:
            conf_loader = conf_loaders[conf_loader_label]
            _debug('Calling method %s mapped to conf loader %s' %
                   (conf_loader.__name__, conf_loader_label))
        except KeyError as missing_key:
            raise InProcessException('No function mapped for conf loader %s' %
                                     missing_key)

        try:
            # Pass-in proxy_conf, swift_conf files
            proxy_conf, swift_conf = conf_loader(proxy_conf, swift_conf)
            _debug('Now using proxy conf %s' % proxy_conf)
            _debug('Now using swift conf %s' % swift_conf)
        except Exception as err:  # noqa
            raise InProcessException(err)

    obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)

    # load new swift.conf file
    if set_swift_dir(os.path.dirname(swift_conf)):
        constraints.reload_constraints()
        storage_policy.reload_storage_policies()

    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        _c = dict((k, str(v))
                  for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
        config.update(_c)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest

    global _test_socks
    _test_socks = []
    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = listen_zero()
    _test_socks.append(prolis)

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update({
        # Values needed by the various in-process swift servers
        'devices': _testdir,
        'swift_dir': _testdir,
        'mount_check': 'false',
        'client_timeout': '4',
        'allow_account_management': 'true',
        'account_autocreate': 'true',
        'allow_versions': 'True',
        'allow_versioned_writes': 'True',
        # TODO: move this into s3api config loader because they are
        #       required by only s3api
        'allowed_headers':
            "Content-Disposition, Content-Encoding, X-Delete-At, "
            "X-Object-Manifest, X-Static-Large-Object, Cache-Control, "
            "Content-Language, Expires, X-Robots-Tag",
        # Below are values used by the functional test framework, as well as
        # by the various in-process swift servers
        'auth_uri': 'http://127.0.0.1:%d/auth/v1.0/' % prolis.getsockname()[1],
        # Primary functional test account (needs admin access to the
        # account)
        'account': 'test',
        'username': '******',
        'password': '******',
        's3_access_key': 'test:tester',
        's3_secret_key': 'testing',
        # Secondary user of the primary test account (needs admin access
        # to the account) for s3api
        's3_access_key2': 'test:tester2',
        's3_secret_key2': 'testing2',
        # User on a second account (needs admin access to the account)
        'account2': 'test2',
        'username2': 'tester2',
        'password2': 'testing2',
        # User on same account as first, but without admin access
        'username3': 'tester3',
        'password3': 'testing3',
        's3_access_key3': 'test:tester3',
        's3_secret_key3': 'testing3',
        # Service user and prefix (emulates glance, cinder, etc. user)
        'account5': 'test5',
        'username5': 'tester5',
        'password5': 'testing5',
        'service_prefix': 'SERVICE',
        # For tempauth middleware. Update reseller_prefix
        'reseller_prefix': 'AUTH, SERVICE',
        'SERVICE_require_group': 'service',
        # Reseller admin user (needs reseller_admin_role)
        'account6': 'test6',
        'username6': 'tester6',
        'password6': 'testing6'
    })

    acc1lis = listen_zero()
    acc2lis = listen_zero()
    con1lis = listen_zero()
    con2lis = listen_zero()
    _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets

    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    with closing(GzipFile(account_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': acc1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': acc2lis.getsockname()[1]}], 30),
                    f)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    with closing(GzipFile(container_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': con1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': con2lis.getsockname()[1]}], 30),
                    f)

    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    def get_logger_name(name):
        if show_debug_logs:
            return debug_logger(name)
        else:
            return None

    acc1srv = account_server.AccountController(
        config, logger=get_logger_name('acct1'))
    acc2srv = account_server.AccountController(
        config, logger=get_logger_name('acct2'))
    con1srv = container_server.ContainerController(
        config, logger=get_logger_name('cont1'))
    con2srv = container_server.ContainerController(
        config, logger=get_logger_name('cont2'))

    objsrvs = [
        (obj_sockets[index],
         the_object_server.ObjectController(
             config, logger=get_logger_name('obj%d' % (index + 1))))
        for index in range(len(obj_sockets))
    ]

    if show_debug_logs:
        logger = get_logger_name('proxy')
    else:
        logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')

    def get_logger(name, *args, **kwargs):
        return logger

    with mock.patch('swift.common.utils.get_logger', get_logger):
        with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
                        FakeMemcacheMiddleware):
            try:
                app = loadapp(proxy_conf, global_conf=config)
            except Exception as e:
                raise InProcessException(e)

    nl = utils.NullLogger()
    global proxy_srv
    proxy_srv = prolis
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl,
                            protocol=SwiftHttpProtocol)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl,
                             protocol=SwiftHttpProtocol)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl,
                             protocol=SwiftHttpProtocol)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl,
                             protocol=SwiftHttpProtocol)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl,
                             protocol=SwiftHttpProtocol)

    objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl,
                             protocol=SwiftHttpProtocol)
              for objsrv in objsrvs]

    global _test_coros
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        account_ring = Ring(_testdir, ring_name='account')
        partition, nodes = account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node['ip'], node['port'], node['device'], partition, 'PUT',
                '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
            resp = conn.getresponse()
            assert resp.status == 201, 'Unable to create account: %s\n%s' % (
                resp.status, resp.read())

    create_account('AUTH_test')
    create_account('AUTH_test2')
예제 #5
0
    def main(self):
        """
        Retrieve and report cluster info from hosts running recon middleware.
        """
        print("=" * 79)
        usage = '''
        usage: %prog <server_type> [<server_type> [<server_type>]]
        [-v] [--suppress] [-a] [-r] [-u] [-d]
        [-l] [-T] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
        [--human-readable]

        <server_type>\taccount|container|object
        Defaults to object server.

        ex: %prog container -l --auditor
        '''
        args = optparse.OptionParser(usage)
        args.add_option('--verbose',
                        '-v',
                        action="store_true",
                        help="Print verbose info")
        args.add_option('--suppress',
                        action="store_true",
                        help="Suppress most connection related errors")
        args.add_option('--async',
                        '-a',
                        action="store_true",
                        help="Get async stats")
        args.add_option('--replication',
                        '-r',
                        action="store_true",
                        help="Get replication stats")
        args.add_option('--auditor',
                        action="store_true",
                        help="Get auditor stats")
        args.add_option('--updater',
                        action="store_true",
                        help="Get updater stats")
        args.add_option('--expirer',
                        action="store_true",
                        help="Get expirer stats")
        args.add_option('--unmounted',
                        '-u',
                        action="store_true",
                        help="Check cluster for unmounted devices")
        args.add_option('--diskusage',
                        '-d',
                        action="store_true",
                        help="Get disk usage stats")
        args.add_option('--human-readable',
                        action="store_true",
                        help="Use human readable suffix for disk usage stats")
        args.add_option('--loadstats',
                        '-l',
                        action="store_true",
                        help="Get cluster load average stats")
        args.add_option('--quarantined',
                        '-q',
                        action="store_true",
                        help="Get cluster quarantine stats")
        args.add_option('--validate-servers',
                        action="store_true",
                        help="Validate servers on the ring")
        args.add_option('--md5',
                        action="store_true",
                        help="Get md5sum of servers ring and compare to "
                        "local copy")
        args.add_option('--sockstat',
                        action="store_true",
                        help="Get cluster socket usage stats")
        args.add_option('--driveaudit',
                        action="store_true",
                        help="Get drive audit error stats")
        args.add_option('--time',
                        '-T',
                        action="store_true",
                        help="Check time synchronization")
        args.add_option('--jitter',
                        type="float",
                        default=0.0,
                        help="Maximal allowed time jitter")
        args.add_option('--swift-versions',
                        action="store_true",
                        help="Check swift versions")
        args.add_option('--top',
                        type='int',
                        metavar='COUNT',
                        default=0,
                        help='Also show the top COUNT entries in rank order.')
        args.add_option('--lowest',
                        type='int',
                        metavar='COUNT',
                        default=0,
                        help='Also show the lowest COUNT entries in rank \
                        order.')
        args.add_option('--all',
                        action="store_true",
                        help="Perform all checks. Equal to \t\t\t-arudlqT "
                        "--md5 --sockstat --auditor --updater --expirer "
                        "--driveaudit --validate-servers --swift-versions")
        args.add_option('--region',
                        type="int",
                        help="Only query servers in specified region")
        args.add_option('--zone',
                        '-z',
                        type="int",
                        help="Only query servers in specified zone")
        args.add_option('--timeout',
                        '-t',
                        type="int",
                        metavar="SECONDS",
                        help="Time to wait for a response from a server",
                        default=5)
        args.add_option('--swiftdir',
                        default="/etc/swift",
                        help="Default = /etc/swift")
        args.add_option('--policy',
                        '-p',
                        help='Only query object servers in specified '
                        'storage policy (specified as name or index).')
        options, arguments = args.parse_args()

        if len(sys.argv) <= 1 or len(arguments) > len(self.check_types):
            args.print_help()
            sys.exit(0)

        if arguments:
            arguments = set(arguments)
            if arguments.issubset(self.check_types):
                server_types = arguments
            else:
                print("Invalid Server Type")
                args.print_help()
                sys.exit(1)
        else:  # default
            server_types = ['object']

        swift_dir = options.swiftdir
        if set_swift_dir(swift_dir):
            reload_storage_policies()

        self.verbose = options.verbose
        self.suppress_errors = options.suppress
        self.timeout = options.timeout

        for server_type in server_types:
            self.server_type = server_type
            ring_names = self._get_ring_names(options.policy)
            if not ring_names:
                print('Invalid Storage Policy: %s' % options.policy)
                args.print_help()
                sys.exit(0)
            hosts = self.get_hosts(options.region, options.zone, swift_dir,
                                   ring_names)
            print("--> Starting reconnaissance on %s hosts (%s)" %
                  (len(hosts), self.server_type))
            print("=" * 79)
            if options.all:
                if self.server_type == 'object':
                    self.async_check(hosts)
                    self.object_auditor_check(hosts)
                    self.updater_check(hosts)
                    self.expirer_check(hosts)
                elif self.server_type == 'container':
                    self.auditor_check(hosts)
                    self.updater_check(hosts)
                elif self.server_type == 'account':
                    self.auditor_check(hosts)
                self.replication_check(hosts)
                self.umount_check(hosts)
                self.load_check(hosts)
                self.disk_usage(hosts, options.top, options.lowest,
                                options.human_readable)
                self.get_ringmd5(hosts, swift_dir)
                self.get_swiftconfmd5(hosts)
                self.quarantine_check(hosts)
                self.socket_usage(hosts)
                self.server_type_check(hosts)
                self.driveaudit_check(hosts)
                self.time_check(hosts, options.jitter)
                self.version_check(hosts)
            else:
                if options. async:
                    if self.server_type == 'object':
                        self.async_check(hosts)
                    else:
                        print("Error: Can't check asyncs on non object "
                              "servers.")
                        print("=" * 79)
                if options.unmounted:
                    self.umount_check(hosts)
                if options.replication:
                    self.replication_check(hosts)
                if options.auditor:
                    if self.server_type == 'object':
                        self.object_auditor_check(hosts)
                    else:
                        self.auditor_check(hosts)
                if options.updater:
                    if self.server_type == 'account':
                        print("Error: Can't check updaters on account "
                              "servers.")
                        print("=" * 79)
                    else:
                        self.updater_check(hosts)
                if options.expirer:
                    if self.server_type == 'object':
                        self.expirer_check(hosts)
                    else:
                        print("Error: Can't check expired on non object "
                              "servers.")
                        print("=" * 79)
                if options.validate_servers:
                    self.server_type_check(hosts)
                if options.loadstats:
                    self.load_check(hosts)
                if options.diskusage:
                    self.disk_usage(hosts, options.top, options.lowest,
                                    options.human_readable)
                if options.md5:
                    self.get_ringmd5(hosts, swift_dir)
                    self.get_swiftconfmd5(hosts)
                if options.quarantined:
                    self.quarantine_check(hosts)
                if options.sockstat:
                    self.socket_usage(hosts)
                if options.driveaudit:
                    self.driveaudit_check(hosts)
                if options.time:
                    self.time_check(hosts, options.jitter)
                if options.swift_versions:
                    self.version_check(hosts)
예제 #6
0
def in_process_setup(the_object_server=object_server):
    print >>sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS'
    print >>sys.stderr, 'Using object_server: %s' % the_object_server.__name__

    monkey_patch_mimetools()

    global _testdir
    _testdir = os.path.join(mkdtemp(), 'tmp_functional')
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, 'sda1'))
    utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))

    swift_conf = os.path.join(_testdir, "swift.conf")
    with open(swift_conf, "w") as scfp:
        scfp.write(functests_swift_conf)

    global orig_swift_conf_name
    orig_swift_conf_name = utils.SWIFT_CONF_FILE
    utils.SWIFT_CONF_FILE = swift_conf
    constraints.reload_constraints()
    storage_policy.SWIFT_CONF_FILE = swift_conf
    storage_policy.reload_storage_policies()
    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        config.update(constraints.EFFECTIVE_CONSTRAINTS)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest
    global orig_hash_path_suff_pref
    orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
    utils.validate_hash_conf()

    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = eventlet.listen(('localhost', 0))

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update({
        # Values needed by the various in-process swift servers
        'devices': _testdir,
        'swift_dir': _testdir,
        'mount_check': 'false',
        'client_timeout': 4,
        'allow_account_management': 'true',
        'account_autocreate': 'true',
        'allowed_headers':
        'content-disposition, content-encoding, x-delete-at,'
        ' x-object-manifest, x-static-large-object',
        'allow_versions': 'True',
        # Below are values used by the functional test framework, as well as
        # by the various in-process swift servers
        'auth_host': '127.0.0.1',
        'auth_port': str(prolis.getsockname()[1]),
        'auth_ssl': 'no',
        'auth_prefix': '/auth/',
        # Primary functional test account (needs admin access to the
        # account)
        'account': 'test',
        'username': '******',
        'password': '******',
        # User on a second account (needs admin access to the account)
        'account2': 'test2',
        'username2': 'tester2',
        'password2': 'testing2',
        # User on same account as first, but without admin access
        'username3': 'tester3',
        'password3': 'testing3',
        # For tempauth middleware
        'user_admin_admin': 'admin .admin .reseller_admin',
        'user_test_tester': 'testing .admin',
        'user_test2_tester2': 'testing2 .admin',
        'user_test_tester3': 'testing3'
    })

    acc1lis = eventlet.listen(('localhost', 0))
    acc2lis = eventlet.listen(('localhost', 0))
    con1lis = eventlet.listen(('localhost', 0))
    con2lis = eventlet.listen(('localhost', 0))
    obj1lis = eventlet.listen(('localhost', 0))
    obj2lis = eventlet.listen(('localhost', 0))
    global _test_sockets
    _test_sockets = \
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis)

    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    with closing(GzipFile(account_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': acc1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': acc2lis.getsockname()[1]}], 30),
                    f)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    with closing(GzipFile(container_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': con1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': con2lis.getsockname()[1]}], 30),
                    f)
    object_ring_path = os.path.join(_testdir, 'object.ring.gz')
    with closing(GzipFile(object_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': obj1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': obj2lis.getsockname()[1]}], 30),
                    f)

    eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
    logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')
    # Redirect logging other messages by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    prosrv = proxy_server.Application(config, logger=debug_logger('proxy'))
    acc1srv = account_server.AccountController(
        config, logger=debug_logger('acct1'))
    acc2srv = account_server.AccountController(
        config, logger=debug_logger('acct2'))
    con1srv = container_server.ContainerController(
        config, logger=debug_logger('cont1'))
    con2srv = container_server.ContainerController(
        config, logger=debug_logger('cont2'))
    obj1srv = the_object_server.ObjectController(
        config, logger=debug_logger('obj1'))
    obj2srv = the_object_server.ObjectController(
        config, logger=debug_logger('obj2'))
    global _test_servers
    _test_servers = \
        (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv)

    pipeline = [
        catch_errors.filter_factory,
        gatekeeper.filter_factory,
        healthcheck.filter_factory,
        proxy_logging.filter_factory,
        fake_memcache_filter_factory,
        container_sync.filter_factory,
        bulk.filter_factory,
        tempurl.filter_factory,
        slo.filter_factory,
        dlo.filter_factory,
        ratelimit.filter_factory,
        tempauth.filter_factory,
        container_quotas.filter_factory,
        account_quotas.filter_factory,
        proxy_logging.filter_factory,
    ]
    app = prosrv
    import mock
    for filter_factory in reversed(pipeline):
        app_filter = filter_factory(config)
        with mock.patch('swift.common.utils') as mock_utils:
            mock_utils.get_logger.return_value = None
            app = app_filter(app)
        app.logger = prosrv.logger

    nl = utils.NullLogger()
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)
    obj1spa = eventlet.spawn(eventlet.wsgi.server, obj1lis, obj1srv, nl)
    obj2spa = eventlet.spawn(eventlet.wsgi.server, obj2lis, obj2srv, nl)
    global _test_coros
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        partition, nodes = prosrv.account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node['ip'], node['port'], node['device'], partition, 'PUT',
                '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
            resp = conn.getresponse()
            assert(resp.status == 201)

    create_account('AUTH_test')
    create_account('AUTH_test2')
예제 #7
0
파일: __init__.py 프로젝트: bkolli/swift
def in_process_setup(the_object_server=object_server):
    _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS')
    _info('Using object_server class: %s' % the_object_server.__name__)
    conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR')

    if conf_src_dir is not None:
        if not os.path.isdir(conf_src_dir):
            msg = 'Config source %s is not a dir' % conf_src_dir
            raise InProcessException(msg)
        _info('Using config source dir: %s' % conf_src_dir)

    # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then
    # prefer config files from there, otherwise read config from source tree
    # sample files. A mixture of files from the two sources is allowed.
    proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf')
    _info('Using proxy config from %s' % proxy_conf)
    swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
    _info('Using swift config from %s' % swift_conf_src)

    monkey_patch_mimetools()

    global _testdir
    _testdir = os.path.join(mkdtemp(), 'tmp_functional')
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, 'sda1'))
    utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))

    swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
    obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)

    global orig_swift_conf_name
    orig_swift_conf_name = utils.SWIFT_CONF_FILE
    utils.SWIFT_CONF_FILE = swift_conf
    constraints.reload_constraints()
    storage_policy.SWIFT_CONF_FILE = swift_conf
    storage_policy.reload_storage_policies()
    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        _c = dict((k, str(v))
                  for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
        config.update(_c)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest
    global orig_hash_path_suff_pref
    orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
    utils.validate_hash_conf()

    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = eventlet.listen(('localhost', 0))

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update({
        # Values needed by the various in-process swift servers
        'devices': _testdir,
        'swift_dir': _testdir,
        'mount_check': 'false',
        'client_timeout': '4',
        'allow_account_management': 'true',
        'account_autocreate': 'true',
        'allow_versions': 'True',
        # Below are values used by the functional test framework, as well as
        # by the various in-process swift servers
        'auth_host': '127.0.0.1',
        'auth_port': str(prolis.getsockname()[1]),
        'auth_ssl': 'no',
        'auth_prefix': '/auth/',
        # Primary functional test account (needs admin access to the
        # account)
        'account': 'test',
        'username': '******',
        'password': '******',
        # User on a second account (needs admin access to the account)
        'account2': 'test2',
        'username2': 'tester2',
        'password2': 'testing2',
        # User on same account as first, but without admin access
        'username3': 'tester3',
        'password3': 'testing3',
        # Service user and prefix (emulates glance, cinder, etc. user)
        'account5': 'test5',
        'username5': 'tester5',
        'password5': 'testing5',
        'service_prefix': 'SERVICE',
        # For tempauth middleware. Update reseller_prefix
        'reseller_prefix': 'AUTH, SERVICE',
        'SERVICE_require_group': 'service'
    })

    acc1lis = eventlet.listen(('localhost', 0))
    acc2lis = eventlet.listen(('localhost', 0))
    con1lis = eventlet.listen(('localhost', 0))
    con2lis = eventlet.listen(('localhost', 0))

    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    with closing(GzipFile(account_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': acc1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': acc2lis.getsockname()[1]}], 30),
                    f)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    with closing(GzipFile(container_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': con1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': con2lis.getsockname()[1]}], 30),
                    f)

    eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
    logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')
    # Redirect logging other messages by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    acc1srv = account_server.AccountController(
        config, logger=debug_logger('acct1'))
    acc2srv = account_server.AccountController(
        config, logger=debug_logger('acct2'))
    con1srv = container_server.ContainerController(
        config, logger=debug_logger('cont1'))
    con2srv = container_server.ContainerController(
        config, logger=debug_logger('cont2'))

    objsrvs = [
        (obj_sockets[index],
         the_object_server.ObjectController(
             config, logger=debug_logger('obj%d' % (index + 1))))
        for index in range(len(obj_sockets))
    ]

    logger = debug_logger('proxy')

    def get_logger(name, *args, **kwargs):
        return logger

    with mock.patch('swift.common.utils.get_logger', get_logger):
        with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
                        FakeMemcacheMiddleware):
            try:
                app = loadapp(proxy_conf, global_conf=config)
            except Exception as e:
                raise InProcessException(e)

    nl = utils.NullLogger()
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)

    objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl)
              for objsrv in objsrvs]

    global _test_coros
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        account_ring = Ring(_testdir, ring_name='account')
        partition, nodes = account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node['ip'], node['port'], node['device'], partition, 'PUT',
                '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
            resp = conn.getresponse()
            assert(resp.status == 201)

    create_account('AUTH_test')
    create_account('AUTH_test2')
예제 #8
0
def in_process_setup(the_object_server=object_server):
    _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS')
    _info('Using object_server class: %s' % the_object_server.__name__)
    conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR')
    show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS')

    if conf_src_dir is not None:
        if not os.path.isdir(conf_src_dir):
            msg = 'Config source %s is not a dir' % conf_src_dir
            raise InProcessException(msg)
        _info('Using config source dir: %s' % conf_src_dir)

    # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then
    # prefer config files from there, otherwise read config from source tree
    # sample files. A mixture of files from the two sources is allowed.
    proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf')
    _info('Using proxy config from %s' % proxy_conf)
    swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
    _info('Using swift config from %s' % swift_conf_src)

    monkey_patch_mimetools()

    global _testdir
    _testdir = os.path.join(mkdtemp(), 'tmp_functional')
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, 'sda1'))
    utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdc1'))
    utils.mkdirs(os.path.join(_testdir, 'sdc1', 'tmp'))

    swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
    _info('prepared swift.conf: %s' % swift_conf)

    # Call the associated method for the value of
    # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists
    conf_loader_label = os.environ.get(
        'SWIFT_TEST_IN_PROCESS_CONF_LOADER')
    if conf_loader_label is not None:
        try:
            conf_loader = conf_loaders[conf_loader_label]
            _debug('Calling method %s mapped to conf loader %s' %
                   (conf_loader.__name__, conf_loader_label))
        except KeyError as missing_key:
            raise InProcessException('No function mapped for conf loader %s' %
                                     missing_key)

        try:
            # Pass-in proxy_conf, swift_conf files
            proxy_conf, swift_conf = conf_loader(proxy_conf, swift_conf)
            _debug('Now using proxy conf %s' % proxy_conf)
            _debug('Now using swift conf %s' % swift_conf)
        except Exception as err:  # noqa
            raise InProcessException(err)

    obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)

    # load new swift.conf file
    if set_swift_dir(os.path.dirname(swift_conf)):
        constraints.reload_constraints()
        storage_policy.reload_storage_policies()

    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        _c = dict((k, str(v))
                  for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
        config.update(_c)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest

    global _test_socks
    _test_socks = []
    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = listen_zero()
    _test_socks.append(prolis)

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update({
        # Values needed by the various in-process swift servers
        'devices': _testdir,
        'swift_dir': _testdir,
        'mount_check': 'false',
        'client_timeout': '4',
        'allow_account_management': 'true',
        'account_autocreate': 'true',
        'allow_versions': 'True',
        'allow_versioned_writes': 'True',
        # Below are values used by the functional test framework, as well as
        # by the various in-process swift servers
        'auth_host': '127.0.0.1',
        'auth_port': str(prolis.getsockname()[1]),
        'auth_ssl': 'no',
        'auth_prefix': '/auth/',
        # Primary functional test account (needs admin access to the
        # account)
        'account': 'test',
        'username': '******',
        'password': '******',
        # User on a second account (needs admin access to the account)
        'account2': 'test2',
        'username2': 'tester2',
        'password2': 'testing2',
        # User on same account as first, but without admin access
        'username3': 'tester3',
        'password3': 'testing3',
        # Service user and prefix (emulates glance, cinder, etc. user)
        'account5': 'test5',
        'username5': 'tester5',
        'password5': 'testing5',
        'service_prefix': 'SERVICE',
        # For tempauth middleware. Update reseller_prefix
        'reseller_prefix': 'AUTH, SERVICE',
        'SERVICE_require_group': 'service',
        # Reseller admin user (needs reseller_admin_role)
        'account6': 'test6',
        'username6': 'tester6',
        'password6': 'testing6'
    })

    # If an env var explicitly specifies the proxy-server object_post_as_copy
    # option then use its value, otherwise leave default config unchanged.
    object_post_as_copy = os.environ.get(
        'SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY')
    if object_post_as_copy is not None:
        object_post_as_copy = config_true_value(object_post_as_copy)
        config['object_post_as_copy'] = str(object_post_as_copy)
        _debug('Setting object_post_as_copy to %r' % object_post_as_copy)

    acc1lis = listen_zero()
    acc2lis = listen_zero()
    con1lis = listen_zero()
    con2lis = listen_zero()
    _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets

    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    with closing(GzipFile(account_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': acc1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': acc2lis.getsockname()[1]}], 30),
                    f)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    with closing(GzipFile(container_ring_path, 'wb')) as f:
        pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                    [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
                      'port': con1lis.getsockname()[1]},
                     {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
                      'port': con2lis.getsockname()[1]}], 30),
                    f)

    eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
    logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')
    # Redirect logging other messages by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    def get_logger_name(name):
        if show_debug_logs:
            return debug_logger(name)
        else:
            return None

    acc1srv = account_server.AccountController(
        config, logger=get_logger_name('acct1'))
    acc2srv = account_server.AccountController(
        config, logger=get_logger_name('acct2'))
    con1srv = container_server.ContainerController(
        config, logger=get_logger_name('cont1'))
    con2srv = container_server.ContainerController(
        config, logger=get_logger_name('cont2'))

    objsrvs = [
        (obj_sockets[index],
         the_object_server.ObjectController(
             config, logger=get_logger_name('obj%d' % (index + 1))))
        for index in range(len(obj_sockets))
    ]

    if show_debug_logs:
        logger = debug_logger('proxy')

    def get_logger(name, *args, **kwargs):
        return logger

    with mock.patch('swift.common.utils.get_logger', get_logger):
        with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
                        FakeMemcacheMiddleware):
            try:
                app = loadapp(proxy_conf, global_conf=config)
            except Exception as e:
                raise InProcessException(e)

    nl = utils.NullLogger()
    global proxy_srv
    proxy_srv = prolis
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)

    objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl)
              for objsrv in objsrvs]

    global _test_coros
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        account_ring = Ring(_testdir, ring_name='account')
        partition, nodes = account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node['ip'], node['port'], node['device'], partition, 'PUT',
                '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
            resp = conn.getresponse()
            assert resp.status == 201, 'Unable to create account: %s\n%s' % (
                resp.status, resp.body)

    create_account('AUTH_test')
    create_account('AUTH_test2')
예제 #9
0
def in_process_setup(the_object_server=object_server):
    print >> sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS'

    monkey_patch_mimetools()

    global _testdir
    _testdir = os.path.join(mkdtemp(), 'tmp_functional')
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, 'sda1'))
    utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))

    swift_conf = os.path.join(_testdir, "swift.conf")
    with open(swift_conf, "w") as scfp:
        scfp.write(functests_swift_conf)

    global orig_swift_conf_name
    orig_swift_conf_name = utils.SWIFT_CONF_FILE
    utils.SWIFT_CONF_FILE = swift_conf
    constraints.reload_constraints()
    storage_policy.SWIFT_CONF_FILE = swift_conf
    storage_policy.reload_storage_policies()
    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        config.update(constraints.EFFECTIVE_CONSTRAINTS)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest
    global orig_hash_path_suff_pref
    orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
    utils.validate_hash_conf()

    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = eventlet.listen(('localhost', 0))

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update({
        # Values needed by the various in-process swift servers
        'devices':
        _testdir,
        'swift_dir':
        _testdir,
        'mount_check':
        'false',
        'client_timeout':
        4,
        'allow_account_management':
        'true',
        'account_autocreate':
        'true',
        'allowed_headers':
        'content-disposition, content-encoding, x-delete-at,'
        ' x-object-manifest, x-static-large-object',
        'allow_versions':
        'True',
        # Below are values used by the functional test framework, as well as
        # by the various in-process swift servers
        'auth_host':
        '127.0.0.1',
        'auth_port':
        str(prolis.getsockname()[1]),
        'auth_ssl':
        'no',
        'auth_prefix':
        '/auth/',
        # Primary functional test account (needs admin access to the
        # account)
        'account':
        'test',
        'username':
        '******',
        'password':
        '******',
        # User on a second account (needs admin access to the account)
        'account2':
        'test2',
        'username2':
        'tester2',
        'password2':
        'testing2',
        # User on same account as first, but without admin access
        'username3':
        'tester3',
        'password3':
        'testing3',
        # For tempauth middleware
        'user_admin_admin':
        'admin .admin .reseller_admin',
        'user_test_tester':
        'testing .admin',
        'user_test2_tester2':
        'testing2 .admin',
        'user_test_tester3':
        'testing3'
    })

    acc1lis = eventlet.listen(('localhost', 0))
    acc2lis = eventlet.listen(('localhost', 0))
    con1lis = eventlet.listen(('localhost', 0))
    con2lis = eventlet.listen(('localhost', 0))
    obj1lis = eventlet.listen(('localhost', 0))
    obj2lis = eventlet.listen(('localhost', 0))
    global _test_sockets
    _test_sockets = \
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis)

    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    with closing(GzipFile(account_ring_path, 'wb')) as f:
        pickle.dump(
            ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                          [{
                              'id': 0,
                              'zone': 0,
                              'device': 'sda1',
                              'ip': '127.0.0.1',
                              'port': acc1lis.getsockname()[1]
                          }, {
                              'id': 1,
                              'zone': 1,
                              'device': 'sdb1',
                              'ip': '127.0.0.1',
                              'port': acc2lis.getsockname()[1]
                          }], 30), f)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    with closing(GzipFile(container_ring_path, 'wb')) as f:
        pickle.dump(
            ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                          [{
                              'id': 0,
                              'zone': 0,
                              'device': 'sda1',
                              'ip': '127.0.0.1',
                              'port': con1lis.getsockname()[1]
                          }, {
                              'id': 1,
                              'zone': 1,
                              'device': 'sdb1',
                              'ip': '127.0.0.1',
                              'port': con2lis.getsockname()[1]
                          }], 30), f)
    object_ring_path = os.path.join(_testdir, 'object.ring.gz')
    with closing(GzipFile(object_ring_path, 'wb')) as f:
        pickle.dump(
            ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                          [{
                              'id': 0,
                              'zone': 0,
                              'device': 'sda1',
                              'ip': '127.0.0.1',
                              'port': obj1lis.getsockname()[1]
                          }, {
                              'id': 1,
                              'zone': 1,
                              'device': 'sdb1',
                              'ip': '127.0.0.1',
                              'port': obj2lis.getsockname()[1]
                          }], 30), f)

    eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
    logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')
    # Redirect logging other messages by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    prosrv = proxy_server.Application(config, logger=debug_logger('proxy'))
    acc1srv = account_server.AccountController(config,
                                               logger=debug_logger('acct1'))
    acc2srv = account_server.AccountController(config,
                                               logger=debug_logger('acct2'))
    con1srv = container_server.ContainerController(
        config, logger=debug_logger('cont1'))
    con2srv = container_server.ContainerController(
        config, logger=debug_logger('cont2'))
    obj1srv = the_object_server.ObjectController(config,
                                                 logger=debug_logger('obj1'))
    obj2srv = the_object_server.ObjectController(config,
                                                 logger=debug_logger('obj2'))
    global _test_servers
    _test_servers = \
        (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv)

    pipeline = [
        catch_errors.filter_factory,
        gatekeeper.filter_factory,
        healthcheck.filter_factory,
        proxy_logging.filter_factory,
        fake_memcache_filter_factory,
        container_sync.filter_factory,
        bulk.filter_factory,
        tempurl.filter_factory,
        slo.filter_factory,
        dlo.filter_factory,
        ratelimit.filter_factory,
        tempauth.filter_factory,
        container_quotas.filter_factory,
        account_quotas.filter_factory,
        proxy_logging.filter_factory,
    ]
    app = prosrv
    import mock
    for filter_factory in reversed(pipeline):
        app_filter = filter_factory(config)
        with mock.patch('swift.common.utils') as mock_utils:
            mock_utils.get_logger.return_value = None
            app = app_filter(app)
        app.logger = prosrv.logger

    nl = utils.NullLogger()
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)
    obj1spa = eventlet.spawn(eventlet.wsgi.server, obj1lis, obj1srv, nl)
    obj2spa = eventlet.spawn(eventlet.wsgi.server, obj2lis, obj2srv, nl)
    global _test_coros
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        partition, nodes = prosrv.account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node['ip'], node['port'], node['device'], partition, 'PUT',
                '/' + act, {
                    'X-Timestamp': ts,
                    'x-trans-id': act
                })
            resp = conn.getresponse()
            assert (resp.status == 201)

    create_account('AUTH_test')
    create_account('AUTH_test2')
예제 #10
0
파일: recon.py 프로젝트: clayg/swift
    def main(self):
        """
        Retrieve and report cluster info from hosts running recon middleware.
        """
        print("=" * 79)
        usage = '''
        usage: %prog <server_type> [<server_type> [<server_type>]]
        [-v] [--suppress] [-a] [-r] [-u] [-d]
        [-l] [-T] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
        [--human-readable]

        <server_type>\taccount|container|object
        Defaults to object server.

        ex: %prog container -l --auditor
        '''
        args = optparse.OptionParser(usage)
        args.add_option('--verbose', '-v', action="store_true",
                        help="Print verbose info")
        args.add_option('--suppress', action="store_true",
                        help="Suppress most connection related errors")
        args.add_option('--async', '-a', action="store_true",
                        help="Get async stats")
        args.add_option('--replication', '-r', action="store_true",
                        help="Get replication stats")
        args.add_option('--auditor', action="store_true",
                        help="Get auditor stats")
        args.add_option('--updater', action="store_true",
                        help="Get updater stats")
        args.add_option('--expirer', action="store_true",
                        help="Get expirer stats")
        args.add_option('--unmounted', '-u', action="store_true",
                        help="Check cluster for unmounted devices")
        args.add_option('--diskusage', '-d', action="store_true",
                        help="Get disk usage stats")
        args.add_option('--human-readable', action="store_true",
                        help="Use human readable suffix for disk usage stats")
        args.add_option('--loadstats', '-l', action="store_true",
                        help="Get cluster load average stats")
        args.add_option('--quarantined', '-q', action="store_true",
                        help="Get cluster quarantine stats")
        args.add_option('--validate-servers', action="store_true",
                        help="Validate servers on the ring")
        args.add_option('--md5', action="store_true",
                        help="Get md5sum of servers ring and compare to "
                        "local copy")
        args.add_option('--sockstat', action="store_true",
                        help="Get cluster socket usage stats")
        args.add_option('--driveaudit', action="store_true",
                        help="Get drive audit error stats")
        args.add_option('--time', '-T', action="store_true",
                        help="Check time synchronization")
        args.add_option('--top', type='int', metavar='COUNT', default=0,
                        help='Also show the top COUNT entries in rank order.')
        args.add_option('--lowest', type='int', metavar='COUNT', default=0,
                        help='Also show the lowest COUNT entries in rank \
                        order.')
        args.add_option('--all', action="store_true",
                        help="Perform all checks. Equal to \t\t\t-arudlqT "
                        "--md5 --sockstat --auditor --updater --expirer "
                        "--driveaudit --validate-servers")
        args.add_option('--region', type="int",
                        help="Only query servers in specified region")
        args.add_option('--zone', '-z', type="int",
                        help="Only query servers in specified zone")
        args.add_option('--timeout', '-t', type="int", metavar="SECONDS",
                        help="Time to wait for a response from a server",
                        default=5)
        args.add_option('--swiftdir', default="/etc/swift",
                        help="Default = /etc/swift")
        args.add_option('--policy', '-p',
                        help='Only query object servers in specified '
                        'storage policy (specified as name or index).')
        options, arguments = args.parse_args()

        if len(sys.argv) <= 1 or len(arguments) > len(self.check_types):
            args.print_help()
            sys.exit(0)

        if arguments:
            arguments = set(arguments)
            if arguments.issubset(self.check_types):
                server_types = arguments
            else:
                print("Invalid Server Type")
                args.print_help()
                sys.exit(1)
        else:  # default
            server_types = ['object']

        swift_dir = options.swiftdir
        if set_swift_dir(swift_dir):
            reload_storage_policies()

        self.verbose = options.verbose
        self.suppress_errors = options.suppress
        self.timeout = options.timeout

        for server_type in server_types:
            self.server_type = server_type
            ring_names = self._get_ring_names(options.policy)
            if not ring_names:
                print('Invalid Storage Policy: %s' % options.policy)
                args.print_help()
                sys.exit(0)
            hosts = self.get_hosts(options.region, options.zone,
                                   swift_dir, ring_names)
            print("--> Starting reconnaissance on %s hosts (%s)" %
                  (len(hosts), self.server_type))
            print("=" * 79)
            if options.all:
                if self.server_type == 'object':
                    self.async_check(hosts)
                    self.object_auditor_check(hosts)
                    self.updater_check(hosts)
                    self.expirer_check(hosts)
                elif self.server_type == 'container':
                    self.auditor_check(hosts)
                    self.updater_check(hosts)
                elif self.server_type == 'account':
                    self.auditor_check(hosts)
                self.replication_check(hosts)
                self.umount_check(hosts)
                self.load_check(hosts)
                self.disk_usage(hosts, options.top, options.lowest,
                                options.human_readable)
                self.get_ringmd5(hosts, swift_dir)
                self.get_swiftconfmd5(hosts)
                self.quarantine_check(hosts)
                self.socket_usage(hosts)
                self.server_type_check(hosts)
                self.driveaudit_check(hosts)
                self.time_check(hosts)
            else:
                if options.async:
                    if self.server_type == 'object':
                        self.async_check(hosts)
                    else:
                        print("Error: Can't check asyncs on non object "
                              "servers.")
                        print("=" * 79)
                if options.unmounted:
                    self.umount_check(hosts)
                if options.replication:
                    self.replication_check(hosts)
                if options.auditor:
                    if self.server_type == 'object':
                        self.object_auditor_check(hosts)
                    else:
                        self.auditor_check(hosts)
                if options.updater:
                    if self.server_type == 'account':
                        print("Error: Can't check updaters on account "
                              "servers.")
                        print("=" * 79)
                    else:
                        self.updater_check(hosts)
                if options.expirer:
                    if self.server_type == 'object':
                        self.expirer_check(hosts)
                    else:
                        print("Error: Can't check expired on non object "
                              "servers.")
                        print("=" * 79)
                if options.validate_servers:
                    self.server_type_check(hosts)
                if options.loadstats:
                    self.load_check(hosts)
                if options.diskusage:
                    self.disk_usage(hosts, options.top, options.lowest,
                                    options.human_readable)
                if options.md5:
                    self.get_ringmd5(hosts, swift_dir)
                    self.get_swiftconfmd5(hosts)
                if options.quarantined:
                    self.quarantine_check(hosts)
                if options.sockstat:
                    self.socket_usage(hosts)
                if options.driveaudit:
                    self.driveaudit_check(hosts)
                if options.time:
                    self.time_check(hosts)
예제 #11
0
 def tearDown(self):
     shutil.rmtree(self.testdir, ignore_errors=1)
     storage_policy.reload_storage_policies()
예제 #12
0
def in_process_setup(the_object_server=object_server):
    print >> sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS'
    print >> sys.stderr, 'Using object_server: %s' % the_object_server.__name__
    _dir = os.path.normpath(
        os.path.join(os.path.abspath(__file__), os.pardir, os.pardir,
                     os.pardir))
    proxy_conf = os.path.join(_dir, 'etc', 'proxy-server.conf-sample')
    if os.path.exists(proxy_conf):
        print >> sys.stderr, 'Using proxy-server config from %s' % proxy_conf

    else:
        print >> sys.stderr, 'Failed to find conf file %s' % proxy_conf
        return

    monkey_patch_mimetools()

    global _testdir
    _testdir = os.path.join(mkdtemp(), 'tmp_functional')
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, 'sda1'))
    utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1'))
    utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))

    swift_conf = os.path.join(_testdir, "swift.conf")
    with open(swift_conf, "w") as scfp:
        scfp.write(functests_swift_conf)

    global orig_swift_conf_name
    orig_swift_conf_name = utils.SWIFT_CONF_FILE
    utils.SWIFT_CONF_FILE = swift_conf
    constraints.reload_constraints()
    storage_policy.SWIFT_CONF_FILE = swift_conf
    storage_policy.reload_storage_policies()
    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        _c = dict(
            (k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
        config.update(_c)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest
    global orig_hash_path_suff_pref
    orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
    utils.validate_hash_conf()

    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = eventlet.listen(('localhost', 0))

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update({
        # Values needed by the various in-process swift servers
        'devices': _testdir,
        'swift_dir': _testdir,
        'mount_check': 'false',
        'client_timeout': '4',
        'allow_account_management': 'true',
        'account_autocreate': 'true',
        'allow_versions': 'True',
        # Below are values used by the functional test framework, as well as
        # by the various in-process swift servers
        'auth_host': '127.0.0.1',
        'auth_port': str(prolis.getsockname()[1]),
        'auth_ssl': 'no',
        'auth_prefix': '/auth/',
        # Primary functional test account (needs admin access to the
        # account)
        'account': 'test',
        'username': '******',
        'password': '******',
        # User on a second account (needs admin access to the account)
        'account2': 'test2',
        'username2': 'tester2',
        'password2': 'testing2',
        # User on same account as first, but without admin access
        'username3': 'tester3',
        'password3': 'testing3',
        # Service user and prefix (emulates glance, cinder, etc. user)
        'account5': 'test5',
        'username5': 'tester5',
        'password5': 'testing5',
        'service_prefix': 'SERVICE',
        # For tempauth middleware. Update reseller_prefix
        'reseller_prefix': 'AUTH, SERVICE',
        'SERVICE_require_group': 'service'
    })

    acc1lis = eventlet.listen(('localhost', 0))
    acc2lis = eventlet.listen(('localhost', 0))
    con1lis = eventlet.listen(('localhost', 0))
    con2lis = eventlet.listen(('localhost', 0))
    obj1lis = eventlet.listen(('localhost', 0))
    obj2lis = eventlet.listen(('localhost', 0))
    global _test_sockets
    _test_sockets = \
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis)

    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    with closing(GzipFile(account_ring_path, 'wb')) as f:
        pickle.dump(
            ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                          [{
                              'id': 0,
                              'zone': 0,
                              'device': 'sda1',
                              'ip': '127.0.0.1',
                              'port': acc1lis.getsockname()[1]
                          }, {
                              'id': 1,
                              'zone': 1,
                              'device': 'sdb1',
                              'ip': '127.0.0.1',
                              'port': acc2lis.getsockname()[1]
                          }], 30), f)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    with closing(GzipFile(container_ring_path, 'wb')) as f:
        pickle.dump(
            ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                          [{
                              'id': 0,
                              'zone': 0,
                              'device': 'sda1',
                              'ip': '127.0.0.1',
                              'port': con1lis.getsockname()[1]
                          }, {
                              'id': 1,
                              'zone': 1,
                              'device': 'sdb1',
                              'ip': '127.0.0.1',
                              'port': con2lis.getsockname()[1]
                          }], 30), f)
    object_ring_path = os.path.join(_testdir, 'object.ring.gz')
    with closing(GzipFile(object_ring_path, 'wb')) as f:
        pickle.dump(
            ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
                          [{
                              'id': 0,
                              'zone': 0,
                              'device': 'sda1',
                              'ip': '127.0.0.1',
                              'port': obj1lis.getsockname()[1]
                          }, {
                              'id': 1,
                              'zone': 1,
                              'device': 'sdb1',
                              'ip': '127.0.0.1',
                              'port': obj2lis.getsockname()[1]
                          }], 30), f)

    eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
    logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi')
    # Redirect logging other messages by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_message = \
        lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    acc1srv = account_server.AccountController(config,
                                               logger=debug_logger('acct1'))
    acc2srv = account_server.AccountController(config,
                                               logger=debug_logger('acct2'))
    con1srv = container_server.ContainerController(
        config, logger=debug_logger('cont1'))
    con2srv = container_server.ContainerController(
        config, logger=debug_logger('cont2'))
    obj1srv = the_object_server.ObjectController(config,
                                                 logger=debug_logger('obj1'))
    obj2srv = the_object_server.ObjectController(config,
                                                 logger=debug_logger('obj2'))

    logger = debug_logger('proxy')

    def get_logger(name, *args, **kwargs):
        return logger

    with mock.patch('swift.common.utils.get_logger', get_logger):
        with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
                        FakeMemcacheMiddleware):
            app = loadapp(proxy_conf, global_conf=config)

    nl = utils.NullLogger()
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)
    obj1spa = eventlet.spawn(eventlet.wsgi.server, obj1lis, obj1srv, nl)
    obj2spa = eventlet.spawn(eventlet.wsgi.server, obj2lis, obj2srv, nl)
    global _test_coros
    _test_coros = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        account_ring = Ring(_testdir, ring_name='account')
        partition, nodes = account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node['ip'], node['port'], node['device'], partition, 'PUT',
                '/' + act, {
                    'X-Timestamp': ts,
                    'x-trans-id': act
                })
            resp = conn.getresponse()
            assert (resp.status == 201)

    create_account('AUTH_test')
    create_account('AUTH_test2')
예제 #13
0
파일: __init__.py 프로젝트: rtblife97/swift
def in_process_setup(the_object_server=object_server):
    _info("IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS")
    _info("Using object_server class: %s" % the_object_server.__name__)
    conf_src_dir = os.environ.get("SWIFT_TEST_IN_PROCESS_CONF_DIR")
    show_debug_logs = os.environ.get("SWIFT_TEST_DEBUG_LOGS")

    if conf_src_dir is not None:
        if not os.path.isdir(conf_src_dir):
            msg = "Config source %s is not a dir" % conf_src_dir
            raise InProcessException(msg)
        _info("Using config source dir: %s" % conf_src_dir)

    # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then
    # prefer config files from there, otherwise read config from source tree
    # sample files. A mixture of files from the two sources is allowed.
    proxy_conf = _in_process_find_conf_file(conf_src_dir, "proxy-server.conf")
    _info("Using proxy config from %s" % proxy_conf)
    swift_conf_src = _in_process_find_conf_file(conf_src_dir, "swift.conf")
    _info("Using swift config from %s" % swift_conf_src)

    monkey_patch_mimetools()

    global _testdir
    _testdir = os.path.join(mkdtemp(), "tmp_functional")
    utils.mkdirs(_testdir)
    rmtree(_testdir)
    utils.mkdirs(os.path.join(_testdir, "sda1"))
    utils.mkdirs(os.path.join(_testdir, "sda1", "tmp"))
    utils.mkdirs(os.path.join(_testdir, "sdb1"))
    utils.mkdirs(os.path.join(_testdir, "sdb1", "tmp"))

    swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir)
    obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir)

    global orig_swift_conf_name
    orig_swift_conf_name = utils.SWIFT_CONF_FILE
    utils.SWIFT_CONF_FILE = swift_conf
    constraints.reload_constraints()
    storage_policy.SWIFT_CONF_FILE = swift_conf
    storage_policy.reload_storage_policies()
    global config
    if constraints.SWIFT_CONSTRAINTS_LOADED:
        # Use the swift constraints that are loaded for the test framework
        # configuration
        _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
        config.update(_c)
    else:
        # In-process swift constraints were not loaded, somethings wrong
        raise SkipTest
    global orig_hash_path_suff_pref
    orig_hash_path_suff_pref = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
    utils.validate_hash_conf()

    global _test_socks
    _test_socks = []
    # We create the proxy server listening socket to get its port number so
    # that we can add it as the "auth_port" value for the functional test
    # clients.
    prolis = eventlet.listen(("localhost", 0))
    _test_socks.append(prolis)

    # The following set of configuration values is used both for the
    # functional test frame work and for the various proxy, account, container
    # and object servers.
    config.update(
        {
            # Values needed by the various in-process swift servers
            "devices": _testdir,
            "swift_dir": _testdir,
            "mount_check": "false",
            "client_timeout": "4",
            "allow_account_management": "true",
            "account_autocreate": "true",
            "allow_versions": "True",
            # Below are values used by the functional test framework, as well as
            # by the various in-process swift servers
            "auth_host": "127.0.0.1",
            "auth_port": str(prolis.getsockname()[1]),
            "auth_ssl": "no",
            "auth_prefix": "/auth/",
            # Primary functional test account (needs admin access to the
            # account)
            "account": "test",
            "username": "******",
            "password": "******",
            # User on a second account (needs admin access to the account)
            "account2": "test2",
            "username2": "tester2",
            "password2": "testing2",
            # User on same account as first, but without admin access
            "username3": "tester3",
            "password3": "testing3",
            # Service user and prefix (emulates glance, cinder, etc. user)
            "account5": "test5",
            "username5": "tester5",
            "password5": "testing5",
            "service_prefix": "SERVICE",
            # For tempauth middleware. Update reseller_prefix
            "reseller_prefix": "AUTH, SERVICE",
            "SERVICE_require_group": "service",
        }
    )

    acc1lis = eventlet.listen(("localhost", 0))
    acc2lis = eventlet.listen(("localhost", 0))
    con1lis = eventlet.listen(("localhost", 0))
    con2lis = eventlet.listen(("localhost", 0))
    _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets

    account_ring_path = os.path.join(_testdir, "account.ring.gz")
    with closing(GzipFile(account_ring_path, "wb")) as f:
        pickle.dump(
            ring.RingData(
                [[0, 1, 0, 1], [1, 0, 1, 0]],
                [
                    {"id": 0, "zone": 0, "device": "sda1", "ip": "127.0.0.1", "port": acc1lis.getsockname()[1]},
                    {"id": 1, "zone": 1, "device": "sdb1", "ip": "127.0.0.1", "port": acc2lis.getsockname()[1]},
                ],
                30,
            ),
            f,
        )
    container_ring_path = os.path.join(_testdir, "container.ring.gz")
    with closing(GzipFile(container_ring_path, "wb")) as f:
        pickle.dump(
            ring.RingData(
                [[0, 1, 0, 1], [1, 0, 1, 0]],
                [
                    {"id": 0, "zone": 0, "device": "sda1", "ip": "127.0.0.1", "port": con1lis.getsockname()[1]},
                    {"id": 1, "zone": 1, "device": "sdb1", "ip": "127.0.0.1", "port": con2lis.getsockname()[1]},
                ],
                30,
            ),
            f,
        )

    eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
    # Turn off logging requests by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_request = lambda *a: None
    logger = utils.get_logger(config, "wsgi-server", log_route="wsgi")
    # Redirect logging other messages by the underlying WSGI software.
    eventlet.wsgi.HttpProtocol.log_message = lambda s, f, *a: logger.error("ERROR WSGI: " + f % a)
    # Default to only 4 seconds for in-process functional test runs
    eventlet.wsgi.WRITE_TIMEOUT = 4

    def get_logger_name(name):
        if show_debug_logs:
            return debug_logger(name)
        else:
            return None

    acc1srv = account_server.AccountController(config, logger=get_logger_name("acct1"))
    acc2srv = account_server.AccountController(config, logger=get_logger_name("acct2"))
    con1srv = container_server.ContainerController(config, logger=get_logger_name("cont1"))
    con2srv = container_server.ContainerController(config, logger=get_logger_name("cont2"))

    objsrvs = [
        (obj_sockets[index], the_object_server.ObjectController(config, logger=get_logger_name("obj%d" % (index + 1))))
        for index in range(len(obj_sockets))
    ]

    if show_debug_logs:
        logger = debug_logger("proxy")

    def get_logger(name, *args, **kwargs):
        return logger

    with mock.patch("swift.common.utils.get_logger", get_logger):
        with mock.patch("swift.common.middleware.memcache.MemcacheMiddleware", FakeMemcacheMiddleware):
            try:
                app = loadapp(proxy_conf, global_conf=config)
            except Exception as e:
                raise InProcessException(e)

    nl = utils.NullLogger()
    global proxy_srv
    proxy_srv = prolis
    prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
    acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl)
    con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl)
    con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl)

    objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl) for objsrv in objsrvs]

    global _test_coros
    _test_coros = (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa)

    # Create accounts "test" and "test2"
    def create_account(act):
        ts = utils.normalize_timestamp(time())
        account_ring = Ring(_testdir, ring_name="account")
        partition, nodes = account_ring.get_nodes(act)
        for node in nodes:
            # Note: we are just using the http_connect method in the object
            # controller here to talk to the account server nodes.
            conn = swift.proxy.controllers.obj.http_connect(
                node["ip"],
                node["port"],
                node["device"],
                partition,
                "PUT",
                "/" + act,
                {"X-Timestamp": ts, "x-trans-id": act},
            )
            resp = conn.getresponse()
            assert resp.status == 201

    create_account("AUTH_test")
    create_account("AUTH_test2")