def test_unicode(self): cu = self._get_container_updater() containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', container='\xce\xa9') cb.initialize(normalize_timestamp(1), 0) obj_name = u'\N{GREEK CAPITAL LETTER OMEGA}' if six.PY2: obj_name = obj_name.encode('utf-8') cb.put_object(obj_name, normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') def accept(sock, addr): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write(b'HTTP/1.1 201 OK\r\nContent-Length: 0\r\n\r\n') out.flush() inc.read() except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen_zero() def spawn_accepts(): events = [] for _junk in range(2): with Timeout(3): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_object_count'], 1) self.assertEqual(info['reported_bytes_used'], 3)
def test_http_connect(self): bindsock = listen_zero() def accept(expected_par): try: with Timeout(3): sock, addr = bindsock.accept() fp = sock.makefile('rwb') fp.write(b'HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n' b'RESPONSE') fp.flush() line = fp.readline() path = b'/dev/' + expected_par + b'/path/..%25/?omg&no=%7f' self.assertEqual( line, b'PUT ' + path + b' HTTP/1.1\r\n') headers = {} line = fp.readline() while line and line != b'\r\n': headers[line.split(b':')[0].lower()] = \ line.split(b':')[1].strip() line = fp.readline() self.assertEqual(headers[b'content-length'], b'7') self.assertEqual(headers[b'x-header'], b'value') self.assertEqual(fp.readline(), b'REQUEST\r\n') except BaseException as err: return err return None for spawn_par, par in ( (b'par', b'par'), (b'up%C3%A5r', u'up\xe5r'), (b'%C3%BCpar', b'\xc3\xbcpar'), (b'1357', 1357)): event = spawn(accept, spawn_par) try: with Timeout(3): conn = bufferedhttp.http_connect( '127.0.0.1', bindsock.getsockname()[1], 'dev', par, 'PUT', '/path/..%/', { 'content-length': 7, 'x-header': 'value'}, query_string='omg&no=%7f') conn.send(b'REQUEST\r\n') self.assertTrue(conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)) resp = conn.getresponse() body = resp.read() conn.close() self.assertEqual(resp.status, 200) self.assertEqual(resp.reason, 'OK') self.assertEqual(body, b'RESPONSE') finally: err = event.wait() if err: raise Exception(err)
def in_process_setup(the_object_server=object_server): _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS') _info('Using object_server class: %s' % the_object_server.__name__) conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR') show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS') if conf_src_dir is not None: if not os.path.isdir(conf_src_dir): msg = 'Config source %s is not a dir' % conf_src_dir raise InProcessException(msg) _info('Using config source dir: %s' % conf_src_dir) # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then # prefer config files from there, otherwise read config from source tree # sample files. A mixture of files from the two sources is allowed. proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf') _info('Using proxy config from %s' % proxy_conf) swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf') _info('Using swift config from %s' % swift_conf_src) global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdc1')) utils.mkdirs(os.path.join(_testdir, 'sdc1', 'tmp')) swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) _info('prepared swift.conf: %s' % swift_conf) # Call the associated method for the value of # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists conf_loader_label = os.environ.get( 'SWIFT_TEST_IN_PROCESS_CONF_LOADER') if conf_loader_label is not None: try: conf_loader = conf_loaders[conf_loader_label] _debug('Calling method %s mapped to conf loader %s' % (conf_loader.__name__, conf_loader_label)) except KeyError as missing_key: raise InProcessException('No function mapped for conf loader %s' % missing_key) try: # Pass-in proxy_conf, swift_conf files proxy_conf, swift_conf = conf_loader(proxy_conf, swift_conf) _debug('Now using proxy conf %s' % proxy_conf) _debug('Now using swift conf %s' % swift_conf) except Exception as err: # noqa raise InProcessException(err) obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) # load new swift.conf file if set_swift_dir(os.path.dirname(swift_conf)): constraints.reload_constraints() storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict((k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global _test_socks _test_socks = [] # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = listen_zero() _test_socks.append(prolis) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', 'allow_versioned_writes': 'True', # TODO: move this into s3api config loader because they are # required by only s3api 'allowed_headers': "Content-Disposition, Content-Encoding, X-Delete-At, " "X-Object-Manifest, X-Static-Large-Object, Cache-Control, " "Content-Language, Expires, X-Robots-Tag", # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_uri': 'http://127.0.0.1:%d/auth/v1.0/' % prolis.getsockname()[1], # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', 's3_access_key': 'test:tester', 's3_secret_key': 'testing', # Secondary user of the primary test account (needs admin access # to the account) for s3api 's3_access_key2': 'test:tester2', 's3_secret_key2': 'testing2', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', 's3_access_key3': 'test:tester3', 's3_secret_key3': 'testing3', # Service user and prefix (emulates glance, cinder, etc. user) 'account5': 'test5', 'username5': 'tester5', 'password5': 'testing5', 'service_prefix': 'SERVICE', # For tempauth middleware. Update reseller_prefix 'reseller_prefix': 'AUTH, SERVICE', 'SERVICE_require_group': 'service', # Reseller admin user (needs reseller_admin_role) 'account6': 'test6', 'username6': 'tester6', 'password6': 'testing6' }) acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 def get_logger_name(name): if show_debug_logs: return debug_logger(name) else: return None acc1srv = account_server.AccountController( config, logger=get_logger_name('acct1')) acc2srv = account_server.AccountController( config, logger=get_logger_name('acct2')) con1srv = container_server.ContainerController( config, logger=get_logger_name('cont1')) con2srv = container_server.ContainerController( config, logger=get_logger_name('cont2')) objsrvs = [ (obj_sockets[index], the_object_server.ObjectController( config, logger=get_logger_name('obj%d' % (index + 1)))) for index in range(len(obj_sockets)) ] if show_debug_logs: logger = get_logger_name('proxy') else: logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): try: app = loadapp(proxy_conf, global_conf=config) except Exception as e: raise InProcessException(e) nl = utils.NullLogger() global proxy_srv proxy_srv = prolis prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl, protocol=SwiftHttpProtocol) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl, protocol=SwiftHttpProtocol) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl, protocol=SwiftHttpProtocol) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl, protocol=SwiftHttpProtocol) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl, protocol=SwiftHttpProtocol) objspa = [eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl, protocol=SwiftHttpProtocol) for objsrv in objsrvs] global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) resp = conn.getresponse() assert resp.status == 201, 'Unable to create account: %s\n%s' % ( resp.status, resp.read()) create_account('AUTH_test') create_account('AUTH_test2')
def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): """ If SWIFT_TEST_POLICY is set: - look in swift.conf file for specified policy - move this to be policy-0 but preserving its options - copy its ring file to test dir, changing its devices to suit in process testing, and renaming it to suit policy-0 Otherwise, create a default ring file. """ conf = ConfigParser() conf.read(swift_conf) sp_prefix = 'storage-policy:' try: # policy index 0 will be created if no policy exists in conf policies = parse_storage_policies(conf) except PolicyError as e: raise InProcessException(e) # clear all policies from test swift.conf before adding test policy back for policy in policies: conf.remove_section(sp_prefix + str(policy.idx)) if policy_specified: policy_to_test = policies.get_by_name(policy_specified) if policy_to_test is None: raise InProcessException('Failed to find policy name "%s"' % policy_specified) _info('Using specified policy %s' % policy_to_test.name) else: policy_to_test = policies.default _info('Defaulting to policy %s' % policy_to_test.name) # make policy_to_test be policy index 0 and default for the test config sp_zero_section = sp_prefix + '0' conf.add_section(sp_zero_section) for (k, v) in policy_to_test.get_info(config=True).items(): conf.set(sp_zero_section, k, str(v)) conf.set(sp_zero_section, 'default', 'True') with open(swift_conf, 'w') as fp: conf.write(fp) # look for a source ring file ring_file_src = ring_file_test = 'object.ring.gz' if policy_to_test.idx: ring_file_src = 'object-%s.ring.gz' % policy_to_test.idx try: ring_file_src = _in_process_find_conf_file(conf_src_dir, ring_file_src, use_sample=False) except InProcessException as e: if policy_specified: raise InProcessException('Failed to find ring file %s' % ring_file_src) ring_file_src = None ring_file_test = os.path.join(testdir, ring_file_test) if ring_file_src: # copy source ring file to a policy-0 test ring file, re-homing servers _info('Using source ring file %s' % ring_file_src) ring_data = ring.RingData.load(ring_file_src) obj_sockets = [] for dev in ring_data.devs: device = 'sd%c1' % chr(len(obj_sockets) + ord('a')) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) obj_socket = listen_zero() obj_sockets.append(obj_socket) dev['port'] = obj_socket.getsockname()[1] dev['ip'] = '127.0.0.1' dev['device'] = device dev['replication_port'] = dev['port'] dev['replication_ip'] = dev['ip'] ring_data.save(ring_file_test) else: # make default test ring, 3 replicas, 4 partitions, 3 devices # which will work for a replication policy or a 2+1 EC policy _info('No source object ring file, creating 3rep/4part/3dev ring') obj_sockets = [listen_zero() for _ in (0, 1, 2)] replica2part2dev_id = [[0, 1, 2, 0], [1, 2, 0, 1], [2, 0, 1, 2]] devs = [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj_sockets[0].getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj_sockets[1].getsockname()[1]}, {'id': 2, 'zone': 2, 'device': 'sdc1', 'ip': '127.0.0.1', 'port': obj_sockets[2].getsockname()[1]}] ring_data = ring.RingData(replica2part2dev_id, devs, 30) with closing(GzipFile(ring_file_test, 'wb')) as f: pickle.dump(ring_data, f) for dev in ring_data.devs: _debug('Ring file dev: %s' % dev) return obj_sockets
def setup_servers(the_object_server=object_server, extra_conf=None): """ Setup proxy, account, container and object servers using a set of fake rings and policies. :param the_object_server: The object server module to use (optional, defaults to swift.obj.server) :param extra_conf: A dict of config options that will update the basic config passed to all server instances. :returns: A dict containing the following entries: orig_POLICIES: the value of storage_policy.POLICIES prior to it being patched with fake policies orig_SysLogHandler: the value of utils.SysLogHandler prior to it being patched testdir: root directory used for test files test_POLICIES: a StoragePolicyCollection of fake policies test_servers: a tuple of test server instances test_sockets: a tuple of sockets used by test servers test_coros: a tuple of greenthreads in which test servers are running """ context = { "orig_POLICIES": storage_policy._POLICIES, "orig_SysLogHandler": utils.SysLogHandler} utils.HASH_PATH_SUFFIX = b'endcap' utils.SysLogHandler = mock.MagicMock() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of # proxy_server.Application we couldn't get to easily otherwise. context["testdir"] = _testdir = \ os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked') mkdirs(_testdir) rmtree(_testdir) for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1', 'sdf1', 'sdg1', 'sdh1', 'sdi1', 'sdj1', 'sdk1', 'sdl1'): mkdirs(os.path.join(_testdir, drive, 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', 'allow_versions': 't'} if extra_conf: conf.update(extra_conf) prolis = listen_zero() acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() obj1lis = listen_zero() obj2lis = listen_zero() obj3lis = listen_zero() obj4lis = listen_zero() obj5lis = listen_zero() obj6lis = listen_zero() objsocks = [obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis] context["test_sockets"] = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') account_devs = [ {'port': acc1lis.getsockname()[1]}, {'port': acc2lis.getsockname()[1]}, ] write_fake_ring(account_ring_path, *account_devs) container_ring_path = os.path.join(_testdir, 'container.ring.gz') container_devs = [ {'port': con1lis.getsockname()[1]}, {'port': con2lis.getsockname()[1]}, ] write_fake_ring(container_ring_path, *container_devs) storage_policy._POLICIES = storage_policy.StoragePolicyCollection([ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False), ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096), ECStoragePolicy(4, 'ec-dup', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096, ec_duplication_factor=2)]) obj_rings = { 0: ('sda1', 'sdb1'), 1: ('sdc1', 'sdd1'), 2: ('sde1', 'sdf1'), # sdg1, sdh1, sdi1 taken by policy 3 (see below) } for policy_index, devices in obj_rings.items(): policy = storage_policy.POLICIES[policy_index] obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz') obj_devs = [ {'port': objsock.getsockname()[1], 'device': dev} for objsock, dev in zip(objsocks, devices)] write_fake_ring(obj_ring_path, *obj_devs) # write_fake_ring can't handle a 3-element ring, and the EC policy needs # at least 6 devs to work with (ec_k=2, ec_m=1, duplication_factor=2), # so we do it manually devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}, {'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1', 'port': obj3lis.getsockname()[1]}, {'id': 3, 'zone': 0, 'device': 'sdj1', 'ip': '127.0.0.1', 'port': obj4lis.getsockname()[1]}, {'id': 4, 'zone': 0, 'device': 'sdk1', 'ip': '127.0.0.1', 'port': obj5lis.getsockname()[1]}, {'id': 5, 'zone': 0, 'device': 'sdl1', 'ip': '127.0.0.1', 'port': obj6lis.getsockname()[1]}] pol3_replica2part2dev_id = [[0, 1, 2, 0], [1, 2, 0, 1], [2, 0, 1, 2]] pol4_replica2part2dev_id = [[0, 1, 2, 3], [1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 0], [4, 5, 0, 1], [5, 0, 1, 2]] obj3_ring_path = os.path.join( _testdir, storage_policy.POLICIES[3].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj3_ring_path, 'wb')) as fh: pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh) obj4_ring_path = os.path.join( _testdir, storage_policy.POLICIES[4].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj4_ring_path, 'wb')) as fh: pickle.dump(RingData(pol4_replica2part2dev_id, devs, part_shift), fh) prosrv = proxy_server.Application(conf, logger=debug_logger('proxy')) for policy in storage_policy.POLICIES: # make sure all the rings are loaded prosrv.get_object_ring(policy.idx) # don't lose this one! context["test_POLICIES"] = storage_policy._POLICIES acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( conf, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( conf, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( conf, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController( conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) obj3srv = the_object_server.ObjectController( conf, logger=debug_logger('obj3')) obj4srv = the_object_server.ObjectController( conf, logger=debug_logger('obj4')) obj5srv = the_object_server.ObjectController( conf, logger=debug_logger('obj5')) obj6srv = the_object_server.ObjectController( conf, logger=debug_logger('obj6')) context["test_servers"] = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv, obj4srv, obj5srv, obj6srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware( listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger) prospa = spawn(wsgi.server, prolis, logging_prosv, nl) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl) con1spa = spawn(wsgi.server, con1lis, con1srv, nl) con2spa = spawn(wsgi.server, con2lis, con2srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl) obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl) obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl) obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl) context["test_coros"] = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa, obj4spa, obj5spa, obj6spa) # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create another account # used for account-to-account tests ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a1') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a1', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create containers, 1 per test policy sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) # Create container in other account # used for account-to-account tests sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( 'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( 'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) return context
def test_run_once(self): cu = self._get_container_updater() cu.run_once() containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) cu.run_once() self.assertTrue(os.path.exists(containers_dir)) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', container='c') cb.initialize(normalize_timestamp(1), 0) cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 0) self.assertEqual(info['bytes_used'], 0) self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0) cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0) def accept(sock, addr, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), 'PUT /sda1/0/a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assertTrue('x-put-timestamp' in headers) self.assertTrue('x-delete-timestamp' in headers) self.assertTrue('x-object-count' in headers) self.assertTrue('x-bytes-used' in headers) except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen_zero() def spawn_accepts(): events = [] for _junk in range(2): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr, 201)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_object_count'], 1) self.assertEqual(info['reported_bytes_used'], 3)
def test_run_once(self, mock_check_drive): mock_check_drive.side_effect = lambda r, d, mc: os.path.join(r, d) ou = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}, logger=self.logger) ou.run_once() async_dir = os.path.join(self.sda1, get_async_dir(POLICIES[0])) os.mkdir(async_dir) ou.run_once() self.assertTrue(os.path.exists(async_dir)) # each run calls check_device self.assertEqual([ mock.call(self.devices_dir, 'sda1', False), mock.call(self.devices_dir, 'sda1', False), ], mock_check_drive.mock_calls) mock_check_drive.reset_mock() ou = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'TrUe', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}, logger=self.logger) odd_dir = os.path.join(async_dir, 'not really supposed ' 'to be here') os.mkdir(odd_dir) ou.run_once() self.assertTrue(os.path.exists(async_dir)) self.assertEqual([ mock.call(self.devices_dir, 'sda1', True), ], mock_check_drive.mock_calls) ohash = hash_path('a', 'c', 'o') odir = os.path.join(async_dir, ohash[-3:]) mkdirs(odir) older_op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time() - 1))) op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time()))) for path in (op_path, older_op_path): with open(path, 'wb') as async_pending: pickle.dump({'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', 'headers': { 'X-Container-Timestamp': normalize_timestamp(0)}}, async_pending) ou.run_once() self.assertTrue(not os.path.exists(older_op_path)) self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1, 'unlinks': 1}) self.assertIsNone(pickle.load(open(op_path)).get('successes')) bindsock = listen_zero() def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = HeaderKeyDict() line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0]] = \ line.split(':')[1].strip() line = inc.readline() self.assertTrue('x-container-timestamp' in headers) self.assertTrue('X-Backend-Storage-Policy-Index' in headers) except BaseException as err: return err return None def accept(return_codes): try: events = [] for code in return_codes: with Timeout(3): sock, addr = bindsock.accept() events.append( spawn(accepter, sock, code)) for event in events: err = event.wait() if err: raise err except BaseException as err: return err return None event = spawn(accept, [201, 500, 500]) for dev in ou.get_container_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [404, 201]) ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0, 2], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [201]) ou.logger._clear() ou.run_once() err = event.wait() if err: raise err # we remove the async_pending and its containing suffix dir, but not # anything above that self.assertFalse(os.path.exists(op_path)) self.assertFalse(os.path.exists(os.path.dirname(op_path))) self.assertTrue(os.path.exists(os.path.dirname(os.path.dirname( op_path)))) self.assertEqual(ou.logger.get_increment_counts(), {'unlinks': 1, 'successes': 1})
def test_shard_container(self): cu = self._get_container_updater() cu.run_once() containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) cu.run_once() self.assertTrue(os.path.exists(containers_dir)) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='.shards_a', container='c') cb.initialize(normalize_timestamp(1), 0) cb.set_sharding_sysmeta('Root', 'a/c') self.assertFalse(cb.is_root_container()) cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 0) self.assertEqual(info['bytes_used'], 0) self.assertEqual(info['reported_put_timestamp'], '0') self.assertEqual(info['reported_delete_timestamp'], '0') self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0) cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') # Fake us having already reported *bad* stats under swift 2.18.0 cb.reported('0', '0', 1, 3) # Should fail with a bunch of connection-refused cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_put_timestamp'], '0') self.assertEqual(info['reported_delete_timestamp'], '0') self.assertEqual(info['reported_object_count'], 1) self.assertEqual(info['reported_bytes_used'], 3) def accept(sock, addr, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write(b'HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), b'PUT /sda1/2/.shards_a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != b'\r\n': headers[line.split(b':')[0].lower()] = \ line.split(b':')[1].strip() line = inc.readline() self.assertIn(b'x-put-timestamp', headers) self.assertIn(b'x-delete-timestamp', headers) self.assertIn(b'x-object-count', headers) self.assertIn(b'x-bytes-used', headers) except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen_zero() def spawn_accepts(): events = [] for _junk in range(2): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr, 201)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_put_timestamp'], '0000000001.00000') self.assertEqual(info['reported_delete_timestamp'], '0') self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0)
def in_process_setup(the_object_server=object_server): _info('IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS') _info('Using object_server class: %s' % the_object_server.__name__) conf_src_dir = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_DIR') show_debug_logs = os.environ.get('SWIFT_TEST_DEBUG_LOGS') if conf_src_dir is not None: if not os.path.isdir(conf_src_dir): msg = 'Config source %s is not a dir' % conf_src_dir raise InProcessException(msg) _info('Using config source dir: %s' % conf_src_dir) # If SWIFT_TEST_IN_PROCESS_CONF specifies a config source dir then # prefer config files from there, otherwise read config from source tree # sample files. A mixture of files from the two sources is allowed. proxy_conf = _in_process_find_conf_file(conf_src_dir, 'proxy-server.conf') _info('Using proxy config from %s' % proxy_conf) swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf') _info('Using swift config from %s' % swift_conf_src) monkey_patch_mimetools() global _testdir _testdir = os.path.join(mkdtemp(), 'tmp_functional') utils.mkdirs(_testdir) rmtree(_testdir) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdb1')) utils.mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) utils.mkdirs(os.path.join(_testdir, 'sdc1')) utils.mkdirs(os.path.join(_testdir, 'sdc1', 'tmp')) swift_conf = _in_process_setup_swift_conf(swift_conf_src, _testdir) _info('prepared swift.conf: %s' % swift_conf) # Call the associated method for the value of # 'SWIFT_TEST_IN_PROCESS_CONF_LOADER', if one exists conf_loader_label = os.environ.get('SWIFT_TEST_IN_PROCESS_CONF_LOADER') if conf_loader_label is not None: try: conf_loader = conf_loaders[conf_loader_label] _debug('Calling method %s mapped to conf loader %s' % (conf_loader.__name__, conf_loader_label)) except KeyError as missing_key: raise InProcessException('No function mapped for conf loader %s' % missing_key) try: # Pass-in proxy_conf, swift_conf files proxy_conf, swift_conf = conf_loader(proxy_conf, swift_conf) _debug('Now using proxy conf %s' % proxy_conf) _debug('Now using swift conf %s' % swift_conf) except Exception as err: # noqa raise InProcessException(err) obj_sockets = _in_process_setup_ring(swift_conf, conf_src_dir, _testdir) # load new swift.conf file if set_swift_dir(os.path.dirname(swift_conf)): constraints.reload_constraints() storage_policy.reload_storage_policies() global config if constraints.SWIFT_CONSTRAINTS_LOADED: # Use the swift constraints that are loaded for the test framework # configuration _c = dict( (k, str(v)) for k, v in constraints.EFFECTIVE_CONSTRAINTS.items()) config.update(_c) else: # In-process swift constraints were not loaded, somethings wrong raise SkipTest global _test_socks _test_socks = [] # We create the proxy server listening socket to get its port number so # that we can add it as the "auth_port" value for the functional test # clients. prolis = listen_zero() _test_socks.append(prolis) # The following set of configuration values is used both for the # functional test frame work and for the various proxy, account, container # and object servers. config.update({ # Values needed by the various in-process swift servers 'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'client_timeout': '4', 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', 'allow_versioned_writes': 'True', # TODO: move this into s3api config loader because they are # required by only s3api 'allowed_headers': "Content-Disposition, Content-Encoding, X-Delete-At, " "X-Object-Manifest, X-Static-Large-Object, Cache-Control, " "Content-Language, Expires, X-Robots-Tag", # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_host': '127.0.0.1', 'auth_port': str(prolis.getsockname()[1]), 'auth_ssl': 'no', 'auth_prefix': '/auth/', # Primary functional test account (needs admin access to the # account) 'account': 'test', 'username': '******', 'password': '******', 's3_access_key': 'test:tester', 's3_secret_key': 'testing', # User on a second account (needs admin access to the account) 'account2': 'test2', 'username2': 'tester2', 'password2': 'testing2', # User on same account as first, but without admin access 'username3': 'tester3', 'password3': 'testing3', 's3_access_key2': 'test:tester3', 's3_secret_key2': 'testing3', # Service user and prefix (emulates glance, cinder, etc. user) 'account5': 'test5', 'username5': 'tester5', 'password5': 'testing5', 'service_prefix': 'SERVICE', # For tempauth middleware. Update reseller_prefix 'reseller_prefix': 'AUTH, SERVICE', 'SERVICE_require_group': 'service', # Reseller admin user (needs reseller_admin_role) 'account6': 'test6', 'username6': 'tester6', 'password6': 'testing6' }) acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() _test_socks += [acc1lis, acc2lis, con1lis, con2lis] + obj_sockets account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump( ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{ 'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1] }, { 'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1] }], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump( ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{ 'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1] }, { 'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1] }], 30), f) # Default to only 4 seconds for in-process functional test runs eventlet.wsgi.WRITE_TIMEOUT = 4 def get_logger_name(name): if show_debug_logs: return debug_logger(name) else: return None acc1srv = account_server.AccountController(config, logger=get_logger_name('acct1')) acc2srv = account_server.AccountController(config, logger=get_logger_name('acct2')) con1srv = container_server.ContainerController( config, logger=get_logger_name('cont1')) con2srv = container_server.ContainerController( config, logger=get_logger_name('cont2')) objsrvs = [(obj_sockets[index], the_object_server.ObjectController(config, logger=get_logger_name( 'obj%d' % (index + 1)))) for index in range(len(obj_sockets))] if show_debug_logs: logger = get_logger_name('proxy') else: logger = utils.get_logger(config, 'wsgi-server', log_route='wsgi') def get_logger(name, *args, **kwargs): return logger with mock.patch('swift.common.utils.get_logger', get_logger): with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware', FakeMemcacheMiddleware): try: app = loadapp(proxy_conf, global_conf=config) except Exception as e: raise InProcessException(e) nl = utils.NullLogger() global proxy_srv proxy_srv = prolis prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl, protocol=SwiftHttpProtocol) acc1spa = eventlet.spawn(eventlet.wsgi.server, acc1lis, acc1srv, nl, protocol=SwiftHttpProtocol) acc2spa = eventlet.spawn(eventlet.wsgi.server, acc2lis, acc2srv, nl, protocol=SwiftHttpProtocol) con1spa = eventlet.spawn(eventlet.wsgi.server, con1lis, con1srv, nl, protocol=SwiftHttpProtocol) con2spa = eventlet.spawn(eventlet.wsgi.server, con2lis, con2srv, nl, protocol=SwiftHttpProtocol) objspa = [ eventlet.spawn(eventlet.wsgi.server, objsrv[0], objsrv[1], nl, protocol=SwiftHttpProtocol) for objsrv in objsrvs ] global _test_coros _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa) + tuple(objspa) # Create accounts "test" and "test2" def create_account(act): ts = utils.normalize_timestamp(time()) account_ring = Ring(_testdir, ring_name='account') partition, nodes = account_ring.get_nodes(act) for node in nodes: # Note: we are just using the http_connect method in the object # controller here to talk to the account server nodes. conn = swift.proxy.controllers.obj.http_connect( node['ip'], node['port'], node['device'], partition, 'PUT', '/' + act, { 'X-Timestamp': ts, 'x-trans-id': act }) resp = conn.getresponse() assert resp.status == 201, 'Unable to create account: %s\n%s' % ( resp.status, resp.body) create_account('AUTH_test') create_account('AUTH_test2')
def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): """ If SWIFT_TEST_POLICY is set: - look in swift.conf file for specified policy - move this to be policy-0 but preserving its options - copy its ring file to test dir, changing its devices to suit in process testing, and renaming it to suit policy-0 Otherwise, create a default ring file. """ conf = ConfigParser() conf.read(swift_conf) sp_prefix = 'storage-policy:' try: # policy index 0 will be created if no policy exists in conf policies = parse_storage_policies(conf) except PolicyError as e: raise InProcessException(e) # clear all policies from test swift.conf before adding test policy back for policy in policies: conf.remove_section(sp_prefix + str(policy.idx)) if policy_specified: policy_to_test = policies.get_by_name(policy_specified) if policy_to_test is None: raise InProcessException('Failed to find policy name "%s"' % policy_specified) _info('Using specified policy %s' % policy_to_test.name) else: policy_to_test = policies.default _info('Defaulting to policy %s' % policy_to_test.name) # make policy_to_test be policy index 0 and default for the test config sp_zero_section = sp_prefix + '0' conf.add_section(sp_zero_section) for (k, v) in policy_to_test.get_info(config=True).items(): conf.set(sp_zero_section, k, v) conf.set(sp_zero_section, 'default', True) with open(swift_conf, 'w') as fp: conf.write(fp) # look for a source ring file ring_file_src = ring_file_test = 'object.ring.gz' if policy_to_test.idx: ring_file_src = 'object-%s.ring.gz' % policy_to_test.idx try: ring_file_src = _in_process_find_conf_file(conf_src_dir, ring_file_src, use_sample=False) except InProcessException as e: if policy_specified: raise InProcessException('Failed to find ring file %s' % ring_file_src) ring_file_src = None ring_file_test = os.path.join(testdir, ring_file_test) if ring_file_src: # copy source ring file to a policy-0 test ring file, re-homing servers _info('Using source ring file %s' % ring_file_src) ring_data = ring.RingData.load(ring_file_src) obj_sockets = [] for dev in ring_data.devs: device = 'sd%c1' % chr(len(obj_sockets) + ord('a')) utils.mkdirs(os.path.join(_testdir, 'sda1')) utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) obj_socket = listen_zero() obj_sockets.append(obj_socket) dev['port'] = obj_socket.getsockname()[1] dev['ip'] = '127.0.0.1' dev['device'] = device dev['replication_port'] = dev['port'] dev['replication_ip'] = dev['ip'] ring_data.save(ring_file_test) else: # make default test ring, 3 replicas, 4 partitions, 3 devices # which will work for a replication policy or a 2+1 EC policy _info('No source object ring file, creating 3rep/4part/3dev ring') obj_sockets = [listen_zero() for _ in (0, 1, 2)] replica2part2dev_id = [[0, 1, 2, 0], [1, 2, 0, 1], [2, 0, 1, 2]] devs = [{ 'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj_sockets[0].getsockname()[1] }, { 'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj_sockets[1].getsockname()[1] }, { 'id': 2, 'zone': 2, 'device': 'sdc1', 'ip': '127.0.0.1', 'port': obj_sockets[2].getsockname()[1] }] ring_data = ring.RingData(replica2part2dev_id, devs, 30) with closing(GzipFile(ring_file_test, 'wb')) as f: pickle.dump(ring_data, f) for dev in ring_data.devs: _debug('Ring file dev: %s' % dev) return obj_sockets
def setup_servers(the_object_server=object_server, extra_conf=None): """ Setup proxy, account, container and object servers using a set of fake rings and policies. :param the_object_server: The object server module to use (optional, defaults to swift.obj.server) :param extra_conf: A dict of config options that will update the basic config passed to all server instances. :returns: A dict containing the following entries: orig_POLICIES: the value of storage_policy.POLICIES prior to it being patched with fake policies orig_SysLogHandler: the value of utils.SysLogHandler prior to it being patched testdir: root directory used for test files test_POLICIES: a StoragePolicyCollection of fake policies test_servers: a tuple of test server instances test_sockets: a tuple of sockets used by test servers test_coros: a tuple of greenthreads in which test servers are running """ context = { "orig_POLICIES": storage_policy._POLICIES, "orig_SysLogHandler": utils.SysLogHandler} utils.HASH_PATH_SUFFIX = b'endcap' utils.SysLogHandler = mock.MagicMock() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of # proxy_server.Application we couldn't get to easily otherwise. context["testdir"] = _testdir = \ os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked') mkdirs(_testdir) rmtree(_testdir) for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1', 'sdf1', 'sdg1', 'sdh1', 'sdi1', 'sdj1', 'sdk1', 'sdl1'): mkdirs(os.path.join(_testdir, drive, 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', 'allow_versions': 't', 'node_timeout': 20} if extra_conf: conf.update(extra_conf) prolis = listen_zero() acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() obj1lis = listen_zero() obj2lis = listen_zero() obj3lis = listen_zero() obj4lis = listen_zero() obj5lis = listen_zero() obj6lis = listen_zero() objsocks = [obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis] context["test_sockets"] = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') account_devs = [ {'port': acc1lis.getsockname()[1]}, {'port': acc2lis.getsockname()[1]}, ] write_fake_ring(account_ring_path, *account_devs) container_ring_path = os.path.join(_testdir, 'container.ring.gz') container_devs = [ {'port': con1lis.getsockname()[1]}, {'port': con2lis.getsockname()[1]}, ] write_fake_ring(container_ring_path, *container_devs) storage_policy._POLICIES = storage_policy.StoragePolicyCollection([ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False), ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096), ECStoragePolicy(4, 'ec-dup', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096, ec_duplication_factor=2)]) obj_rings = { 0: ('sda1', 'sdb1'), 1: ('sdc1', 'sdd1'), 2: ('sde1', 'sdf1'), # sdg1, sdh1, sdi1 taken by policy 3 (see below) } for policy_index, devices in obj_rings.items(): policy = storage_policy.POLICIES[policy_index] obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz') obj_devs = [ {'port': objsock.getsockname()[1], 'device': dev} for objsock, dev in zip(objsocks, devices)] write_fake_ring(obj_ring_path, *obj_devs) # write_fake_ring can't handle a 3-element ring, and the EC policy needs # at least 6 devs to work with (ec_k=2, ec_m=1, duplication_factor=2), # so we do it manually devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}, {'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1', 'port': obj3lis.getsockname()[1]}, {'id': 3, 'zone': 0, 'device': 'sdj1', 'ip': '127.0.0.1', 'port': obj4lis.getsockname()[1]}, {'id': 4, 'zone': 0, 'device': 'sdk1', 'ip': '127.0.0.1', 'port': obj5lis.getsockname()[1]}, {'id': 5, 'zone': 0, 'device': 'sdl1', 'ip': '127.0.0.1', 'port': obj6lis.getsockname()[1]}] pol3_replica2part2dev_id = [[0, 1, 2, 0], [1, 2, 0, 1], [2, 0, 1, 2]] pol4_replica2part2dev_id = [[0, 1, 2, 3], [1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 0], [4, 5, 0, 1], [5, 0, 1, 2]] obj3_ring_path = os.path.join( _testdir, storage_policy.POLICIES[3].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj3_ring_path, 'wb')) as fh: pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh) obj4_ring_path = os.path.join( _testdir, storage_policy.POLICIES[4].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj4_ring_path, 'wb')) as fh: pickle.dump(RingData(pol4_replica2part2dev_id, devs, part_shift), fh) prosrv = proxy_server.Application(conf, logger=debug_logger('proxy')) for policy in storage_policy.POLICIES: # make sure all the rings are loaded prosrv.get_object_ring(policy.idx) # don't lose this one! context["test_POLICIES"] = storage_policy._POLICIES acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( conf, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( conf, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( conf, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController( conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) obj3srv = the_object_server.ObjectController( conf, logger=debug_logger('obj3')) obj4srv = the_object_server.ObjectController( conf, logger=debug_logger('obj4')) obj5srv = the_object_server.ObjectController( conf, logger=debug_logger('obj5')) obj6srv = the_object_server.ObjectController( conf, logger=debug_logger('obj6')) context["test_servers"] = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv, obj4srv, obj5srv, obj6srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware( listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger) prospa = spawn(wsgi.server, prolis, logging_prosv, nl, protocol=SwiftHttpProtocol) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl, protocol=SwiftHttpProtocol) acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl, protocol=SwiftHttpProtocol) con1spa = spawn(wsgi.server, con1lis, con1srv, nl, protocol=SwiftHttpProtocol) con2spa = spawn(wsgi.server, con2lis, con2srv, nl, protocol=SwiftHttpProtocol) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl, protocol=SwiftHttpProtocol) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl, protocol=SwiftHttpProtocol) obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl, protocol=SwiftHttpProtocol) obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl, protocol=SwiftHttpProtocol) obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl, protocol=SwiftHttpProtocol) obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl, protocol=SwiftHttpProtocol) context["test_coros"] = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa, obj4spa, obj5spa, obj6spa) # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create another account # used for account-to-account tests ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a1') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a1', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create containers, 1 per test policy sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile('rwb') fd.write(b'PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' b'Connection: close\r\nX-Auth-Token: t\r\n' b'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = b'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) # Create container in other account # used for account-to-account tests sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile('rwb') fd.write(b'PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n' b'Connection: close\r\nX-Auth-Token: t\r\n' b'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = b'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile('rwb') fd.write( b'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n' b'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n' b'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = b'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected %r, encountered %r" % (exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile('rwb') fd.write( b'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n' b'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n' b'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = b'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) return context
def test_run_once(self, mock_ismount): mock_ismount.return_value = True ou = object_updater.ObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15' }, logger=self.logger) ou.run_once() async_dir = os.path.join(self.sda1, get_async_dir(POLICIES[0])) os.mkdir(async_dir) ou.run_once() self.assertTrue(os.path.exists(async_dir)) # mount_check == False means no call to ismount self.assertEqual([], mock_ismount.mock_calls) ou = object_updater.ObjectUpdater( { 'devices': self.devices_dir, 'mount_check': 'TrUe', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15' }, logger=self.logger) odd_dir = os.path.join(async_dir, 'not really supposed ' 'to be here') os.mkdir(odd_dir) ou.run_once() self.assertTrue(os.path.exists(async_dir)) self.assertTrue(not os.path.exists(odd_dir)) # mount_check == True means ismount was checked self.assertEqual([ mock.call(self.sda1), ], mock_ismount.mock_calls) ohash = hash_path('a', 'c', 'o') odir = os.path.join(async_dir, ohash[-3:]) mkdirs(odir) older_op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time() - 1))) op_path = os.path.join(odir, '%s-%s' % (ohash, normalize_timestamp(time()))) for path in (op_path, older_op_path): with open(path, 'wb') as async_pending: pickle.dump( { 'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', 'headers': { 'X-Container-Timestamp': normalize_timestamp(0) } }, async_pending) ou.run_once() self.assertTrue(not os.path.exists(older_op_path)) self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), { 'failures': 1, 'unlinks': 1 }) self.assertIsNone(pickle.load(open(op_path)).get('successes')) bindsock = listen_zero() def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = HeaderKeyDict() line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0]] = \ line.split(':')[1].strip() line = inc.readline() self.assertTrue('x-container-timestamp' in headers) self.assertTrue( 'X-Backend-Storage-Policy-Index' in headers) except BaseException as err: return err return None def accept(return_codes): try: events = [] for code in return_codes: with Timeout(3): sock, addr = bindsock.accept() events.append(spawn(accepter, sock, code)) for event in events: err = event.wait() if err: raise err except BaseException as err: return err return None event = spawn(accept, [201, 500, 500]) for dev in ou.get_container_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [404, 201]) ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0, 2], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [201]) ou.logger._clear() ou.run_once() err = event.wait() if err: raise err self.assertTrue(not os.path.exists(op_path)) self.assertEqual(ou.logger.get_increment_counts(), { 'unlinks': 1, 'successes': 1 })
def test_run_once(self): cu = self._get_container_updater() cu.run_once() containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) cu.run_once() self.assertTrue(os.path.exists(containers_dir)) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', container='c') cb.initialize(normalize_timestamp(1), 0) cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 0) self.assertEqual(info['bytes_used'], 0) self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0) cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0) def accept(sock, addr, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), 'PUT /sda1/0/a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assertTrue('x-put-timestamp' in headers) self.assertTrue('x-delete-timestamp' in headers) self.assertTrue('x-object-count' in headers) self.assertTrue('x-bytes-used' in headers) except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen_zero() def spawn_accepts(): events = [] for _junk in range(2): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr, 201)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_object_count'], 1) self.assertEqual(info['reported_bytes_used'], 3)
def test_shard_container(self): cu = self._get_container_updater() cu.run_once() containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) cu.run_once() self.assertTrue(os.path.exists(containers_dir)) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='.shards_a', container='c') cb.initialize(normalize_timestamp(1), 0) cb.set_sharding_sysmeta('Quoted-Root', 'a/c') self.assertFalse(cb.is_root_container()) cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 0) self.assertEqual(info['bytes_used'], 0) self.assertEqual(info['reported_put_timestamp'], '0') self.assertEqual(info['reported_delete_timestamp'], '0') self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0) cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') # Fake us having already reported *bad* stats under swift 2.18.0 cb.reported('0', '0', 1, 3) # Should fail with a bunch of connection-refused cu.run_once() info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_put_timestamp'], '0') self.assertEqual(info['reported_delete_timestamp'], '0') self.assertEqual(info['reported_object_count'], 1) self.assertEqual(info['reported_bytes_used'], 3) def accept(sock, addr, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write(b'HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEqual(inc.readline(), b'PUT /sda1/2/.shards_a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != b'\r\n': headers[line.split(b':')[0].lower()] = \ line.split(b':')[1].strip() line = inc.readline() self.assertIn(b'x-put-timestamp', headers) self.assertIn(b'x-delete-timestamp', headers) self.assertIn(b'x-object-count', headers) self.assertIn(b'x-bytes-used', headers) except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen_zero() def spawn_accepts(): events = [] for _junk in range(2): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr, 201)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEqual(info['object_count'], 1) self.assertEqual(info['bytes_used'], 3) self.assertEqual(info['reported_put_timestamp'], '0000000001.00000') self.assertEqual(info['reported_delete_timestamp'], '0') self.assertEqual(info['reported_object_count'], 0) self.assertEqual(info['reported_bytes_used'], 0)