def __init__(self, conf): """ :param conf: configuration object obtained from ConfigParser :param logger: logging object """ self.conf = conf self.logger = get_logger(conf, log_route='object-mover') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.bind_ip = conf.get('bind_ip', '0.0.0.0') self.servers_per_port = int(conf.get('servers_per_port', '0') or 0) self.port = None if self.servers_per_port else \ int(conf.get('bind_port', 6000)) self.concurrency = int(conf.get('concurrency', 1)) self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) self.handoffs_first = config_true_value( conf.get('handoffs_first', False)) self.data_moving_map_dump = (conf.get('data_moving_map_dump') or DEFAULT_DUMP_FILE) self._diskfile_mgr = DiskFileManager(conf, self.logger) self.mover_tmp_dir = (conf.get('mover_tmp_dir') or 'data_mover') self.retries = int(conf.get('retries', 3)) self.test = bool(conf.get('test', False)) self.retrie_list = []
def do_test(headers_out, expected, container_path=None): # write an async dfmanager = DiskFileManager(conf, daemon.logger) self._write_async_update(dfmanager, next(ts_iter), policies[0], headers=headers_out, container_path=container_path) request_log = [] def capture(*args, **kwargs): request_log.append((args, kwargs)) # run once fake_status_codes = [ 200, # object update success 200, # object update success 200, # object update conflict ] with mocked_http_conn(*fake_status_codes, give_connect=capture): daemon.run_once() self.assertEqual(len(fake_status_codes), len(request_log)) for request_args, request_kwargs in request_log: ip, part, method, path, headers, qs, ssl = request_args self.assertEqual(method, 'PUT') self.assertDictEqual(expected, headers) self.assertEqual(daemon.logger.get_increment_counts(), { 'successes': 1, 'unlinks': 1, 'async_pendings': 1 }) self.assertFalse(os.listdir(async_dir)) daemon.logger.clear()
def setUp(self): self.testdir = os.path.join(mkdtemp(), 'tmp_test_object_auditor') self.devices = os.path.join(self.testdir, 'node') self.rcache = os.path.join(self.testdir, 'object.recon') self.logger = FakeLogger() rmtree(self.testdir, ignore_errors=1) mkdirs(os.path.join(self.devices, 'sda')) os.mkdir(os.path.join(self.devices, 'sdb')) # policy 0 self.objects = os.path.join(self.devices, 'sda', get_data_dir(0)) self.objects_2 = os.path.join(self.devices, 'sdb', get_data_dir(0)) os.mkdir(self.objects) # policy 1 self.objects_p1 = os.path.join(self.devices, 'sda', get_data_dir(1)) self.objects_2_p1 = os.path.join(self.devices, 'sdb', get_data_dir(1)) os.mkdir(self.objects_p1) self.parts = self.parts_p1 = {} for part in ['0', '1', '2', '3']: self.parts[part] = os.path.join(self.objects, part) self.parts_p1[part] = os.path.join(self.objects_p1, part) os.mkdir(os.path.join(self.objects, part)) os.mkdir(os.path.join(self.objects_p1, part)) self.conf = dict( devices=self.devices, mount_check='false', object_size_stats='10,100,1024,10240') self.df_mgr = DiskFileManager(self.conf, self.logger) # diskfiles for policy 0, 1 self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o', 0) self.disk_file_p1 = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o', 1)
def test_obj_put_async_root_update_redirected_previous_success(self): policies = list(POLICIES) random.shuffle(policies) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } daemon = object_updater.ObjectUpdater(conf, logger=self.logger) async_dir = os.path.join(self.sda1, get_async_dir(policies[0])) os.mkdir(async_dir) dfmanager = DiskFileManager(conf, daemon.logger) ts_obj = next(self.ts_iter) self._write_async_update(dfmanager, ts_obj, policies[0]) orig_async_path, orig_async_data = self._check_async_file(async_dir) # run once with mocked_http_conn( 507, 200, 507) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests, ts_obj, policies[0]) self.assertEqual(['/sda1/0/a/c/o'] * 3, [req['path'] for req in conn.requests]) self.assertEqual( {'failures': 1, 'async_pendings': 1}, daemon.logger.get_increment_counts()) async_path, async_data = self._check_async_file(async_dir) self.assertEqual(dict(orig_async_data, successes=[1]), async_data) # run again - expect 3 redirected updates despite previous success ts_redirect = next(self.ts_iter) resp_headers_1 = {'Location': '/.shards_a/c_shard_1/o', 'X-Backend-Redirect-Timestamp': ts_redirect.internal} fake_responses = ( # 1st round of redirects, 2nd round of redirects [(301, resp_headers_1)] * 2 + [(200, {})] * 3) fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn( *fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests[:2], ts_obj, policies[0]) self._check_update_requests(conn.requests[2:], ts_obj, policies[0]) root_part = daemon.container_ring.get_part('a/c') shard_1_part = daemon.container_ring.get_part('.shards_a/c_shard_1') self.assertEqual( ['/sda1/%s/a/c/o' % root_part] * 2 + ['/sda1/%s/.shards_a/c_shard_1/o' % shard_1_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual( {'redirects': 1, 'successes': 1, 'failures': 1, 'unlinks': 1, 'async_pendings': 1}, daemon.logger.get_increment_counts()) self.assertFalse(os.listdir(async_dir)) # no async file
def test_obj_put_async_root_update_redirected(self): policies = list(POLICIES) random.shuffle(policies) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } daemon = object_updater.ObjectUpdater(conf, logger=self.logger) async_dir = os.path.join(self.sda1, get_async_dir(policies[0])) os.mkdir(async_dir) dfmanager = DiskFileManager(conf, daemon.logger) ts_obj = next(self.ts_iter) self._write_async_update(dfmanager, ts_obj, policies[0]) # run once ts_redirect_1 = next(self.ts_iter) ts_redirect_2 = next(self.ts_iter) fake_responses = [ # first round of update attempts, newest redirect should be chosen (200, {}), (301, { 'Location': '/.shards_a/c_shard_new/o', 'X-Backend-Redirect-Timestamp': ts_redirect_2.internal }), (301, { 'Location': '/.shards_a/c_shard_old/o', 'X-Backend-Redirect-Timestamp': ts_redirect_1.internal }), # second round of update attempts (200, {}), (200, {}), (200, {}), ] fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests[:3], ts_obj, policies[0]) self._check_update_requests(conn.requests[3:], ts_obj, policies[0]) self.assertEqual(['/sda1/0/a/c/o'] * 3 + ['/sda1/0/.shards_a/c_shard_new/o'] * 3, [req['path'] for req in conn.requests]) self.assertEqual( { 'redirects': 1, 'successes': 1, 'unlinks': 1, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) self.assertFalse(os.listdir(async_dir)) # no async file
def test_obj_put_legacy_updates(self): ts = (normalize_timestamp(t) for t in itertools.count(int(time()))) policy = POLICIES.get_by_index(0) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } async_dir = os.path.join(self.sda1, get_async_dir(policy)) os.mkdir(async_dir) account, container, obj = 'a', 'c', 'o' # write an async for op in ('PUT', 'DELETE'): self.logger._clear() daemon = object_updater.ObjectUpdater(conf, logger=self.logger) dfmanager = DiskFileManager(conf, daemon.logger) # don't include storage-policy-index in headers_out pickle headers_out = HeaderKeyDict({ 'x-size': 0, 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-timestamp': next(ts), }) data = { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out } dfmanager.pickle_async_update(self.sda1, account, container, obj, data, next(ts), policy) request_log = [] def capture(*args, **kwargs): request_log.append((args, kwargs)) # run once fake_status_codes = [200, 200, 200] with mocked_http_conn(*fake_status_codes, give_connect=capture): daemon.run_once() self.assertEqual(len(fake_status_codes), len(request_log)) for request_args, request_kwargs in request_log: ip, part, method, path, headers, qs, ssl = request_args self.assertEqual(method, op) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], str(int(policy))) self.assertEqual(daemon.logger.get_increment_counts(), { 'successes': 1, 'unlinks': 1, 'async_pendings': 1 })
def __init__(self, conf, logger=None): """ :param conf: configuration object obtained from ConfigParser :param logger: logging object """ self.conf = conf self.logger = logger or get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.bind_ip = conf.get('bind_ip', '0.0.0.0') self.servers_per_port = int(conf.get('servers_per_port', '0') or 0) self.port = None if self.servers_per_port else \ int(conf.get('bind_port', 6000)) self.concurrency = int(conf.get('concurrency', 1)) self.stats_interval = int(conf.get('stats_interval', '300')) self.ring_check_interval = int(conf.get('ring_check_interval', 15)) self.next_check = time.time() + self.ring_check_interval self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) self.partition_times = [] self.interval = int( conf.get('interval') or conf.get('run_pause') or 30) self.rsync_timeout = int(conf.get('rsync_timeout', 900)) self.rsync_io_timeout = conf.get('rsync_io_timeout', '30') self.rsync_bwlimit = conf.get('rsync_bwlimit', '0') self.rsync_compress = config_true_value( conf.get('rsync_compress', 'no')) self.http_timeout = int(conf.get('http_timeout', 60)) self.lockup_timeout = int(conf.get('lockup_timeout', 1800)) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "object.recon") self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.node_timeout = float(conf.get('node_timeout', 10)) self.sync_method = getattr(self, conf.get('sync_method') or 'rsync') self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.default_headers = { 'Content-Length': '0', 'user-agent': 'object-replicator %s' % os.getpid() } self.rsync_error_log_line_length = \ int(conf.get('rsync_error_log_line_length', 0)) self.handoffs_first = config_true_value( conf.get('handoffs_first', False)) self.handoff_delete = config_auto_int_value( conf.get('handoff_delete', 'auto'), 0) if any((self.handoff_delete, self.handoffs_first)): self.logger.warn('Handoff only mode is not intended for normal ' 'operation, please disable handoffs_first and ' 'handoff_delete before the next ' 'normal rebalance') self._diskfile_mgr = DiskFileManager(conf, self.logger)
def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger)
def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='object-updater') self.devices = conf.get('devices', '/srv/node') self.ip = conf.get('md-server-ip', '127.0.0.1') self.port = conf.get('md-server-port', '6090') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.interval = int(conf.get('interval', 30)) #self.container_ring = None #self.concurrency = int(conf.get('concurrency', 1)) #self.slowdown = float(conf.get('slowdown', 0.01)) #self.node_timeout = int(conf.get('node_timeout', 10)) #self.conn_timeout = float(conf.get('conn_timeout', .5)) self.last_time_ran = 0 self.diskfile_mgr = DiskFileManager(conf, self.logger)
def __init__(self, conf): """ :param conf: configuration object obtained from ConfigParser :param logger: logging object """ self.conf = conf self.logger = get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.port = int(conf.get('bind_port', 6000)) self.concurrency = int(conf.get('concurrency', 1)) self.stats_interval = int(conf.get('stats_interval', '300')) self.object_ring = Ring(self.swift_dir, ring_name='object') self.ring_check_interval = int(conf.get('ring_check_interval', 15)) self.next_check = time.time() + self.ring_check_interval self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) self.partition_times = [] self.run_pause = int(conf.get('run_pause', 30)) self.rsync_timeout = int(conf.get('rsync_timeout', 900)) self.rsync_io_timeout = conf.get('rsync_io_timeout', '30') self.rsync_bwlimit = conf.get('rsync_bwlimit', '0') self.http_timeout = int(conf.get('http_timeout', 60)) self.lockup_timeout = int(conf.get('lockup_timeout', 1800)) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "object.recon") self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.node_timeout = float(conf.get('node_timeout', 10)) self.sync_method = getattr(self, conf.get('sync_method') or 'rsync') self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.headers = { 'Content-Length': '0', 'user-agent': 'obj-replicator %s' % os.getpid() } self.rsync_error_log_line_length = \ int(conf.get('rsync_error_log_line_length', 0)) self.handoffs_first = config_true_value( conf.get('handoffs_first', False)) self.handoff_delete = config_auto_int_value( conf.get('handoff_delete', 'auto'), 0) self._diskfile_mgr = DiskFileManager(conf, self.logger)
def _get_object_info(self, account, container, obj, number): obj_conf = self.configs['object-server'] config_path = obj_conf[number] options = utils.readconf(config_path, 'app:object-server') swift_dir = options.get('swift_dir', '/etc/swift') ring = POLICIES.get_object_ring(int(self.policy), swift_dir) part, nodes = ring.get_nodes(account, container, obj) for node in nodes: # assumes one to one mapping if node['port'] == int(options.get('bind_port')): device = node['device'] break else: return None mgr = DiskFileManager(options, get_logger(options)) disk_file = mgr.get_diskfile(device, part, account, container, obj, self.policy) info = disk_file.read_metadata() return info
def __init__(self, conf, logger=None): # location/directory of the metadata database (meta.db) self.location = conf.get('location', '/srv/node/sdb1/metadata/') # path the the actual file #self.db_file = os.path.join(self.location, 'meta.db') self.logger = logger or get_logger(conf, log_route='metadata-server') self.root = conf.get('devices', '/srv/node') #workaround for device listings self.node_count = conf.get('nodecount', '8') self.devicelist = [] for x in range(0, int(self.node_count)): self.devicelist.append(conf.get('device' + str(x), '')) self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('node_timeout', 3)) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, MetadataBroker, self.mount_check, logger=self.logger) self.diskfile_mgr = DiskFileManager(conf, self.logger) self.db_ip = conf.get('db_ip', '127.0.0.1') self.db_port = int(conf.get('db_port', 2424)) self.db_user = conf.get('db_user', 'root') self.db_pw = conf.get('db_pw', 'root') if config_true_value(conf.get('allow_versions', 'f')): self.save_headers.append('x-versions-location') swift.common.db.DB_PREALLOCATION = config_true_value( conf.get('db_preallocation', 'f')) self.broker = self._get_metadata_broker() # self.broker.initialize() #
def _check_obj_put_async_update_bad_redirect_headers(self, headers): policies = list(POLICIES) random.shuffle(policies) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } daemon = object_updater.ObjectUpdater(conf, logger=self.logger) async_dir = os.path.join(self.sda1, get_async_dir(policies[0])) os.mkdir(async_dir) dfmanager = DiskFileManager(conf, daemon.logger) ts_obj = next(self.ts_iter) self._write_async_update(dfmanager, ts_obj, policies[0]) orig_async_path, orig_async_data = self._check_async_file(async_dir) fake_responses = [ (301, headers), (301, headers), (301, headers), ] fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests, ts_obj, policies[0]) self.assertEqual(['/sda1/0/a/c/o'] * 3, [req['path'] for req in conn.requests]) self.assertEqual({ 'failures': 1, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) # async file still intact async_path, async_data = self._check_async_file(async_dir) self.assertEqual(orig_async_path, async_path) self.assertEqual(orig_async_data, async_data) return daemon
def setUp(self): self.testdir = os.path.join(mkdtemp(), 'tmp_test_object_auditor') self.devices = os.path.join(self.testdir, 'node') self.logger = FakeLogger() rmtree(self.testdir, ignore_errors=1) mkdirs(os.path.join(self.devices, 'sda')) self.objects = os.path.join(self.devices, 'sda', 'objects') os.mkdir(os.path.join(self.devices, 'sdb')) self.objects_2 = os.path.join(self.devices, 'sdb', 'objects') os.mkdir(self.objects) self.parts = {} for part in ['0', '1', '2', '3']: self.parts[part] = os.path.join(self.objects, part) os.mkdir(os.path.join(self.objects, part)) self.conf = dict(devices=self.devices, mount_check='false', object_size_stats='10,100,1024,10240') self.df_mgr = DiskFileManager(self.conf, self.logger) self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o')
def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf['replication_semaphore'][0] else: self.replication_semaphore = None self.replication_failure_threshold = int( conf.get('replication_failure_threshold') or 100) self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0)
def test_obj_put_async_shard_update_redirected_twice(self): policies = list(POLICIES) random.shuffle(policies) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } daemon = object_updater.ObjectUpdater(conf, logger=self.logger) async_dir = os.path.join(self.sda1, get_async_dir(policies[0])) os.mkdir(async_dir) dfmanager = DiskFileManager(conf, daemon.logger) ts_obj = next(self.ts_iter) self._write_async_update(dfmanager, ts_obj, policies[0], container_path='.shards_a/c_shard_older') orig_async_path, orig_async_data = self._check_async_file(async_dir) # run once ts_redirect_1 = next(self.ts_iter) ts_redirect_2 = next(self.ts_iter) ts_redirect_3 = next(self.ts_iter) fake_responses = [ # 1st round of redirects, newest redirect should be chosen (301, { 'Location': '/.shards_a/c_shard_old/o', 'X-Backend-Redirect-Timestamp': ts_redirect_1.internal }), (301, { 'Location': '/.shards_a/c_shard_new/o', 'X-Backend-Redirect-Timestamp': ts_redirect_2.internal }), (301, { 'Location': '/.shards_a/c_shard_old/o', 'X-Backend-Redirect-Timestamp': ts_redirect_1.internal }), # 2nd round of redirects (301, { 'Location': '/.shards_a/c_shard_newer/o', 'X-Backend-Redirect-Timestamp': ts_redirect_3.internal }), (301, { 'Location': '/.shards_a/c_shard_newer/o', 'X-Backend-Redirect-Timestamp': ts_redirect_3.internal }), (301, { 'Location': '/.shards_a/c_shard_newer/o', 'X-Backend-Redirect-Timestamp': ts_redirect_3.internal }), ] fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests, ts_obj, policies[0]) # only *one* set of redirected requests is attempted per cycle older_part = daemon.container_ring.get_part('.shards_a/c_shard_older') new_part = daemon.container_ring.get_part('.shards_a/c_shard_new') newer_part = daemon.container_ring.get_part('.shards_a/c_shard_newer') self.assertEqual( ['/sda1/%s/.shards_a/c_shard_older/o' % older_part] * 3 + ['/sda1/%s/.shards_a/c_shard_new/o' % new_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual({ 'redirects': 2, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) # update failed, we still have pending file with most recent redirect # response Location header value added to data async_path, async_data = self._check_async_file(async_dir) self.assertEqual(orig_async_path, async_path) self.assertEqual( dict(orig_async_data, container_path='.shards_a/c_shard_newer', redirect_history=[ '.shards_a/c_shard_new', '.shards_a/c_shard_newer' ]), async_data) # next cycle, should get latest redirect from pickled async update fake_responses = [(200, {})] * 3 fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests, ts_obj, policies[0]) self.assertEqual(['/sda1/%s/.shards_a/c_shard_newer/o' % newer_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual( { 'redirects': 2, 'successes': 1, 'unlinks': 1, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) self.assertFalse(os.listdir(async_dir)) # no async file
def test_obj_put_async_update_redirection_loop(self): policies = list(POLICIES) random.shuffle(policies) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } daemon = object_updater.ObjectUpdater(conf, logger=self.logger) async_dir = os.path.join(self.sda1, get_async_dir(policies[0])) os.mkdir(async_dir) dfmanager = DiskFileManager(conf, daemon.logger) ts_obj = next(self.ts_iter) self._write_async_update(dfmanager, ts_obj, policies[0]) orig_async_path, orig_async_data = self._check_async_file(async_dir) # run once ts_redirect = next(self.ts_iter) resp_headers_1 = { 'Location': '/.shards_a/c_shard_1/o', 'X-Backend-Redirect-Timestamp': ts_redirect.internal } resp_headers_2 = { 'Location': '/.shards_a/c_shard_2/o', 'X-Backend-Redirect-Timestamp': ts_redirect.internal } fake_responses = ( # 1st round of redirects, 2nd round of redirects [(301, resp_headers_1)] * 3 + [(301, resp_headers_2)] * 3) fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests[:3], ts_obj, policies[0]) self._check_update_requests(conn.requests[3:], ts_obj, policies[0]) # only *one* set of redirected requests is attempted per cycle root_part = daemon.container_ring.get_part('a/c') shard_1_part = daemon.container_ring.get_part('.shards_a/c_shard_1') shard_2_part = daemon.container_ring.get_part('.shards_a/c_shard_2') shard_3_part = daemon.container_ring.get_part('.shards_a/c_shard_3') self.assertEqual(['/sda1/%s/a/c/o' % root_part] * 3 + ['/sda1/%s/.shards_a/c_shard_1/o' % shard_1_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual({ 'redirects': 2, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) # update failed, we still have pending file with most recent redirect # response Location header value added to data async_path, async_data = self._check_async_file(async_dir) self.assertEqual(orig_async_path, async_path) self.assertEqual( dict(orig_async_data, container_path='.shards_a/c_shard_2', redirect_history=[ '.shards_a/c_shard_1', '.shards_a/c_shard_2' ]), async_data) # next cycle, more redirects! first is to previously visited location resp_headers_3 = { 'Location': '/.shards_a/c_shard_3/o', 'X-Backend-Redirect-Timestamp': ts_redirect.internal } fake_responses = ( # 1st round of redirects, 2nd round of redirects [(301, resp_headers_1)] * 3 + [(301, resp_headers_3)] * 3) fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests[:3], ts_obj, policies[0]) self._check_update_requests(conn.requests[3:], ts_obj, policies[0]) # first try the previously persisted container path, response to that # creates a loop so ignore and send to root self.assertEqual( ['/sda1/%s/.shards_a/c_shard_2/o' % shard_2_part] * 3 + ['/sda1/%s/a/c/o' % root_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual({ 'redirects': 4, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) # update failed, we still have pending file with most recent redirect # response Location header value from root added to persisted data async_path, async_data = self._check_async_file(async_dir) self.assertEqual(orig_async_path, async_path) # note: redirect_history was reset when falling back to root self.assertEqual( dict(orig_async_data, container_path='.shards_a/c_shard_3', redirect_history=['.shards_a/c_shard_3']), async_data) # next cycle, more redirects! first is to a location visited previously # but not since last fall back to root, so that location IS tried; # second is to a location visited since last fall back to root so that # location is NOT tried fake_responses = ( # 1st round of redirects, 2nd round of redirects [(301, resp_headers_1)] * 3 + [(301, resp_headers_3)] * 3) fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests, ts_obj, policies[0]) self.assertEqual( ['/sda1/%s/.shards_a/c_shard_3/o' % shard_3_part] * 3 + ['/sda1/%s/.shards_a/c_shard_1/o' % shard_1_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual({ 'redirects': 6, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) # update failed, we still have pending file, but container_path is None # because most recent redirect location was a repeat async_path, async_data = self._check_async_file(async_dir) self.assertEqual(orig_async_path, async_path) self.assertEqual( dict(orig_async_data, container_path=None, redirect_history=[]), async_data) # next cycle, persisted container path is None so update should go to # root, this time it succeeds fake_responses = [(200, {})] * 3 fake_status_codes, fake_headers = zip(*fake_responses) with mocked_http_conn(*fake_status_codes, headers=fake_headers) as conn: with mock.patch('swift.obj.updater.dump_recon_cache'): daemon.run_once() self._check_update_requests(conn.requests, ts_obj, policies[0]) self.assertEqual(['/sda1/%s/a/c/o' % root_part] * 3, [req['path'] for req in conn.requests]) self.assertEqual( { 'redirects': 6, 'successes': 1, 'unlinks': 1, 'async_pendings': 1 }, daemon.logger.get_increment_counts()) self.assertFalse(os.listdir(async_dir)) # no async file