def get_to_final_state(): replicators = Manager(["account-replicator", "container-replicator", "object-replicator"]) replicators.stop() updaters = Manager(["container-updater", "object-updater"]) updaters.stop() replicators.once() updaters.once() replicators.once()
def get_to_final_state(): replicators = Manager(['account-replicator', 'container-replicator', 'object-replicator']) replicators.stop() updaters = Manager(['container-updater', 'object-updater']) updaters.stop() replicators.once() updaters.once() replicators.once()
def test_main(self): reconciler = Manager(['container-reconciler']) with spawn_services(self.ip_ports) as q: reconciler.start() # wait for the reconciler to connect q.get() # once it's hung in our connection - send it sig term print('Attempting to stop reconciler!') reconciler.stop() self.assertEqual(1, reconciler.status())
class BrainSplitter(object): __metaclass__ = meta_command def __init__(self, url, token, container_name='test', object_name='test'): self.url = url self.token = token self.account = utils.split_path(urlparse(url).path, 2, 2)[1] self.container_name = container_name self.object_name = object_name self.servers = Manager(['container-server']) policies = list(POLICIES) random.shuffle(policies) self.policies = itertools.cycle(policies) container_part, container_nodes = ring.Ring( '/etc/swift/container.ring.gz').get_nodes( self.account, self.container_name) container_node_ids = [n['id'] for n in container_nodes] if all(n_id in container_node_ids for n_id in (0, 1)): self.primary_numbers = (1, 2) self.handoff_numbers = (3, 4) else: self.primary_numbers = (3, 4) self.handoff_numbers = (1, 2) @command def start_primary_half(self): """ start container servers 1 & 2 """ tuple(self.servers.start(number=n) for n in self.primary_numbers) @command def stop_primary_half(self): """ stop container servers 1 & 2 """ tuple(self.servers.stop(number=n) for n in self.primary_numbers) @command def start_handoff_half(self): """ start container servers 3 & 4 """ tuple(self.servers.start(number=n) for n in self.handoff_numbers) @command def stop_handoff_half(self): """ stop container servers 3 & 4 """ tuple(self.servers.stop(number=n) for n in self.handoff_numbers) @command def put_container(self, policy_index=None): """ put container with next storage policy """ policy = self.policies.next() if policy_index is not None: policy = POLICIES.get_by_index(int(policy_index)) if not policy: raise ValueError('Unknown policy with index %s' % policy) headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) @command def delete_container(self): """ delete container """ client.delete_container(self.url, self.token, self.container_name) @command def put_object(self, headers=None): """ issue put for zero byte test object """ client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers) @command def delete_object(self): """ issue delete for test object """ try: client.delete_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def setUp(self): resetswift() self.pids = {} try: self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring( "account", self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs, ) self.container_ring = get_ring( "container", self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs, ) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server="object", ipport2server=self.ipport2server, config_paths=self.configs, ) self.servers_per_port = any( int(readconf(c, section_name="object-replicator").get("servers_per_port", "0")) for c in self.configs["object-replicator"].values() ) Manager(["main"]).start(wait=False) for ipport in self.ipport2server: check_server(ipport, self.ipport2server, self.pids) proxy_ipport = ("127.0.0.1", 8080) self.ipport2server[proxy_ipport] = "proxy" self.url, self.token, self.account = check_server(proxy_ipport, self.ipport2server, self.pids) self.replicators = Manager(["account-replicator", "container-replicator", "object-replicator"]) self.updaters = Manager(["container-updater", "object-updater"]) except BaseException: try: raise finally: try: Manager(["all"]).kill() except Exception: pass def tearDown(self): Manager(["all"]).kill() def device_dir(self, server, node): server_type, config_number = get_server_number((node["ip"], node["port"]), self.ipport2server) repl_server = "%s-replicator" % server_type conf = readconf(self.configs[repl_server][config_number], section_name=repl_server) return os.path.join(conf["devices"], node["device"]) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number((node["ip"], node["port"]), self.ipport2server) return config_number def is_local_to(self, node1, node2): """ Return True if both ring devices are "local" to each other (on the same "server". """ if self.servers_per_port: return node1["ip"] == node2["ip"] # Without a disambiguating IP, for SAIOs, we have to assume ports # uniquely identify "servers". SAIOs should be configured to *either* # have unique IPs per node (e.g. 127.0.0.1, 127.0.0.2, etc.) OR unique # ports per server (i.e. sdb1 & sdb5 would have same port numbers in # the 8-disk EC ring). return node1["port"] == node2["port"] def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once() def kill_drive(self, device): if os.path.ismount(device): os.system("sudo umount %s" % device) else: renamer(device, device + "X") def revive_drive(self, device): disabled_name = device + "X" if os.path.isdir(disabled_name): renamer(device + "X", device) else: os.system("sudo mount %s" % device)
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def setUp(self): p = Popen("resetswift 2>&1", shell=True, stdout=PIPE) stdout, _stderr = p.communicate() print stdout Manager(['all']).stop() self.pids = {} try: self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object') Manager(['main']).start(wait=False) self.port2server = {} for server, port in [('account', 6002), ('container', 6001), ('object', 6000)]: for number in xrange(1, 9): self.port2server[port + (number * 10)] = \ '%s%d' % (server, number) for port in self.port2server: check_server(port, self.port2server, self.pids) self.port2server[8080] = 'proxy' self.url, self.token, self.account = \ check_server(8080, self.port2server, self.pids) self.configs = defaultdict(dict) for name in ('account', 'container', 'object'): for server_name in (name, '%s-replicator' % name): for server in Manager([server_name]): for i, conf in enumerate(server.conf_files(), 1): self.configs[server.server][i] = conf self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) self.server_port_to_conf = {} # get some configs backend daemon configs loaded up for server in ('account', 'container', 'object'): self.server_port_to_conf[server] = build_port_to_conf(server) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass def tearDown(self): Manager(['all']).kill() def device_dir(self, server, node): conf = self.server_port_to_conf[server][node['port']] return os.path.join(conf['devices'], node['device']) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number( node['port'], self.port2server) return config_number def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once()
class TestObjectExpirer(unittest.TestCase): def setUp(self): if len(POLICIES) < 2: raise SkipTest('Need more than one policy') self.expirer = Manager(['object-expirer']) self.expirer.start() err = self.expirer.stop() if err: raise SkipTest('Unable to verify object-expirer service') conf_files = [] for server in self.expirer.servers: conf_files.extend(server.conf_files()) conf_file = conf_files[0] self.client = InternalClient(conf_file, 'probe-test', 3) (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.policy, self.url, self.token, self.account, self.configs) = reset_environment() self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name) def test_expirer_object_split_brain(self): old_policy = random.choice(list(POLICIES)) wrong_policy = random.choice([p for p in POLICIES if p != old_policy]) # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up Manager(['object-updater']).once() self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # don't start handoff servers, only wrong policy is available # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # this guy should no-op since it's unable to expire the object self.expirer.once() self.brain.start_handoff_half() get_to_final_state() # validate object is expired found_in_policy = None metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assert_('x-backend-timestamp' in metadata) self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: break else: self.fail('Did not find listing for %s' % self.object_name) # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # run the expirier again after replication self.expirer.once() # object is not in the listing for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: self.fail('Found listing for %s' % self.object_name) # and validate object is tombstoned found_in_policy = None for policy in POLICIES: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(policy)}) if 'x-backend-timestamp' in metadata: if found_in_policy: self.fail('found object in %s and also %s' % (found_in_policy, policy)) found_in_policy = policy self.assert_('x-backend-timestamp' in metadata) self.assert_(Timestamp(metadata['x-backend-timestamp']) > create_timestamp)
class BrainSplitter(object): __metaclass__ = meta_command def __init__(self, url, token, container_name='test', object_name='test', server_type='container', policy=None): self.url = url self.token = token self.account = utils.split_path(urlparse(url).path, 2, 2)[1] self.container_name = container_name self.object_name = object_name server_list = ['%s-server' % server_type] if server_type else ['all'] self.servers = Manager(server_list) policies = list(ENABLED_POLICIES) random.shuffle(policies) self.policies = itertools.cycle(policies) o = object_name if server_type == 'object' else None c = container_name if server_type in ('object', 'container') else None if server_type in ('container', 'account'): if policy: raise TypeError('Metadata server brains do not ' 'support specific storage policies') self.policy = None self.ring = ring.Ring( '/etc/swift/%s.ring.gz' % server_type) elif server_type == 'object': if not policy: raise TypeError('Object BrainSplitters need to ' 'specify the storage policy') self.policy = policy policy.load_ring('/etc/swift') self.ring = policy.object_ring else: raise ValueError('Unkonwn server_type: %r' % server_type) self.server_type = server_type part, nodes = self.ring.get_nodes(self.account, c, o) node_ids = [n['id'] for n in nodes] if all(n_id in node_ids for n_id in (0, 1)): self.primary_numbers = (1, 2) self.handoff_numbers = (3, 4) else: self.primary_numbers = (3, 4) self.handoff_numbers = (1, 2) @command def start_primary_half(self): """ start servers 1 & 2 """ tuple(self.servers.start(number=n) for n in self.primary_numbers) @command def stop_primary_half(self): """ stop servers 1 & 2 """ tuple(self.servers.stop(number=n) for n in self.primary_numbers) @command def start_handoff_half(self): """ start servers 3 & 4 """ tuple(self.servers.start(number=n) for n in self.handoff_numbers) @command def stop_handoff_half(self): """ stop servers 3 & 4 """ tuple(self.servers.stop(number=n) for n in self.handoff_numbers) @command def put_container(self, policy_index=None): """ put container with next storage policy """ policy = self.policies.next() if policy_index is not None: policy = POLICIES.get_by_index(int(policy_index)) if not policy: raise ValueError('Unknown policy with index %s' % policy) headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) @command def delete_container(self): """ delete container """ client.delete_container(self.url, self.token, self.container_name) @command def put_object(self, headers=None): """ issue put for zero byte test object """ client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers) @command def delete_object(self): """ issue delete for test object """ try: client.delete_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def _load_rings_and_configs(self): self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring('account', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.container_ring = get_ring('container', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring(self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object', ipport2server=self.ipport2server, config_paths=self.configs) for server in Manager(['proxy-server']): for conf in server.conf_files(): self.configs['proxy-server'] = conf def setUp(self): resetswift() kill_orphans() self._load_rings_and_configs() try: self.servers_per_port = any( int( readconf(c, section_name='object-replicator').get( 'servers_per_port', '0')) for c in self.configs['object-replicator'].values()) Manager(['main']).start(wait=True) for ipport in self.ipport2server: check_server(ipport, self.ipport2server) proxy_conf = readconf(self.configs['proxy-server'], section_name='app:proxy-server') proxy_ipport = (proxy_conf.get('bind_ip', '127.0.0.1'), int(proxy_conf.get('bind_port', 8080))) self.ipport2server[proxy_ipport] = 'proxy' self.url, self.token, self.account = check_server( proxy_ipport, self.ipport2server) self.account_1 = { 'url': self.url, 'token': self.token, 'account': self.account } rv = _retry_timeout(_check_proxy, args=(proxy_ipport, 'test2:tester2', 'testing2')) self.account_2 = { k: v for (k, v) in zip(('url', 'token', 'account'), rv) } self.replicators = Manager([ 'account-replicator', 'container-replicator', 'object-replicator' ]) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass info_url = "%s://%s/info" % (urlparse( self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) self.cluster_info = client.get_capabilities(proxy_conn) def tearDown(self): Manager(['all']).kill() def device_dir(self, server, node): server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) repl_server = '%s-replicator' % server_type conf = readconf(self.configs[repl_server][config_number], section_name=repl_server) return os.path.join(conf['devices'], node['device']) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) return config_number def is_local_to(self, node1, node2): """ Return True if both ring devices are "local" to each other (on the same "server". """ if self.servers_per_port: return node1['ip'] == node2['ip'] # Without a disambiguating IP, for SAIOs, we have to assume ports # uniquely identify "servers". SAIOs should be configured to *either* # have unique IPs per node (e.g. 127.0.0.1, 127.0.0.2, etc.) OR unique # ports per server (i.e. sdb1 & sdb5 would have same port numbers in # the 8-disk EC ring). return node1['port'] == node2['port'] def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once() def kill_drive(self, device): if os.path.ismount(device): os.system('sudo umount %s' % device) else: renamer(device, device + "X") def revive_drive(self, device): disabled_name = device + "X" if os.path.isdir(disabled_name): renamer(disabled_name, device) else: os.system('sudo mount %s' % device) def make_internal_client(self): tempdir = mkdtemp() try: conf_path = os.path.join(tempdir, 'internal_client.conf') conf_body = """ [DEFAULT] swift_dir = /etc/swift [pipeline:main] pipeline = catch_errors cache copy proxy-server [app:proxy-server] use = egg:swift#proxy allow_account_management = True [filter:copy] use = egg:swift#copy [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ with open(conf_path, 'w') as f: f.write(dedent(conf_body)) return internal_client.InternalClient(conf_path, 'test', 1) finally: shutil.rmtree(tempdir) def get_all_object_nodes(self): """ Returns a list of all nodes in all object storage policies. :return: a list of node dicts. """ all_obj_nodes = {} for policy in ENABLED_POLICIES: for dev in policy.object_ring.devs: all_obj_nodes[dev['device']] = dev return all_obj_nodes.values() def gather_async_pendings(self, onodes): """ Returns a list of paths to async pending files found on given nodes. :param onodes: a list of nodes. :return: a list of file paths. """ async_pendings = [] for onode in onodes: device_dir = self.device_dir('', onode) for ap_pol_dir in os.listdir(device_dir): if not ap_pol_dir.startswith('async_pending'): # skip 'objects', 'containers', etc. continue async_pending_dir = os.path.join(device_dir, ap_pol_dir) try: ap_dirs = os.listdir(async_pending_dir) except OSError as err: if err.errno == errno.ENOENT: pass else: raise else: for ap_dir in ap_dirs: ap_dir_fullpath = os.path.join(async_pending_dir, ap_dir) async_pendings.extend([ os.path.join(ap_dir_fullpath, ent) for ent in os.listdir(ap_dir_fullpath) ]) return async_pendings
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def setUp(self): resetswift() self.pids = {} try: self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object', ipport2server=self.ipport2server, config_paths=self.configs) self.servers_per_port = any( int(readconf(c, section_name='object-replicator').get( 'servers_per_port', '0')) for c in self.configs['object-replicator'].values()) Manager(['main']).start(wait=False) for ipport in self.ipport2server: check_server(ipport, self.ipport2server, self.pids) proxy_ipport = ('127.0.0.1', 8080) self.ipport2server[proxy_ipport] = 'proxy' self.url, self.token, self.account = check_server( proxy_ipport, self.ipport2server, self.pids) self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass def tearDown(self): Manager(['all']).kill() def device_dir(self, server, node): server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) repl_server = '%s-replicator' % server_type conf = readconf(self.configs[repl_server][config_number], section_name=repl_server) return os.path.join(conf['devices'], node['device']) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) return config_number def is_local_to(self, node1, node2): """ Return True if both ring devices are "local" to each other (on the same "server". """ if self.servers_per_port: return node1['ip'] == node2['ip'] # Without a disambiguating IP, for SAIOs, we have to assume ports # uniquely identify "servers". SAIOs should be configured to *either* # have unique IPs per node (e.g. 127.0.0.1, 127.0.0.2, etc.) OR unique # ports per server (i.e. sdb1 & sdb5 would have same port numbers in # the 8-disk EC ring). return node1['port'] == node2['port'] def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once() def kill_drive(self, device): if os.path.ismount(device): os.system('sudo umount %s' % device) else: renamer(device, device + "X") def revive_drive(self, device): disabled_name = device + "X" if os.path.isdir(disabled_name): renamer(device + "X", device) else: os.system('sudo mount %s' % device)
class BrainSplitter(object): __metaclass__ = meta_command def __init__(self, url, token, container_name='test', object_name='test', server_type='container'): self.url = url self.token = token self.account = utils.split_path(urlparse(url).path, 2, 2)[1] self.container_name = container_name self.object_name = object_name server_list = ['%s-server' % server_type] if server_type else ['all'] self.servers = Manager(server_list) policies = list(ENABLED_POLICIES) random.shuffle(policies) self.policies = itertools.cycle(policies) o = object_name if server_type == 'object' else None c = container_name if server_type in ('object', 'container') else None part, nodes = ring.Ring( '/etc/swift/%s.ring.gz' % server_type).get_nodes( self.account, c, o) node_ids = [n['id'] for n in nodes] if all(n_id in node_ids for n_id in (0, 1)): self.primary_numbers = (1, 2) self.handoff_numbers = (3, 4) else: self.primary_numbers = (3, 4) self.handoff_numbers = (1, 2) @command def start_primary_half(self): """ start servers 1 & 2 """ tuple(self.servers.start(number=n) for n in self.primary_numbers) @command def stop_primary_half(self): """ stop servers 1 & 2 """ tuple(self.servers.stop(number=n) for n in self.primary_numbers) @command def start_handoff_half(self): """ start servers 3 & 4 """ tuple(self.servers.start(number=n) for n in self.handoff_numbers) @command def stop_handoff_half(self): """ stop servers 3 & 4 """ tuple(self.servers.stop(number=n) for n in self.handoff_numbers) @command def put_container(self, policy_index=None): """ put container with next storage policy """ policy = self.policies.next() if policy_index is not None: policy = POLICIES.get_by_index(int(policy_index)) if not policy: raise ValueError('Unknown policy with index %s' % policy) headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) @command def delete_container(self): """ delete container """ client.delete_container(self.url, self.token, self.container_name) @command def put_object(self, headers=None): """ issue put for zero byte test object """ client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers) @command def delete_object(self): """ issue delete for test object """ try: client.delete_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def _load_rings_and_configs(self): self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object', ipport2server=self.ipport2server, config_paths=self.configs) def setUp(self): resetswift() kill_orphans() self._load_rings_and_configs() try: self.servers_per_port = any( int(readconf(c, section_name='object-replicator').get( 'servers_per_port', '0')) for c in self.configs['object-replicator'].values()) Manager(['main']).start(wait=True) for ipport in self.ipport2server: check_server(ipport, self.ipport2server) proxy_ipport = ('127.0.0.1', 8080) self.ipport2server[proxy_ipport] = 'proxy' self.url, self.token, self.account = check_server( proxy_ipport, self.ipport2server) self.account_1 = { 'url': self.url, 'token': self.token, 'account': self.account} rv = _retry_timeout(_check_proxy, args=( proxy_ipport, 'test2:tester2', 'testing2')) self.account_2 = { k: v for (k, v) in zip(('url', 'token', 'account'), rv)} self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass info_url = "%s://%s/info" % (urlparse(self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) self.cluster_info = client.get_capabilities(proxy_conn) def tearDown(self): Manager(['all']).kill() def device_dir(self, server, node): server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) repl_server = '%s-replicator' % server_type conf = readconf(self.configs[repl_server][config_number], section_name=repl_server) return os.path.join(conf['devices'], node['device']) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) return config_number def is_local_to(self, node1, node2): """ Return True if both ring devices are "local" to each other (on the same "server". """ if self.servers_per_port: return node1['ip'] == node2['ip'] # Without a disambiguating IP, for SAIOs, we have to assume ports # uniquely identify "servers". SAIOs should be configured to *either* # have unique IPs per node (e.g. 127.0.0.1, 127.0.0.2, etc.) OR unique # ports per server (i.e. sdb1 & sdb5 would have same port numbers in # the 8-disk EC ring). return node1['port'] == node2['port'] def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once() def kill_drive(self, device): if os.path.ismount(device): os.system('sudo umount %s' % device) else: renamer(device, device + "X") def revive_drive(self, device): disabled_name = device + "X" if os.path.isdir(disabled_name): renamer(disabled_name, device) else: os.system('sudo mount %s' % device) def make_internal_client(self): tempdir = mkdtemp() try: conf_path = os.path.join(tempdir, 'internal_client.conf') conf_body = """ [DEFAULT] swift_dir = /etc/swift [pipeline:main] pipeline = catch_errors cache copy proxy-server [app:proxy-server] use = egg:swift#proxy [filter:copy] use = egg:swift#copy [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ with open(conf_path, 'w') as f: f.write(dedent(conf_body)) return internal_client.InternalClient(conf_path, 'test', 1) finally: shutil.rmtree(tempdir) def get_all_object_nodes(self): """ Returns a list of all nodes in all object storage policies. :return: a list of node dicts. """ all_obj_nodes = {} for policy in ENABLED_POLICIES: for dev in policy.object_ring.devs: all_obj_nodes[dev['device']] = dev return all_obj_nodes.values() def gather_async_pendings(self, onodes): """ Returns a list of paths to async pending files found on given nodes. :param onodes: a list of nodes. :return: a list of file paths. """ async_pendings = [] for onode in onodes: device_dir = self.device_dir('', onode) for ap_pol_dir in os.listdir(device_dir): if not ap_pol_dir.startswith('async_pending'): # skip 'objects', 'containers', etc. continue async_pending_dir = os.path.join(device_dir, ap_pol_dir) try: ap_dirs = os.listdir(async_pending_dir) except OSError as err: if err.errno == errno.ENOENT: pass else: raise else: for ap_dir in ap_dirs: ap_dir_fullpath = os.path.join( async_pending_dir, ap_dir) async_pendings.extend([ os.path.join(ap_dir_fullpath, ent) for ent in os.listdir(ap_dir_fullpath)]) return async_pendings
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def setUp(self): resetswift() try: self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object', ipport2server=self.ipport2server, config_paths=self.configs) self.servers_per_port = any( int(readconf(c, section_name='object-replicator').get( 'servers_per_port', '0')) for c in self.configs['object-replicator'].values()) Manager(['main']).start(wait=True) for ipport in self.ipport2server: check_server(ipport, self.ipport2server) proxy_ipport = ('127.0.0.1', 8080) self.ipport2server[proxy_ipport] = 'proxy' self.url, self.token, self.account = check_server( proxy_ipport, self.ipport2server) self.account_1 = { 'url': self.url, 'token': self.token, 'account': self.account} rv = _retry_timeout(_check_proxy, args=( proxy_ipport, 'test2:tester2', 'testing2')) self.account_2 = { k: v for (k, v) in zip(('url', 'token', 'account'), rv)} self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass def tearDown(self): Manager(['all']).kill() def device_dir(self, server, node): server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) repl_server = '%s-replicator' % server_type conf = readconf(self.configs[repl_server][config_number], section_name=repl_server) return os.path.join(conf['devices'], node['device']) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) return config_number def is_local_to(self, node1, node2): """ Return True if both ring devices are "local" to each other (on the same "server". """ if self.servers_per_port: return node1['ip'] == node2['ip'] # Without a disambiguating IP, for SAIOs, we have to assume ports # uniquely identify "servers". SAIOs should be configured to *either* # have unique IPs per node (e.g. 127.0.0.1, 127.0.0.2, etc.) OR unique # ports per server (i.e. sdb1 & sdb5 would have same port numbers in # the 8-disk EC ring). return node1['port'] == node2['port'] def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once() def kill_drive(self, device): if os.path.ismount(device): os.system('sudo umount %s' % device) else: renamer(device, device + "X") def revive_drive(self, device): disabled_name = device + "X" if os.path.isdir(disabled_name): renamer(device + "X", device) else: os.system('sudo mount %s' % device) def make_internal_client(self): tempdir = mkdtemp() try: conf_path = os.path.join(tempdir, 'internal_client.conf') conf_body = """ [DEFAULT] swift_dir = /etc/swift [pipeline:main] pipeline = catch_errors cache copy proxy-server [app:proxy-server] use = egg:swift#proxy [filter:copy] use = egg:swift#copy [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ with open(conf_path, 'w') as f: f.write(dedent(conf_body)) return internal_client.InternalClient(conf_path, 'test', 1) finally: shutil.rmtree(tempdir)
class BrainSplitter(object): def __init__(self, url, token, container_name='test', object_name='test', server_type='container', policy=None): self.url = url self.token = token self.account = utils.split_path(urlparse(url).path, 2, 2)[1] self.container_name = container_name self.object_name = object_name server_list = ['%s-server' % server_type] if server_type else ['all'] self.servers = Manager(server_list) policies = list(ENABLED_POLICIES) random.shuffle(policies) self.policies = itertools.cycle(policies) o = object_name if server_type == 'object' else None c = container_name if server_type in ('object', 'container') else None if server_type in ('container', 'account'): if policy: raise TypeError('Metadata server brains do not ' 'support specific storage policies') self.policy = None self.ring = ring.Ring('/etc/swift/%s.ring.gz' % server_type) elif server_type == 'object': if not policy: raise TypeError('Object BrainSplitters need to ' 'specify the storage policy') self.policy = policy policy.load_ring('/etc/swift') self.ring = policy.object_ring else: raise ValueError('Unknown server_type: %r' % server_type) self.server_type = server_type self.part, self.nodes = self.ring.get_nodes(self.account, c, o) self.node_numbers = [n['id'] + 1 for n in self.nodes] if 1 in self.node_numbers and 2 in self.node_numbers: self.primary_numbers = (1, 2) self.handoff_numbers = (3, 4) else: self.primary_numbers = (3, 4) self.handoff_numbers = (1, 2) @command def start_primary_half(self): """ start servers 1 & 2 """ tuple(self.servers.start(number=n) for n in self.primary_numbers) @command def stop_primary_half(self): """ stop servers 1 & 2 """ tuple(self.servers.stop(number=n) for n in self.primary_numbers) @command def start_handoff_half(self): """ start servers 3 & 4 """ tuple(self.servers.start(number=n) for n in self.handoff_numbers) @command def stop_handoff_half(self): """ stop servers 3 & 4 """ tuple(self.servers.stop(number=n) for n in self.handoff_numbers) @command def put_container(self, policy_index=None): """ put container with next storage policy """ if policy_index is not None: policy = POLICIES.get_by_index(int(policy_index)) if not policy: raise ValueError('Unknown policy with index %s' % policy) elif not self.policy: policy = next(self.policies) else: policy = self.policy headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) @command def delete_container(self): """ delete container """ client.delete_container(self.url, self.token, self.container_name) @command def put_object(self, headers=None, contents=None): """ issue put for test object """ client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers, contents=contents) @command def delete_object(self): """ issue delete for test object """ try: client.delete_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def setUp(self): resetswift() self.pids = {} try: self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object', ipport2server=self.ipport2server, config_paths=self.configs) self.servers_per_port = any( int(readconf(c, section_name='object-replicator').get( 'servers_per_port', '0')) for c in self.configs['object-replicator'].values()) Manager(['main']).start(wait=False) for ipport in self.ipport2server: check_server(ipport, self.ipport2server, self.pids) proxy_ipport = ('127.0.0.1', 8080) self.ipport2server[proxy_ipport] = 'proxy' self.url, self.token, self.account = check_server( proxy_ipport, self.ipport2server, self.pids) self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass def tearDown(self): Manager(['all']).kill() def device_dir(self, server, node): server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) repl_server = '%s-replicator' % server_type conf = readconf(self.configs[repl_server][config_number], section_name=repl_server) return os.path.join(conf['devices'], node['device']) def storage_dir(self, server, node, part=None, policy=None): policy = policy or self.policy device_path = self.device_dir(server, node) path_parts = [device_path, get_data_dir(policy)] if part is not None: path_parts.append(str(part)) return os.path.join(*path_parts) def config_number(self, node): _server_type, config_number = get_server_number( (node['ip'], node['port']), self.ipport2server) return config_number def is_local_to(self, node1, node2): """ Return True if both ring devices are "local" to each other (on the same "server". """ if self.servers_per_port: return node1['ip'] == node2['ip'] # Without a disambiguating IP, for SAIOs, we have to assume ports # uniquely identify "servers". SAIOs should be configured to *either* # have unique IPs per node (e.g. 127.0.0.1, 127.0.0.2, etc.) OR unique # ports per server (i.e. sdb1 & sdb5 would have same port numbers in # the 8-disk EC ring). return node1['port'] == node2['port'] def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once() def kill_drive(self, device): if os.path.ismount(device): os.system('sudo umount %s' % device) else: renamer(device, device + "X") def revive_drive(self, device): disabled_name = device + "X" if os.path.isdir(disabled_name): renamer(device + "X", device) else: os.system('sudo mount %s' % device) def make_internal_client(self, object_post_as_copy=True): tempdir = mkdtemp() try: conf_path = os.path.join(tempdir, 'internal_client.conf') conf_body = """ [DEFAULT] swift_dir = /etc/swift [pipeline:main] pipeline = catch_errors cache proxy-server [app:proxy-server] use = egg:swift#proxy object_post_as_copy = %s [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ % object_post_as_copy with open(conf_path, 'w') as f: f.write(dedent(conf_body)) return internal_client.InternalClient(conf_path, 'test', 1) finally: shutil.rmtree(tempdir)
class ProbeTest(unittest.TestCase): """ Don't instantiate this directly, use a child class instead. """ def setUp(self): p = Popen("resetswift 2>&1", shell=True, stdout=PIPE) stdout, _stderr = p.communicate() print stdout Manager(['all']).stop() self.pids = {} try: self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object') Manager(['main']).start(wait=False) self.port2server = {} for server, port in [('account', 6002), ('container', 6001), ('object', 6000)]: for number in xrange(1, 9): self.port2server[port + (number * 10)] = \ '%s%d' % (server, number) for port in self.port2server: check_server(port, self.port2server, self.pids) self.port2server[8080] = 'proxy' self.url, self.token, self.account = \ check_server(8080, self.port2server, self.pids) self.configs = defaultdict(dict) for name in ('account', 'container', 'object'): for server_name in (name, '%s-replicator' % name): for server in Manager([server_name]): for i, conf in enumerate(server.conf_files(), 1): self.configs[server.server][i] = conf self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass def tearDown(self): Manager(['all']).kill() def get_to_final_state(self): # these .stop()s are probably not strictly necessary, # but may prevent race conditions self.replicators.stop() self.updaters.stop() self.replicators.once() self.updaters.once() self.replicators.once()
class TestObjectExpirer(ReplProbeTest): def setUp(self): self.expirer = Manager(['object-expirer']) self.expirer.start() err = self.expirer.stop() if err: raise unittest.SkipTest('Unable to verify object-expirer service') conf_files = [] for server in self.expirer.servers: conf_files.extend(server.conf_files()) conf_file = conf_files[0] self.client = InternalClient(conf_file, 'probe-test', 3) super(TestObjectExpirer, self).setUp() self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name) def _check_obj_in_container_listing(self): for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: return True return False @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") def test_expirer_object_split_brain(self): old_policy = random.choice(ENABLED_POLICIES) wrong_policy = random.choice( [p for p in ENABLED_POLICIES if p != old_policy]) # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up Manager(['object-updater']).once() self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # don't start handoff servers, only wrong policy is available # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # this guy should no-op since it's unable to expire the object self.expirer.once() self.brain.start_handoff_half() self.get_to_final_state() # validate object is expired found_in_policy = None metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4, ), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assertIn('x-backend-timestamp', metadata) self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing self.assertTrue(self._check_obj_in_container_listing(), msg='Did not find listing for %s' % self.object_name) # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # run the expirer again after replication self.expirer.once() # object is not in the listing self.assertFalse(self._check_obj_in_container_listing(), msg='Found listing for %s' % self.object_name) # and validate object is tombstoned found_in_policy = None for policy in ENABLED_POLICIES: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4, ), headers={'X-Backend-Storage-Policy-Index': int(policy)}) if 'x-backend-timestamp' in metadata: if found_in_policy: self.fail('found object in %s and also %s' % (found_in_policy, policy)) found_in_policy = policy self.assertIn('x-backend-timestamp', metadata) self.assertGreater(Timestamp(metadata['x-backend-timestamp']), create_timestamp) def test_expirer_doesnt_make_async_pendings(self): # The object expirer cleans up its own queue. The inner loop # basically looks like this: # # for obj in stuff_to_delete: # delete_the_object(obj) # remove_the_queue_entry(obj) # # By default, upon receipt of a DELETE request for an expiring # object, the object servers will create async_pending records to # clean the expirer queue. Since the expirer cleans its own queue, # this is unnecessary. The expirer can make requests in such a way # tha the object server does not write out any async pendings; this # test asserts that this is the case. def gather_async_pendings(onodes): async_pendings = [] for onode in onodes: device_dir = self.device_dir('', onode) for ap_pol_dir in os.listdir(device_dir): if not ap_pol_dir.startswith('async_pending'): # skip 'objects', 'containers', etc. continue async_pending_dir = os.path.join(device_dir, ap_pol_dir) try: ap_dirs = os.listdir(async_pending_dir) except OSError as err: if err.errno == errno.ENOENT: pass else: raise else: for ap_dir in ap_dirs: ap_dir_fullpath = os.path.join( async_pending_dir, ap_dir) async_pendings.extend([ os.path.join(ap_dir_fullpath, ent) for ent in os.listdir(ap_dir_fullpath) ]) return async_pendings # Make an expiring object in each policy for policy in ENABLED_POLICIES: container_name = "expirer-test-%d" % policy.idx container_headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, container_name, headers=container_headers) now = time.time() delete_at = int(now + 2.0) client.put_object(self.url, self.token, container_name, "some-object", headers={ 'X-Delete-At': str(delete_at), 'X-Timestamp': Timestamp(now).normal }, contents='dontcare') time.sleep(2.0) # make sure auto-created expirer-queue containers get in the account # listing so the expirer can find them Manager(['container-updater']).once() # Make sure there's no async_pendings anywhere. Probe tests only run # on single-node installs anyway, so this set should be small enough # that an exhaustive check doesn't take too long. all_obj_nodes = {} for policy in ENABLED_POLICIES: for dev in policy.object_ring.devs: all_obj_nodes[dev['device']] = dev pendings_before = gather_async_pendings(all_obj_nodes.values()) # expire the objects Manager(['object-expirer']).once() pendings_after = gather_async_pendings(all_obj_nodes.values()) self.assertEqual(pendings_after, pendings_before) def test_expirer_object_should_not_be_expired(self): # Current object-expirer checks the correctness via x-if-delete-at # header that it can be deleted by expirer. If there are objects # either which doesn't have x-delete-at header as metadata or which # has different x-delete-at value from x-if-delete-at value, # object-expirer's delete will fail as 412 PreconditionFailed. # However, if some of the objects are in handoff nodes, the expirer # can put the tombstone with the timestamp as same as x-delete-at and # the object consistency will be resolved as the newer timestamp will # be winner (in particular, overwritten case w/o x-delete-at). This # test asserts such a situation that, at least, the overwriten object # which have larger timestamp than the original expirered date should # be safe. def put_object(headers): # use internal client to PUT objects so that X-Timestamp in headers # is effective headers['Content-Length'] = '0' path = self.client.make_path(self.account, self.container_name, self.object_name) try: self.client.make_request('PUT', path, headers, (2, )) except UnexpectedResponse as e: self.fail('Expected 201 for PUT object but got %s' % e.resp.status) obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) # T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated) # < T(expirer_executed) # Recreated obj should be appeared in any split brain case obj_brain.put_container() # T(obj_deleted with x-delete-at) # object-server accepts req only if X-Delete-At is later than 'now' # so here, T(obj_created) < T(obj_deleted with x-delete-at) now = time.time() delete_at = int(now + 2.0) recreate_at = delete_at + 1.0 put_object(headers={ 'X-Delete-At': str(delete_at), 'X-Timestamp': Timestamp(now).normal }) # some object servers stopped to make a situation that the # object-expirer can put tombstone in the primary nodes. obj_brain.stop_primary_half() # increment the X-Timestamp explicitly # (will be T(obj_deleted with x-delete-at) < T(obj_recreated)) put_object( headers={ 'X-Object-Meta-Expired': 'False', 'X-Timestamp': Timestamp(recreate_at).normal }) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # sanity, the newer object is still there try: metadata = self.client.get_object_metadata(self.account, self.container_name, self.object_name) except UnexpectedResponse as e: self.fail('Expected 200 for HEAD object but got %s' % e.resp.status) self.assertIn('x-object-meta-expired', metadata) # some object servers recovered obj_brain.start_primary_half() # sleep until after recreated_at while time.time() <= recreate_at: time.sleep(0.1) # Now, expirer runs at the time after obj is recreated self.expirer.once() # verify that original object was deleted by expirer obj_brain.stop_handoff_half() try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4, )) except UnexpectedResponse as e: self.fail('Expected 404 for HEAD object but got %s' % e.resp.status) obj_brain.start_handoff_half() # and inconsistent state of objects is recovered by replicator Manager(['object-replicator']).once() # check if you can get recreated object try: metadata = self.client.get_object_metadata(self.account, self.container_name, self.object_name) except UnexpectedResponse as e: self.fail('Expected 200 for HEAD object but got %s' % e.resp.status) self.assertIn('x-object-meta-expired', metadata) def _test_expirer_delete_outdated_object_version(self, object_exists): # This test simulates a case where the expirer tries to delete # an outdated version of an object. # One case is where the expirer gets a 404, whereas the newest version # of the object is offline. # Another case is where the expirer gets a 412, since the old version # of the object mismatches the expiration time sent by the expirer. # In any of these cases, the expirer should retry deleting the object # later, for as long as a reclaim age has not passed. obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) obj_brain.put_container() if object_exists: obj_brain.put_object() # currently, the object either doesn't exist, or does not have # an expiration # stop primary servers and put a newer version of the object, this # time with an expiration. only the handoff servers will have # the new version obj_brain.stop_primary_half() now = time.time() delete_at = int(now + 2.0) obj_brain.put_object({'X-Delete-At': str(delete_at)}) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # update object record in the container listing Manager(['container-replicator']).once() # take handoff servers down, and bring up the outdated primary servers obj_brain.start_primary_half() obj_brain.stop_handoff_half() # wait until object expiration time while time.time() <= delete_at: time.sleep(0.1) # run expirer against the outdated servers. it should fail since # the outdated version does not match the expiration time self.expirer.once() # bring all servers up, and run replicator to update servers obj_brain.start_handoff_half() Manager(['object-replicator']).once() # verify the deletion has failed by checking the container listing self.assertTrue(self._check_obj_in_container_listing(), msg='Did not find listing for %s' % self.object_name) # run expirer again, delete should now succeed self.expirer.once() # verify the deletion by checking the container listing self.assertFalse(self._check_obj_in_container_listing(), msg='Found listing for %s' % self.object_name) def test_expirer_delete_returns_outdated_404(self): self._test_expirer_delete_outdated_object_version(object_exists=False) def test_expirer_delete_returns_outdated_412(self): self._test_expirer_delete_outdated_object_version(object_exists=True)
class TestObjectExpirer(ReplProbeTest): def setUp(self): if len(ENABLED_POLICIES) < 2: raise SkipTest('Need more than one policy') self.expirer = Manager(['object-expirer']) self.expirer.start() err = self.expirer.stop() if err: raise SkipTest('Unable to verify object-expirer service') conf_files = [] for server in self.expirer.servers: conf_files.extend(server.conf_files()) conf_file = conf_files[0] self.client = InternalClient(conf_file, 'probe-test', 3) super(TestObjectExpirer, self).setUp() self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name) def test_expirer_object_split_brain(self): old_policy = random.choice(ENABLED_POLICIES) wrong_policy = random.choice([p for p in ENABLED_POLICIES if p != old_policy]) # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up Manager(['object-updater']).once() self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # don't start handoff servers, only wrong policy is available # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # this guy should no-op since it's unable to expire the object self.expirer.once() self.brain.start_handoff_half() self.get_to_final_state() # validate object is expired found_in_policy = None metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assertTrue('x-backend-timestamp' in metadata) self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: break else: self.fail('Did not find listing for %s' % self.object_name) # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # run the expirier again after replication self.expirer.once() # object is not in the listing for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: self.fail('Found listing for %s' % self.object_name) # and validate object is tombstoned found_in_policy = None for policy in ENABLED_POLICIES: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(policy)}) if 'x-backend-timestamp' in metadata: if found_in_policy: self.fail('found object in %s and also %s' % (found_in_policy, policy)) found_in_policy = policy self.assertTrue('x-backend-timestamp' in metadata) self.assertTrue(Timestamp(metadata['x-backend-timestamp']) > create_timestamp) def test_expirer_object_should_not_be_expired(self): obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) # T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated) # < T(expirer_executed) # Recreated obj should be appeared in any split brain case # T(obj_created) first_created_at = time.time() # T(obj_deleted with x-delete-at) # object-server accepts req only if X-Delete-At is later than 'now' delete_at = int(time.time() + 1.5) # T(obj_recreated) recreated_at = time.time() + 2.0 # T(expirer_executed) - 'now' sleep_for_expirer = 2.01 obj_brain.put_container(int(self.policy)) obj_brain.put_object( headers={'X-Delete-At': delete_at, 'X-Timestamp': Timestamp(first_created_at).internal}) # some object servers stopped obj_brain.stop_primary_half() obj_brain.put_object( headers={'X-Timestamp': Timestamp(recreated_at).internal, 'X-Object-Meta-Expired': 'False'}) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # some object servers recovered obj_brain.start_primary_half() # sleep to make sure expirer runs at the time after obj is recreated time.sleep(sleep_for_expirer) self.expirer.once() # inconsistent state of objects is recovered Manager(['object-replicator']).once() # check if you can get recreated object metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name) self.assertIn('x-object-meta-expired', metadata)
class TestObjectExpirer(ReplProbeTest): def setUp(self): if len(ENABLED_POLICIES) < 2: raise SkipTest("Need more than one policy") self.expirer = Manager(["object-expirer"]) self.expirer.start() err = self.expirer.stop() if err: raise SkipTest("Unable to verify object-expirer service") conf_files = [] for server in self.expirer.servers: conf_files.extend(server.conf_files()) conf_file = conf_files[0] self.client = InternalClient(conf_file, "probe-test", 3) super(TestObjectExpirer, self).setUp() self.container_name = "container-%s" % uuid.uuid4() self.object_name = "object-%s" % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name) def test_expirer_object_split_brain(self): old_policy = random.choice(ENABLED_POLICIES) wrong_policy = random.choice([p for p in ENABLED_POLICIES if p != old_policy]) # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) self.brain.put_object(headers={"X-Delete-After": 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={"X-Backend-Storage-Policy-Index": int(old_policy)}, ) create_timestamp = Timestamp(metadata["x-timestamp"]) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up Manager(["object-updater"]).once() self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # don't start handoff servers, only wrong policy is available # make sure auto-created containers get in the account listing Manager(["container-updater"]).once() # this guy should no-op since it's unable to expire the object self.expirer.once() self.brain.start_handoff_half() self.get_to_final_state() # validate object is expired found_in_policy = None metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={"X-Backend-Storage-Policy-Index": int(old_policy)}, ) self.assertTrue("x-backend-timestamp" in metadata) self.assertEqual(Timestamp(metadata["x-backend-timestamp"]), create_timestamp) # but it is still in the listing for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj["name"]: break else: self.fail("Did not find listing for %s" % self.object_name) # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # run the expirier again after replication self.expirer.once() # object is not in the listing for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj["name"]: self.fail("Found listing for %s" % self.object_name) # and validate object is tombstoned found_in_policy = None for policy in ENABLED_POLICIES: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={"X-Backend-Storage-Policy-Index": int(policy)}, ) if "x-backend-timestamp" in metadata: if found_in_policy: self.fail("found object in %s and also %s" % (found_in_policy, policy)) found_in_policy = policy self.assertTrue("x-backend-timestamp" in metadata) self.assertTrue(Timestamp(metadata["x-backend-timestamp"]) > create_timestamp)
class TestObjectExpirer(ReplProbeTest): def setUp(self): self.expirer = Manager(['object-expirer']) self.expirer.start() err = self.expirer.stop() if err: raise unittest.SkipTest('Unable to verify object-expirer service') conf_files = [] for server in self.expirer.servers: conf_files.extend(server.conf_files()) conf_file = conf_files[0] self.client = InternalClient(conf_file, 'probe-test', 3) super(TestObjectExpirer, self).setUp() self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name) def _check_obj_in_container_listing(self): for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: return True return False @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") def test_expirer_object_split_brain(self): old_policy = random.choice(ENABLED_POLICIES) wrong_policy = random.choice([p for p in ENABLED_POLICIES if p != old_policy]) # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up Manager(['object-updater']).once() self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # don't start handoff servers, only wrong policy is available # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # this guy should no-op since it's unable to expire the object self.expirer.once() self.brain.start_handoff_half() self.get_to_final_state() # validate object is expired found_in_policy = None metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assertIn('x-backend-timestamp', metadata) self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing self.assertTrue(self._check_obj_in_container_listing(), msg='Did not find listing for %s' % self.object_name) # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # run the expirer again after replication self.expirer.once() # object is not in the listing self.assertFalse(self._check_obj_in_container_listing(), msg='Found listing for %s' % self.object_name) # and validate object is tombstoned found_in_policy = None for policy in ENABLED_POLICIES: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(policy)}) if 'x-backend-timestamp' in metadata: if found_in_policy: self.fail('found object in %s and also %s' % (found_in_policy, policy)) found_in_policy = policy self.assertIn('x-backend-timestamp', metadata) self.assertGreater(Timestamp(metadata['x-backend-timestamp']), create_timestamp) def test_expirer_doesnt_make_async_pendings(self): # The object expirer cleans up its own queue. The inner loop # basically looks like this: # # for obj in stuff_to_delete: # delete_the_object(obj) # remove_the_queue_entry(obj) # # By default, upon receipt of a DELETE request for an expiring # object, the object servers will create async_pending records to # clean the expirer queue. Since the expirer cleans its own queue, # this is unnecessary. The expirer can make requests in such a way # tha the object server does not write out any async pendings; this # test asserts that this is the case. # Make an expiring object in each policy for policy in ENABLED_POLICIES: container_name = "expirer-test-%d" % policy.idx container_headers = {'X-Storage-Policy': policy.name} client.put_container(self.url, self.token, container_name, headers=container_headers) now = time.time() delete_at = int(now + 2.0) client.put_object( self.url, self.token, container_name, "some-object", headers={'X-Delete-At': str(delete_at), 'X-Timestamp': Timestamp(now).normal}, contents='dontcare') time.sleep(2.0) # make sure auto-created expirer-queue containers get in the account # listing so the expirer can find them Manager(['container-updater']).once() # Make sure there's no async_pendings anywhere. Probe tests only run # on single-node installs anyway, so this set should be small enough # that an exhaustive check doesn't take too long. all_obj_nodes = self.get_all_object_nodes() pendings_before = self.gather_async_pendings(all_obj_nodes) # expire the objects Manager(['object-expirer']).once() pendings_after = self.gather_async_pendings(all_obj_nodes) self.assertEqual(pendings_after, pendings_before) def test_expirer_object_should_not_be_expired(self): # Current object-expirer checks the correctness via x-if-delete-at # header that it can be deleted by expirer. If there are objects # either which doesn't have x-delete-at header as metadata or which # has different x-delete-at value from x-if-delete-at value, # object-expirer's delete will fail as 412 PreconditionFailed. # However, if some of the objects are in handoff nodes, the expirer # can put the tombstone with the timestamp as same as x-delete-at and # the object consistency will be resolved as the newer timestamp will # be winner (in particular, overwritten case w/o x-delete-at). This # test asserts such a situation that, at least, the overwriten object # which have larger timestamp than the original expirered date should # be safe. def put_object(headers): # use internal client to PUT objects so that X-Timestamp in headers # is effective headers['Content-Length'] = '0' path = self.client.make_path( self.account, self.container_name, self.object_name) try: self.client.make_request('PUT', path, headers, (2,)) except UnexpectedResponse as e: self.fail( 'Expected 201 for PUT object but got %s' % e.resp.status) obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) # T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated) # < T(expirer_executed) # Recreated obj should be appeared in any split brain case obj_brain.put_container() # T(obj_deleted with x-delete-at) # object-server accepts req only if X-Delete-At is later than 'now' # so here, T(obj_created) < T(obj_deleted with x-delete-at) now = time.time() delete_at = int(now + 2.0) recreate_at = delete_at + 1.0 put_object(headers={'X-Delete-At': str(delete_at), 'X-Timestamp': Timestamp(now).normal}) # some object servers stopped to make a situation that the # object-expirer can put tombstone in the primary nodes. obj_brain.stop_primary_half() # increment the X-Timestamp explicitly # (will be T(obj_deleted with x-delete-at) < T(obj_recreated)) put_object(headers={'X-Object-Meta-Expired': 'False', 'X-Timestamp': Timestamp(recreate_at).normal}) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # sanity, the newer object is still there try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name) except UnexpectedResponse as e: self.fail( 'Expected 200 for HEAD object but got %s' % e.resp.status) self.assertIn('x-object-meta-expired', metadata) # some object servers recovered obj_brain.start_primary_half() # sleep until after recreated_at while time.time() <= recreate_at: time.sleep(0.1) # Now, expirer runs at the time after obj is recreated self.expirer.once() # verify that original object was deleted by expirer obj_brain.stop_handoff_half() try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,)) except UnexpectedResponse as e: self.fail( 'Expected 404 for HEAD object but got %s' % e.resp.status) obj_brain.start_handoff_half() # and inconsistent state of objects is recovered by replicator Manager(['object-replicator']).once() # check if you can get recreated object try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name) except UnexpectedResponse as e: self.fail( 'Expected 200 for HEAD object but got %s' % e.resp.status) self.assertIn('x-object-meta-expired', metadata) def _test_expirer_delete_outdated_object_version(self, object_exists): # This test simulates a case where the expirer tries to delete # an outdated version of an object. # One case is where the expirer gets a 404, whereas the newest version # of the object is offline. # Another case is where the expirer gets a 412, since the old version # of the object mismatches the expiration time sent by the expirer. # In any of these cases, the expirer should retry deleting the object # later, for as long as a reclaim age has not passed. obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) obj_brain.put_container() if object_exists: obj_brain.put_object() # currently, the object either doesn't exist, or does not have # an expiration # stop primary servers and put a newer version of the object, this # time with an expiration. only the handoff servers will have # the new version obj_brain.stop_primary_half() now = time.time() delete_at = int(now + 2.0) obj_brain.put_object({'X-Delete-At': str(delete_at)}) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # update object record in the container listing Manager(['container-replicator']).once() # take handoff servers down, and bring up the outdated primary servers obj_brain.start_primary_half() obj_brain.stop_handoff_half() # wait until object expiration time while time.time() <= delete_at: time.sleep(0.1) # run expirer against the outdated servers. it should fail since # the outdated version does not match the expiration time self.expirer.once() # bring all servers up, and run replicator to update servers obj_brain.start_handoff_half() Manager(['object-replicator']).once() # verify the deletion has failed by checking the container listing self.assertTrue(self._check_obj_in_container_listing(), msg='Did not find listing for %s' % self.object_name) # run expirer again, delete should now succeed self.expirer.once() # verify the deletion by checking the container listing self.assertFalse(self._check_obj_in_container_listing(), msg='Found listing for %s' % self.object_name) def test_expirer_delete_returns_outdated_404(self): self._test_expirer_delete_outdated_object_version(object_exists=False) def test_expirer_delete_returns_outdated_412(self): self._test_expirer_delete_outdated_object_version(object_exists=True)
class TestObjectExpirer(ReplProbeTest): def setUp(self): self.expirer = Manager(['object-expirer']) self.expirer.start() err = self.expirer.stop() if err: raise SkipTest('Unable to verify object-expirer service') conf_files = [] for server in self.expirer.servers: conf_files.extend(server.conf_files()) conf_file = conf_files[0] self.client = InternalClient(conf_file, 'probe-test', 3) super(TestObjectExpirer, self).setUp() self.container_name = 'container-%s' % uuid.uuid4() self.object_name = 'object-%s' % uuid.uuid4() self.brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name) def _check_obj_in_container_listing(self): for obj in self.client.iter_objects(self.account, self.container_name): if self.object_name == obj['name']: return True return False def test_expirer_object_split_brain(self): if len(ENABLED_POLICIES) < 2: raise SkipTest('Need more than one policy') old_policy = random.choice(ENABLED_POLICIES) wrong_policy = random.choice([p for p in ENABLED_POLICIES if p != old_policy]) # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up Manager(['object-updater']).once() self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # don't start handoff servers, only wrong policy is available # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # this guy should no-op since it's unable to expire the object self.expirer.once() self.brain.start_handoff_half() self.get_to_final_state() # validate object is expired found_in_policy = None metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assertIn('x-backend-timestamp', metadata) self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing self.assertTrue(self._check_obj_in_container_listing(), msg='Did not find listing for %s' % self.object_name) # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # run the expirier again after replication self.expirer.once() # object is not in the listing self.assertFalse(self._check_obj_in_container_listing(), msg='Found listing for %s' % self.object_name) # and validate object is tombstoned found_in_policy = None for policy in ENABLED_POLICIES: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(policy)}) if 'x-backend-timestamp' in metadata: if found_in_policy: self.fail('found object in %s and also %s' % (found_in_policy, policy)) found_in_policy = policy self.assertIn('x-backend-timestamp', metadata) self.assertGreater(Timestamp(metadata['x-backend-timestamp']), create_timestamp) def test_expirer_object_should_not_be_expired(self): # Current object-expirer checks the correctness via x-if-delete-at # header that it can be deleted by expirer. If there are objects # either which doesn't have x-delete-at header as metadata or which # has different x-delete-at value from x-if-delete-at value, # object-expirer's delete will fail as 412 PreconditionFailed. # However, if some of the objects are in handoff nodes, the expirer # can put the tombstone with the timestamp as same as x-delete-at and # the object consistency will be resolved as the newer timestamp will # be winner (in particular, overwritten case w/o x-delete-at). This # test asserts such a situation that, at least, the overwriten object # which have larger timestamp than the original expirered date should # be safe. def put_object(headers): # use internal client to PUT objects so that X-Timestamp in headers # is effective headers['Content-Length'] = '0' path = self.client.make_path( self.account, self.container_name, self.object_name) try: self.client.make_request('PUT', path, headers, (2,)) except UnexpectedResponse as e: self.fail( 'Expected 201 for PUT object but got %s' % e.resp.status) obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) # T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated) # < T(expirer_executed) # Recreated obj should be appeared in any split brain case obj_brain.put_container() # T(obj_deleted with x-delete-at) # object-server accepts req only if X-Delete-At is later than 'now' # so here, T(obj_created) < T(obj_deleted with x-delete-at) now = time.time() delete_at = int(now + 2.0) recreate_at = delete_at + 1.0 put_object(headers={'X-Delete-At': delete_at, 'X-Timestamp': Timestamp(now).normal}) # some object servers stopped to make a situation that the # object-expirer can put tombstone in the primary nodes. obj_brain.stop_primary_half() # increment the X-Timestamp explicitly # (will be T(obj_deleted with x-delete-at) < T(obj_recreated)) put_object(headers={'X-Object-Meta-Expired': 'False', 'X-Timestamp': Timestamp(recreate_at).normal}) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # sanity, the newer object is still there try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name) except UnexpectedResponse as e: self.fail( 'Expected 200 for HEAD object but got %s' % e.resp.status) self.assertIn('x-object-meta-expired', metadata) # some object servers recovered obj_brain.start_primary_half() # sleep until after recreated_at while time.time() <= recreate_at: time.sleep(0.1) # Now, expirer runs at the time after obj is recreated self.expirer.once() # verify that original object was deleted by expirer obj_brain.stop_handoff_half() try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, acceptable_statuses=(4,)) except UnexpectedResponse as e: self.fail( 'Expected 404 for HEAD object but got %s' % e.resp.status) obj_brain.start_handoff_half() # and inconsistent state of objects is recovered by replicator Manager(['object-replicator']).once() # check if you can get recreated object try: metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name) except UnexpectedResponse as e: self.fail( 'Expected 200 for HEAD object but got %s' % e.resp.status) self.assertIn('x-object-meta-expired', metadata) def _test_expirer_delete_outdated_object_version(self, object_exists): # This test simulates a case where the expirer tries to delete # an outdated version of an object. # One case is where the expirer gets a 404, whereas the newest version # of the object is offline. # Another case is where the expirer gets a 412, since the old version # of the object mismatches the expiration time sent by the expirer. # In any of these cases, the expirer should retry deleting the object # later, for as long as a reclaim age has not passed. obj_brain = BrainSplitter(self.url, self.token, self.container_name, self.object_name, 'object', self.policy) obj_brain.put_container() if object_exists: obj_brain.put_object() # currently, the object either doesn't exist, or does not have # an expiration # stop primary servers and put a newer version of the object, this # time with an expiration. only the handoff servers will have # the new version obj_brain.stop_primary_half() now = time.time() delete_at = int(now + 2.0) obj_brain.put_object({'X-Delete-At': delete_at}) # make sure auto-created containers get in the account listing Manager(['container-updater']).once() # update object record in the container listing Manager(['container-replicator']).once() # take handoff servers down, and bring up the outdated primary servers obj_brain.start_primary_half() obj_brain.stop_handoff_half() # wait until object expiration time while time.time() <= delete_at: time.sleep(0.1) # run expirer against the outdated servers. it should fail since # the outdated version does not match the expiration time self.expirer.once() # bring all servers up, and run replicator to update servers obj_brain.start_handoff_half() Manager(['object-replicator']).once() # verify the deletion has failed by checking the container listing self.assertTrue(self._check_obj_in_container_listing(), msg='Did not find listing for %s' % self.object_name) # run expirer again, delete should now succeed self.expirer.once() # verify the deletion by checking the container listing self.assertFalse(self._check_obj_in_container_listing(), msg='Found listing for %s' % self.object_name) def test_expirer_delete_returns_outdated_404(self): self._test_expirer_delete_outdated_object_version(object_exists=False) def test_expirer_delete_returns_outdated_412(self): self._test_expirer_delete_outdated_object_version(object_exists=True)