def test_reap_object(self): conf = { 'mount_check': 'false', } r = reaper.AccountReaper(conf, logger=unit.debug_logger()) ring = unit.FakeRing() mock_path = 'swift.account.reaper.direct_delete_object' for policy in POLICIES: r.reset_stats() with patch(mock_path) as fake_direct_delete: r.reap_object('a', 'c', 'partition', cont_nodes, 'o', policy.idx) for i, call_args in enumerate( fake_direct_delete.call_args_list): cnode = cont_nodes[i] host = '%(ip)s:%(port)s' % cnode device = cnode['device'] headers = { 'X-Container-Host': host, 'X-Container-Partition': 'partition', 'X-Container-Device': device, 'X-Backend-Storage-Policy-Index': policy.idx } ring = r.get_object_ring(policy.idx) expected = call(ring.devs[i], 0, 'a', 'c', 'o', headers=headers, conn_timeout=0.5, response_timeout=10) self.assertEqual(call_args, expected) self.assertEqual(r.stats_objects_deleted, 3)
class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None) def tearDown(self): for todel in self.to_delete: shutil.rmtree(todel) def fake_direct_delete_object(self, *args, **kwargs): if self.amount_fail < self.max_fail: self.amount_fail += 1 raise self.myexp if self.reap_obj_timeout: raise eventlet.Timeout() def fake_direct_delete_container(self, *args, **kwargs): if self.amount_delete_fail < self.max_delete_fail: self.amount_delete_fail += 1 raise self.myexp def fake_direct_get_container(self, *args, **kwargs): if self.get_fail: raise self.myexp if self.timeout: raise eventlet.Timeout() objects = [{ 'name': 'o1' }, { 'name': 'o2' }, { 'name': six.text_type('o3') }, { 'name': '' }] return None, objects def fake_container_ring(self): return FakeRing() def fake_reap_object(self, *args, **kwargs): if self.reap_obj_fail: raise Exception def prepare_data_dir(self, ts=False, device='sda1'): devices_path = tempfile.mkdtemp() # will be deleted by teardown self.to_delete.append(devices_path) path = os.path.join(devices_path, device, DATADIR) os.makedirs(path) path = os.path.join(path, '100', 'a86', 'a8c682d2472e1720f2d81ff8993aba6') os.makedirs(path) suffix = 'db' if ts: suffix = 'ts' with open(os.path.join(path, 'a8c682203aba6.%s' % suffix), 'w') as fd: fd.write('') return devices_path def init_reaper(self, conf=None, myips=None, fakelogger=False): if conf is None: conf = {} if myips is None: myips = ['10.10.10.1'] r = reaper.AccountReaper(conf) r.myips = myips if fakelogger: r.logger = unit.debug_logger('test-reaper') return r def fake_reap_account(self, *args, **kwargs): self.called_amount += 1 def fake_account_ring(self): return FakeRing() def test_creation(self): # later config should be extended to assert more config options r = reaper.AccountReaper({'node_timeout': '3.5'}) self.assertEqual(r.node_timeout, 3.5) def test_delay_reaping_conf_default(self): r = reaper.AccountReaper({}) self.assertEqual(r.delay_reaping, 0) r = reaper.AccountReaper({'delay_reaping': ''}) self.assertEqual(r.delay_reaping, 0) def test_delay_reaping_conf_set(self): r = reaper.AccountReaper({'delay_reaping': '123'}) self.assertEqual(r.delay_reaping, 123) def test_delay_reaping_conf_bad_value(self): self.assertRaises(ValueError, reaper.AccountReaper, {'delay_reaping': 'abc'}) def test_reap_warn_after_conf_set(self): conf = {'delay_reaping': '2', 'reap_warn_after': '3'} r = reaper.AccountReaper(conf) self.assertEqual(r.reap_not_done_after, 5) def test_reap_warn_after_conf_bad_value(self): self.assertRaises(ValueError, reaper.AccountReaper, {'reap_warn_after': 'abc'}) def test_reap_delay(self): time_value = [100] def _time(): return time_value[0] time_orig = reaper.time try: reaper.time = _time r = reaper.AccountReaper({'delay_reaping': '10'}) b = FakeBroker() b.info['delete_timestamp'] = normalize_timestamp(110) self.assertFalse(r.reap_account(b, 0, None)) b.info['delete_timestamp'] = normalize_timestamp(100) self.assertFalse(r.reap_account(b, 0, None)) b.info['delete_timestamp'] = normalize_timestamp(90) self.assertFalse(r.reap_account(b, 0, None)) # KeyError raised immediately as reap_account tries to get the # account's name to do the reaping. b.info['delete_timestamp'] = normalize_timestamp(89) self.assertRaises(KeyError, r.reap_account, b, 0, None) b.info['delete_timestamp'] = normalize_timestamp(1) self.assertRaises(KeyError, r.reap_account, b, 0, None) finally: reaper.time = time_orig def test_reset_stats(self): conf = {} r = reaper.AccountReaper(conf) self.assertDictEqual(r.stats_return_codes, {}) self.assertEqual(r.stats_containers_deleted, 0) self.assertEqual(r.stats_containers_remaining, 0) self.assertEqual(r.stats_containers_possibly_remaining, 0) self.assertEqual(r.stats_objects_deleted, 0) self.assertEqual(r.stats_objects_remaining, 0) self.assertEqual(r.stats_objects_possibly_remaining, 0) # also make sure reset actually resets values r.stats_return_codes = {"hello": "swift"} r.stats_containers_deleted = random.randint(1, 100) r.stats_containers_remaining = random.randint(1, 100) r.stats_containers_possibly_remaining = random.randint(1, 100) r.stats_objects_deleted = random.randint(1, 100) r.stats_objects_remaining = random.randint(1, 100) r.stats_objects_possibly_remaining = random.randint(1, 100) r.reset_stats() self.assertDictEqual(r.stats_return_codes, {}) self.assertEqual(r.stats_containers_deleted, 0) self.assertEqual(r.stats_containers_remaining, 0) self.assertEqual(r.stats_containers_possibly_remaining, 0) self.assertEqual(r.stats_objects_deleted, 0) self.assertEqual(r.stats_objects_remaining, 0) self.assertEqual(r.stats_objects_possibly_remaining, 0) def test_reap_object(self): conf = { 'mount_check': 'false', } r = reaper.AccountReaper(conf, logger=unit.debug_logger()) mock_path = 'swift.account.reaper.direct_delete_object' for policy in POLICIES: r.reset_stats() with patch(mock_path) as fake_direct_delete: with patch('swift.common.utils.Timestamp.now') as mock_now: mock_now.return_value = Timestamp(1429117638.86767) r.reap_object('a', 'c', 'partition', cont_nodes, 'o', policy.idx) mock_now.assert_called_once_with() for i, call_args in enumerate( fake_direct_delete.call_args_list): cnode = cont_nodes[i % len(cont_nodes)] host = '%(ip)s:%(port)s' % cnode device = cnode['device'] headers = { 'X-Container-Host': host, 'X-Container-Partition': 'partition', 'X-Container-Device': device, 'X-Backend-Storage-Policy-Index': policy.idx, 'X-Timestamp': '1429117638.86767' } ring = r.get_object_ring(policy.idx) expected = call(dict(ring.devs[i], index=i), 0, 'a', 'c', 'o', headers=headers, conn_timeout=0.5, response_timeout=10) self.assertEqual(call_args, expected) self.assertEqual(policy.object_ring.replicas - 1, i) self.assertEqual(r.stats_objects_deleted, policy.object_ring.replicas) def test_reap_object_fail(self): r = self.init_reaper({}, fakelogger=True) self.amount_fail = 0 self.max_fail = 1 self.reap_obj_timeout = False policy = random.choice(list(POLICIES)) with patch('swift.account.reaper.direct_delete_object', self.fake_direct_delete_object): r.reap_object('a', 'c', 'partition', cont_nodes, 'o', policy.idx) # IMHO, the stat handling in the node loop of reap object is # over indented, but no one has complained, so I'm not inclined # to move it. However it's worth noting we're currently keeping # stats on deletes per *replica* - which is rather obvious from # these tests, but this results is surprising because of some # funny logic to *skip* increments on successful deletes of # replicas until we have more successful responses than # failures. This means that while the first replica doesn't # increment deleted because of the failure, the second one # *does* get successfully deleted, but *also does not* increment # the counter (!?). # # In the three replica case this leaves only the last deleted # object incrementing the counter - in the four replica case # this leaves the last two. # # Basically this test will always result in: # deleted == num_replicas - 2 self.assertEqual(r.stats_objects_deleted, policy.object_ring.replicas - 2) self.assertEqual(r.stats_objects_remaining, 1) self.assertEqual(r.stats_objects_possibly_remaining, 1) self.assertEqual(r.stats_return_codes[2], policy.object_ring.replicas - 1) self.assertEqual(r.stats_return_codes[4], 1) def test_reap_object_timeout(self): r = self.init_reaper({}, fakelogger=True) self.amount_fail = 1 self.max_fail = 0 self.reap_obj_timeout = True with patch('swift.account.reaper.direct_delete_object', self.fake_direct_delete_object): r.reap_object('a', 'c', 'partition', cont_nodes, 'o', 1) self.assertEqual(r.stats_objects_deleted, 0) self.assertEqual(r.stats_objects_remaining, 4) self.assertEqual(r.stats_objects_possibly_remaining, 0) self.assertTrue( r.logger.get_lines_for_level('error')[-1].startswith( 'Timeout Exception')) def test_reap_object_non_exist_policy_index(self): r = self.init_reaper({}, fakelogger=True) r.reap_object('a', 'c', 'partition', cont_nodes, 'o', 2) self.assertEqual(r.stats_objects_deleted, 0) self.assertEqual(r.stats_objects_remaining, 1) self.assertEqual(r.stats_objects_possibly_remaining, 0) @patch('swift.account.reaper.Ring', lambda *args, **kwargs: unit.FakeRing()) def test_reap_container(self): policy = random.choice(list(POLICIES)) r = self.init_reaper({}, fakelogger=True) with patch.multiple('swift.account.reaper', direct_get_container=DEFAULT, direct_delete_object=DEFAULT, direct_delete_container=DEFAULT) as mocks: headers = {'X-Backend-Storage-Policy-Index': policy.idx} obj_listing = [{'name': 'o'}] def fake_get_container(*args, **kwargs): try: obj = obj_listing.pop(0) except IndexError: obj_list = [] else: obj_list = [obj] return headers, obj_list mocks['direct_get_container'].side_effect = fake_get_container with patch('swift.common.utils.Timestamp.now') as mock_now: mock_now.side_effect = [ Timestamp(1429117638.86767), Timestamp(1429117639.67676) ] r.reap_container('a', 'partition', acc_nodes, 'c') # verify calls to direct_delete_object mock_calls = mocks['direct_delete_object'].call_args_list self.assertEqual(policy.object_ring.replicas, len(mock_calls)) for call_args in mock_calls: _args, kwargs = call_args self.assertEqual( kwargs['headers']['X-Backend-Storage-Policy-Index'], policy.idx) self.assertEqual(kwargs['headers']['X-Timestamp'], '1429117638.86767') # verify calls to direct_delete_container self.assertEqual(mocks['direct_delete_container'].call_count, 3) for i, call_args in enumerate( mocks['direct_delete_container'].call_args_list): anode = acc_nodes[i % len(acc_nodes)] host = '%(ip)s:%(port)s' % anode device = anode['device'] headers = { 'X-Account-Host': host, 'X-Account-Partition': 'partition', 'X-Account-Device': device, 'X-Account-Override-Deleted': 'yes', 'X-Timestamp': '1429117639.67676' } ring = r.get_object_ring(policy.idx) expected = call(dict(ring.devs[i], index=i), 0, 'a', 'c', headers=headers, conn_timeout=0.5, response_timeout=10) self.assertEqual(call_args, expected) self.assertEqual(r.stats_objects_deleted, policy.object_ring.replicas) def test_reap_container_get_object_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = True self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 0 with patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), \ patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), \ patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), \ patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.get_increment_counts()['return_codes.4'], 1) self.assertEqual(r.stats_containers_deleted, 1) def test_reap_container_partial_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.timeout = False self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 4 with patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), \ patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), \ patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), \ patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.get_increment_counts()['return_codes.4'], 4) self.assertEqual(r.stats_containers_possibly_remaining, 1) def test_reap_container_full_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.timeout = False self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 5 with patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), \ patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), \ patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), \ patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.get_increment_counts()['return_codes.4'], 5) self.assertEqual(r.stats_containers_remaining, 1) def test_reap_container_get_object_timeout(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.timeout = True self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 0 with patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), \ patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), \ patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), \ patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertTrue( r.logger.get_lines_for_level('error')[-1].startswith( 'Timeout Exception')) @patch('swift.account.reaper.Ring', lambda *args, **kwargs: unit.FakeRing()) def test_reap_container_non_exist_policy_index(self): r = self.init_reaper({}, fakelogger=True) with patch.multiple('swift.account.reaper', direct_get_container=DEFAULT, direct_delete_object=DEFAULT, direct_delete_container=DEFAULT) as mocks: headers = {'X-Backend-Storage-Policy-Index': 2} obj_listing = [{'name': 'o'}] def fake_get_container(*args, **kwargs): try: obj = obj_listing.pop(0) except IndexError: obj_list = [] else: obj_list = [obj] return headers, obj_list mocks['direct_get_container'].side_effect = fake_get_container r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.get_lines_for_level('error'), ['ERROR: invalid storage policy index: 2']) def fake_reap_container(self, *args, **kwargs): self.called_amount += 1 self.r.stats_containers_deleted = 1 self.r.stats_objects_deleted = 1 self.r.stats_containers_remaining = 1 self.r.stats_objects_remaining = 1 self.r.stats_containers_possibly_remaining = 1 self.r.stats_objects_possibly_remaining = 1 self.r.stats_return_codes[2] = \ self.r.stats_return_codes.get(2, 0) + 1 def test_reap_account(self): containers = ('c1', 'c2', 'c3', '') broker = FakeAccountBroker(containers) self.called_amount = 0 self.r = r = self.init_reaper({}, fakelogger=True) r.start_time = time.time() with patch('swift.account.reaper.AccountReaper.reap_container', self.fake_reap_container), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring): nodes = r.get_account_ring().get_part_nodes() for container_shard, node in enumerate(nodes): self.assertTrue( r.reap_account(broker, 'partition', nodes, container_shard=container_shard)) self.assertEqual(self.called_amount, 4) info_lines = r.logger.get_lines_for_level('info') self.assertEqual(len(info_lines), 10) for start_line, stat_line in zip(*[iter(info_lines)] * 2): self.assertEqual(start_line, 'Beginning pass on account a') self.assertTrue(stat_line.find('1 containers deleted')) self.assertTrue(stat_line.find('1 objects deleted')) self.assertTrue(stat_line.find('1 containers remaining')) self.assertTrue(stat_line.find('1 objects remaining')) self.assertTrue(stat_line.find('1 containers possibly remaining')) self.assertTrue(stat_line.find('1 objects possibly remaining')) self.assertTrue(stat_line.find('return codes: 2 2xxs')) @patch('swift.account.reaper.Ring', lambda *args, **kwargs: unit.FakeRing()) def test_basic_reap_account(self): self.r = reaper.AccountReaper({}) self.r.account_ring = None self.r.get_account_ring() self.assertEqual(self.r.account_ring.replica_count, 3) self.assertEqual(len(self.r.account_ring.devs), 3) def test_reap_account_no_container(self): broker = FakeAccountBroker(tuple()) self.r = r = self.init_reaper({}, fakelogger=True) self.called_amount = 0 r.start_time = time.time() with patch('swift.account.reaper.AccountReaper.reap_container', self.fake_reap_container), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) self.assertTrue( r.logger.get_lines_for_level('info')[-1].startswith( 'Completed pass')) self.assertEqual(self.called_amount, 0) def test_reap_device(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf) with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), \ patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account): r.reap_device('sda1') self.assertEqual(self.called_amount, 1) def test_reap_device_with_ts(self): devices = self.prepare_data_dir(ts=True) self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf=conf) with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), \ patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account): r.reap_device('sda1') self.assertEqual(self.called_amount, 0) def test_reap_device_with_not_my_ip(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf, myips=['10.10.1.2']) with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), \ patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account): r.reap_device('sda1') self.assertEqual(self.called_amount, 0) def test_reap_device_with_sharding(self): devices = self.prepare_data_dir() conf = {'devices': devices} r = self.init_reaper(conf, myips=['10.10.10.2']) container_shard_used = [-1] def fake_reap_account(*args, **kwargs): container_shard_used[0] = kwargs.get('container_shard') with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), \ patch('swift.account.reaper.AccountReaper.reap_account', fake_reap_account): r.reap_device('sda1') # 10.10.10.2 is second node from ring self.assertEqual(container_shard_used[0], 1) def test_reap_device_with_sharding_and_various_devices(self): devices = self.prepare_data_dir(device='sda2') conf = {'devices': devices} r = self.init_reaper(conf) container_shard_used = [-1] def fake_reap_account(*args, **kwargs): container_shard_used[0] = kwargs.get('container_shard') with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), \ patch('swift.account.reaper.AccountReaper.reap_account', fake_reap_account): r.reap_device('sda2') # 10.10.10.2 is second node from ring self.assertEqual(container_shard_used[0], 3) devices = self.prepare_data_dir(device='sda3') conf = {'devices': devices} r = self.init_reaper(conf) container_shard_used = [-1] with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), \ patch('swift.account.reaper.AccountReaper.reap_account', fake_reap_account): r.reap_device('sda3') # 10.10.10.2 is second node from ring self.assertEqual(container_shard_used[0], 4) def test_reap_account_with_sharding(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf, myips=['10.10.10.2']) container_reaped = [0] def fake_list_containers_iter(self, *args): for container in self.containers: if container in self.containers_yielded: continue yield container, None, None, None, None self.containers_yielded.append(container) def fake_reap_container(self, account, account_partition, account_nodes, container): container_reaped[0] += 1 fake_ring = FakeRing() with patch('swift.account.reaper.AccountBroker', FakeAccountBroker), \ patch( 'swift.account.reaper.AccountBroker.list_containers_iter', fake_list_containers_iter), \ patch('swift.account.reaper.AccountReaper.reap_container', fake_reap_container): fake_broker = FakeAccountBroker(['c', 'd', 'e', 'f', 'g']) r.reap_account(fake_broker, 10, fake_ring.nodes, 0) self.assertEqual(container_reaped[0], 0) fake_broker = FakeAccountBroker(['c', 'd', 'e', 'f', 'g']) container_reaped[0] = 0 r.reap_account(fake_broker, 10, fake_ring.nodes, 1) self.assertEqual(container_reaped[0], 1) container_reaped[0] = 0 fake_broker = FakeAccountBroker(['c', 'd', 'e', 'f', 'g']) r.reap_account(fake_broker, 10, fake_ring.nodes, 2) self.assertEqual(container_reaped[0], 0) container_reaped[0] = 0 fake_broker = FakeAccountBroker(['c', 'd', 'e', 'f', 'g']) r.reap_account(fake_broker, 10, fake_ring.nodes, 3) self.assertEqual(container_reaped[0], 3) container_reaped[0] = 0 fake_broker = FakeAccountBroker(['c', 'd', 'e', 'f', 'g']) r.reap_account(fake_broker, 10, fake_ring.nodes, 4) self.assertEqual(container_reaped[0], 1) def test_run_once(self): def prepare_data_dir(): devices_path = tempfile.mkdtemp() # will be deleted by teardown self.to_delete.append(devices_path) path = os.path.join(devices_path, 'sda1', DATADIR) os.makedirs(path) return devices_path def init_reaper(devices): r = reaper.AccountReaper({'devices': devices}) return r devices = prepare_data_dir() r = init_reaper(devices) with patch('swift.account.reaper.AccountReaper.reap_device') as foo, \ unit.mock_check_drive(ismount=True): r.run_once() self.assertEqual(foo.called, 1) with patch('swift.account.reaper.AccountReaper.reap_device') as foo, \ unit.mock_check_drive(ismount=False): r.run_once() self.assertFalse(foo.called) with patch('swift.account.reaper.AccountReaper.reap_device') as foo: r.logger = unit.debug_logger('test-reaper') r.devices = 'thisdeviceisbad' r.run_once() self.assertTrue( r.logger.get_lines_for_level('error')[-1].startswith( 'Exception in top-level account reaper')) def test_run_forever(self): def fake_sleep(val): self.val = val def fake_random(): return 1 def fake_run_once(): raise Exception('exit') def init_reaper(): r = reaper.AccountReaper({'interval': 1}) r.run_once = fake_run_once return r r = init_reaper() with patch('swift.account.reaper.sleep', fake_sleep): with patch('swift.account.reaper.random.random', fake_random): try: r.run_forever() except Exception as err: pass self.assertEqual(self.val, 1) self.assertEqual(str(err), 'exit')
'device': 'sda1', 'ip': '', 'port': '' }, { 'device': 'sda1', 'ip': '', 'port': '' }, { 'device': 'sda1', 'ip': '', 'port': '' }] @unit.patch_policies([ StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()), StoragePolicy(1, 'one', True, object_ring=unit.FakeRing(replicas=4)) ]) class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None) def tearDown(self): for todel in self.to_delete: shutil.rmtree(todel)
'ip': '', 'port': ''}] cont_nodes = [{'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}] @unit.patch_policies([StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()), StoragePolicy(1, 'one', True, object_ring=unit.FakeRing(replicas=4))]) class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None ) def tearDown(self): for todel in self.to_delete:
class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None ) def tearDown(self): for todel in self.to_delete: shutil.rmtree(todel) def fake_direct_delete_object(self, *args, **kwargs): if self.amount_fail < self.max_fail: self.amount_fail += 1 raise self.myexp def fake_direct_delete_container(self, *args, **kwargs): if self.amount_delete_fail < self.max_delete_fail: self.amount_delete_fail += 1 raise self.myexp def fake_direct_get_container(self, *args, **kwargs): if self.get_fail: raise self.myexp objects = [{'name': 'o1'}, {'name': 'o2'}, {'name': unicode('o3')}, {'name': ''}] return None, objects def fake_container_ring(self): return FakeRing() def fake_reap_object(self, *args, **kwargs): if self.reap_obj_fail: raise Exception def prepare_data_dir(self, ts=False): devices_path = tempfile.mkdtemp() # will be deleted by teardown self.to_delete.append(devices_path) path = os.path.join(devices_path, 'sda1', DATADIR) os.makedirs(path) path = os.path.join(path, '100', 'a86', 'a8c682d2472e1720f2d81ff8993aba6') os.makedirs(path) suffix = 'db' if ts: suffix = 'ts' with open(os.path.join(path, 'a8c682203aba6.%s' % suffix), 'w') as fd: fd.write('') return devices_path def init_reaper(self, conf=None, myips=None, fakelogger=False): if conf is None: conf = {} if myips is None: myips = ['10.10.10.1'] r = reaper.AccountReaper(conf) r.stats_return_codes = {} r.stats_containers_deleted = 0 r.stats_containers_remaining = 0 r.stats_containers_possibly_remaining = 0 r.stats_objects_deleted = 0 r.stats_objects_remaining = 0 r.stats_objects_possibly_remaining = 0 r.myips = myips if fakelogger: r.logger = FakeLogger() return r def fake_reap_account(self, *args, **kwargs): self.called_amount += 1 def fake_account_ring(self): return FakeRing() def test_delay_reaping_conf_default(self): r = reaper.AccountReaper({}) self.assertEqual(r.delay_reaping, 0) r = reaper.AccountReaper({'delay_reaping': ''}) self.assertEqual(r.delay_reaping, 0) def test_delay_reaping_conf_set(self): r = reaper.AccountReaper({'delay_reaping': '123'}) self.assertEqual(r.delay_reaping, 123) def test_delay_reaping_conf_bad_value(self): self.assertRaises(ValueError, reaper.AccountReaper, {'delay_reaping': 'abc'}) def test_reap_warn_after_conf_set(self): conf = {'delay_reaping': '2', 'reap_warn_after': '3'} r = reaper.AccountReaper(conf) self.assertEqual(r.reap_not_done_after, 5) def test_reap_warn_after_conf_bad_value(self): self.assertRaises(ValueError, reaper.AccountReaper, {'reap_warn_after': 'abc'}) def test_reap_delay(self): time_value = [100] def _time(): return time_value[0] time_orig = reaper.time try: reaper.time = _time r = reaper.AccountReaper({'delay_reaping': '10'}) b = FakeBroker() b.info['delete_timestamp'] = normalize_timestamp(110) self.assertFalse(r.reap_account(b, 0, None)) b.info['delete_timestamp'] = normalize_timestamp(100) self.assertFalse(r.reap_account(b, 0, None)) b.info['delete_timestamp'] = normalize_timestamp(90) self.assertFalse(r.reap_account(b, 0, None)) # KeyError raised immediately as reap_account tries to get the # account's name to do the reaping. b.info['delete_timestamp'] = normalize_timestamp(89) self.assertRaises(KeyError, r.reap_account, b, 0, None) b.info['delete_timestamp'] = normalize_timestamp(1) self.assertRaises(KeyError, r.reap_account, b, 0, None) finally: reaper.time = time_orig def test_reap_object(self): conf = { 'mount_check': 'false', } r = reaper.AccountReaper(conf, logger=unit.debug_logger()) ring = unit.FakeRing() mock_path = 'swift.account.reaper.direct_delete_object' for policy in POLICIES: r.reset_stats() with patch(mock_path) as fake_direct_delete: r.reap_object('a', 'c', 'partition', cont_nodes, 'o', policy.idx) for i, call_args in enumerate( fake_direct_delete.call_args_list): cnode = cont_nodes[i] host = '%(ip)s:%(port)s' % cnode device = cnode['device'] headers = { 'X-Container-Host': host, 'X-Container-Partition': 'partition', 'X-Container-Device': device, 'X-Backend-Storage-Policy-Index': policy.idx } ring = r.get_object_ring(policy.idx) expected = call(ring.devs[i], 0, 'a', 'c', 'o', headers=headers, conn_timeout=0.5, response_timeout=10) self.assertEqual(call_args, expected) self.assertEqual(r.stats_objects_deleted, 3) def test_reap_object_fail(self): r = self.init_reaper({}, fakelogger=True) self.amount_fail = 0 self.max_fail = 1 policy = random.choice(list(POLICIES)) with patch('swift.account.reaper.direct_delete_object', self.fake_direct_delete_object): r.reap_object('a', 'c', 'partition', cont_nodes, 'o', policy.idx) self.assertEqual(r.stats_objects_deleted, 1) self.assertEqual(r.stats_objects_remaining, 1) self.assertEqual(r.stats_objects_possibly_remaining, 1) def test_reap_object_non_exist_policy_index(self): r = self.init_reaper({}, fakelogger=True) r.reap_object('a', 'c', 'partition', cont_nodes, 'o', 2) self.assertEqual(r.stats_objects_deleted, 0) self.assertEqual(r.stats_objects_remaining, 1) self.assertEqual(r.stats_objects_possibly_remaining, 0) @patch('swift.account.reaper.Ring', lambda *args, **kwargs: unit.FakeRing()) def test_reap_container(self): policy = random.choice(list(POLICIES)) r = self.init_reaper({}, fakelogger=True) with patch.multiple('swift.account.reaper', direct_get_container=DEFAULT, direct_delete_object=DEFAULT, direct_delete_container=DEFAULT) as mocks: headers = {'X-Backend-Storage-Policy-Index': policy.idx} obj_listing = [{'name': 'o'}] def fake_get_container(*args, **kwargs): try: obj = obj_listing.pop(0) except IndexError: obj_list = [] else: obj_list = [obj] return headers, obj_list mocks['direct_get_container'].side_effect = fake_get_container r.reap_container('a', 'partition', acc_nodes, 'c') mock_calls = mocks['direct_delete_object'].call_args_list self.assertEqual(3, len(mock_calls)) for call_args in mock_calls: _args, kwargs = call_args self.assertEqual(kwargs['headers'] ['X-Backend-Storage-Policy-Index'], policy.idx) self.assertEquals(mocks['direct_delete_container'].call_count, 3) self.assertEqual(r.stats_objects_deleted, 3) def test_reap_container_get_object_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = True self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 0 ctx = [patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.inc['return_codes.4'], 1) self.assertEqual(r.stats_containers_deleted, 1) def test_reap_container_partial_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 2 ctx = [patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.inc['return_codes.4'], 2) self.assertEqual(r.stats_containers_possibly_remaining, 1) def test_reap_container_full_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 3 ctx = [patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.inc['return_codes.4'], 3) self.assertEqual(r.stats_containers_remaining, 1) @patch('swift.account.reaper.Ring', lambda *args, **kwargs: unit.FakeRing()) def test_reap_container_non_exist_policy_index(self): r = self.init_reaper({}, fakelogger=True) with patch.multiple('swift.account.reaper', direct_get_container=DEFAULT, direct_delete_object=DEFAULT, direct_delete_container=DEFAULT) as mocks: headers = {'X-Backend-Storage-Policy-Index': 2} obj_listing = [{'name': 'o'}] def fake_get_container(*args, **kwargs): try: obj = obj_listing.pop(0) except IndexError: obj_list = [] else: obj_list = [obj] return headers, obj_list mocks['direct_get_container'].side_effect = fake_get_container r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.msg, 'ERROR: invalid storage policy index: 2') def fake_reap_container(self, *args, **kwargs): self.called_amount += 1 self.r.stats_containers_deleted = 1 self.r.stats_objects_deleted = 1 self.r.stats_containers_remaining = 1 self.r.stats_objects_remaining = 1 self.r.stats_containers_possibly_remaining = 1 self.r.stats_objects_possibly_remaining = 1 def test_reap_account(self): containers = ('c1', 'c2', 'c3', '') broker = FakeAccountBroker(containers) self.called_amount = 0 self.r = r = self.init_reaper({}, fakelogger=True) r.start_time = time.time() ctx = [patch('swift.account.reaper.AccountReaper.reap_container', self.fake_reap_container), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring)] with nested(*ctx): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) self.assertEqual(self.called_amount, 4) self.assertEqual(r.logger.msg.find('Completed pass'), 0) self.assertTrue(r.logger.msg.find('1 containers deleted')) self.assertTrue(r.logger.msg.find('1 objects deleted')) self.assertTrue(r.logger.msg.find('1 containers remaining')) self.assertTrue(r.logger.msg.find('1 objects remaining')) self.assertTrue(r.logger.msg.find('1 containers possibly remaining')) self.assertTrue(r.logger.msg.find('1 objects possibly remaining')) def test_reap_account_no_container(self): broker = FakeAccountBroker(tuple()) self.r = r = self.init_reaper({}, fakelogger=True) self.called_amount = 0 r.start_time = time.time() ctx = [patch('swift.account.reaper.AccountReaper.reap_container', self.fake_reap_container), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring)] with nested(*ctx): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) self.assertEqual(r.logger.msg.find('Completed pass'), 0) self.assertEqual(self.called_amount, 0) def test_reap_device(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf) ctx = [patch('swift.account.reaper.AccountBroker', FakeAccountBroker), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account)] with nested(*ctx): r.reap_device('sda1') self.assertEqual(self.called_amount, 1) def test_reap_device_with_ts(self): devices = self.prepare_data_dir(ts=True) self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf=conf) ctx = [patch('swift.account.reaper.AccountBroker', FakeAccountBroker), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account)] with nested(*ctx): r.reap_device('sda1') self.assertEqual(self.called_amount, 0) def test_reap_device_with_not_my_ip(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf, myips=['10.10.1.2']) ctx = [patch('swift.account.reaper.AccountBroker', FakeAccountBroker), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account)] with nested(*ctx): r.reap_device('sda1') self.assertEqual(self.called_amount, 0) def test_run_once(self): def prepare_data_dir(): devices_path = tempfile.mkdtemp() # will be deleted by teardown self.to_delete.append(devices_path) path = os.path.join(devices_path, 'sda1', DATADIR) os.makedirs(path) return devices_path def init_reaper(devices): r = reaper.AccountReaper({'devices': devices}) return r devices = prepare_data_dir() r = init_reaper(devices) with patch('swift.account.reaper.ismount', lambda x: True): with patch( 'swift.account.reaper.AccountReaper.reap_device') as foo: r.run_once() self.assertEqual(foo.called, 1) with patch('swift.account.reaper.ismount', lambda x: False): with patch( 'swift.account.reaper.AccountReaper.reap_device') as foo: r.run_once() self.assertFalse(foo.called) def test_run_forever(self): def fake_sleep(val): self.val = val def fake_random(): return 1 def fake_run_once(): raise Exception('exit') def init_reaper(): r = reaper.AccountReaper({'interval': 1}) r.run_once = fake_run_once return r r = init_reaper() with patch('swift.account.reaper.sleep', fake_sleep): with patch('swift.account.reaper.random.random', fake_random): try: r.run_forever() except Exception as err: pass self.assertEqual(self.val, 1) self.assertEqual(str(err), 'exit')
'ip': '', 'port': ''}] cont_nodes = [{'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}] @unit.patch_policies([StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()), StoragePolicy(1, 'one', True, object_ring=unit.FakeRing())]) class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None ) def tearDown(self): for todel in self.to_delete: