def test_find_replace_enable(self): db_file = os.path.join(self.testdir, 'hash.db') broker = ContainerBroker(db_file) broker.account = 'a' broker.container = 'c' broker.initialize() ts = utils.Timestamp.now() broker.merge_items([{ 'name': 'obj%02d' % i, 'created_at': ts.internal, 'size': 0, 'content_type': 'application/octet-stream', 'etag': 'not-really', 'deleted': 0, 'storage_policy_index': 0, 'ctype_timestamp': ts.internal, 'meta_timestamp': ts.internal } for i in range(100)]) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now() as now: main([broker.db_file, 'find_and_replace', '10', '--enable']) expected = [ 'No shard ranges found to delete.', 'Injected 10 shard ranges.', 'Run container-replicator to replicate them to other nodes.', "Container moved to state 'sharding' with epoch %s." % now.internal, 'Run container-sharder on all nodes to shard the container.' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now) found_shard_ranges = broker.get_shard_ranges() self.assertEqual([(data['lower'], data['upper']) for data in self.shard_data], [(sr.lower_str, sr.upper_str) for sr in found_shard_ranges]) # Do another find & replace but quit when prompted about existing # shard ranges out = StringIO() err = StringIO() to_patch = 'swift.cli.manage_shard_ranges.input' with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err), \ mock_timestamp_now() as now, \ mock.patch(to_patch, return_value='q'): main([broker.db_file, 'find_and_replace', '10']) # Shard ranges haven't changed at all self.assertEqual(found_shard_ranges, broker.get_shard_ranges()) expected = ['This will delete existing 10 shard ranges.'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines())
def test_find_replace_enable(self): db_file = os.path.join(self.testdir, 'hash.db') broker = ContainerBroker(db_file) broker.account = 'a' broker.container = 'c' broker.initialize() ts = utils.Timestamp.now() broker.merge_items([ {'name': 'obj%02d' % i, 'created_at': ts.internal, 'size': 0, 'content_type': 'application/octet-stream', 'etag': 'not-really', 'deleted': 0, 'storage_policy_index': 0, 'ctype_timestamp': ts.internal, 'meta_timestamp': ts.internal} for i in range(100)]) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now() as now: main([broker.db_file, 'find_and_replace', '10', '--enable']) expected = [ 'No shard ranges found to delete.', 'Injected 10 shard ranges.', 'Run container-replicator to replicate them to other nodes.', "Container moved to state 'sharding' with epoch %s." % now.internal, 'Run container-sharder on all nodes to shard the container.'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now) self.assertEqual( [(data['lower'], data['upper']) for data in self.shard_data], [(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])
def test_enable(self): broker = self._make_broker() broker.update_metadata( {'X-Container-Sysmeta-Sharding': (True, Timestamp.now().internal)}) # no shard ranges out = StringIO() err = StringIO() with self.assertRaises(SystemExit): with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): main([broker.db_file, 'enable']) expected = [ "WARNING: invalid shard ranges: ['No shard ranges.'].", 'Aborting.' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) # success shard_ranges = [] for data in self.shard_data: path = ShardRange.make_path('.shards_a', 'c', 'c', Timestamp.now(), data['index']) shard_ranges.append( ShardRange(path, Timestamp.now(), data['lower'], data['upper'], data['object_count'])) broker.merge_shard_ranges(shard_ranges) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now() as now: main([broker.db_file, 'enable']) expected = [ "Container moved to state 'sharding' with epoch %s." % now.internal, 'Run container-sharder on all nodes to shard the container.' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now) # already enabled out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): main([broker.db_file, 'enable']) expected = [ "Container already in state 'sharding' with epoch %s." % now.internal, 'No action required.', 'Run container-sharder on all nodes to shard the container.' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now)
def test_compact_donors_but_no_suitable_acceptor(self): # if shard ranges are already shrinking, check that the final one is # not made into an acceptor if a suitable adjacent acceptor is not # found (unexpected scenario but possible in an overlap situation) broker = self._make_broker() shard_ranges = make_shard_ranges(broker, self.shard_data, '.shards_') for i, state in enumerate([ShardRange.SHRINKING] * 3 + [ShardRange.SHARDING] + [ShardRange.ACTIVE] * 6): shard_ranges[i].update_state(state) broker.merge_shard_ranges(shard_ranges) epoch = self._move_broker_to_sharded_state(broker) with mock_timestamp_now(epoch): own_sr = broker.get_own_shard_range(no_default=True) self.assertEqual(epoch, own_sr.state_timestamp) # sanity check self.assertEqual(ShardRange.SHARDED, own_sr.state) # sanity check out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): ret = main( [broker.db_file, 'compact', '--yes', '--max-shrinking', '99']) self.assertEqual( 0, ret, 'stdout:\n%s\nstderr\n%s' % (out.getvalue(), err.getvalue())) err_lines = err.getvalue().split('\n') self.assert_starts_with(err_lines[0], 'Loaded db broker for ') out_lines = out.getvalue().split('\n') self.assertEqual(['Updated 1 shard sequences for compaction.'], out_lines[:1]) updated_ranges = broker.get_shard_ranges() shard_ranges[9].lower = shard_ranges[4].lower # expanded acceptor self.assertEqual(shard_ranges, updated_ranges) self.assertEqual( [ShardRange.SHRINKING] * 3 + # unchanged [ShardRange.SHARDING] + # unchanged [ShardRange.SHRINKING] * 5 + # moved to shrinking [ShardRange.ACTIVE], # unchanged [sr.state for sr in updated_ranges]) with mock_timestamp_now(epoch): # force equal meta-timestamp updated_own_sr = broker.get_own_shard_range(no_default=True) self.assertEqual(dict(own_sr), dict(updated_own_sr))
def test_enable(self): broker = self._make_broker() broker.update_metadata({'X-Container-Sysmeta-Sharding': (True, Timestamp.now().internal)}) # no shard ranges out = StringIO() err = StringIO() with self.assertRaises(SystemExit): with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): main([broker.db_file, 'enable']) expected = ["WARNING: invalid shard ranges: ['No shard ranges.'].", 'Aborting.'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) # success shard_ranges = [] for data in self.shard_data: path = ShardRange.make_path( '.shards_a', 'c', 'c', Timestamp.now(), data['index']) shard_ranges.append( ShardRange(path, Timestamp.now(), data['lower'], data['upper'], data['object_count'])) broker.merge_shard_ranges(shard_ranges) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now() as now: main([broker.db_file, 'enable']) expected = [ "Container moved to state 'sharding' with epoch %s." % now.internal, 'Run container-sharder on all nodes to shard the container.'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now) # already enabled out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): main([broker.db_file, 'enable']) expected = [ "Container already in state 'sharding' with epoch %s." % now.internal, 'No action required.', 'Run container-sharder on all nodes to shard the container.'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now)
def test_info(self): broker = self._make_broker() broker.update_metadata( {'X-Container-Sysmeta-Sharding': (True, Timestamp.now().internal)}) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): main([broker.db_file, 'info']) expected = [ 'Sharding enabled = True', 'Own shard range: None', 'db_state = unsharded', 'Metadata:', ' X-Container-Sysmeta-Sharding = True' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) retiring_db_id = broker.get_info()['id'] broker.merge_shard_ranges(ShardRange('.shards/cc', Timestamp.now())) epoch = Timestamp.now() with mock_timestamp_now(epoch) as now: broker.enable_sharding(epoch) self.assertTrue(broker.set_sharding_state()) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now(now): main([broker.db_file, 'info']) expected = [ 'Sharding enabled = True', 'Own shard range: {', ' "bytes_used": 0, ', ' "deleted": 0, ', ' "epoch": "%s", ' % epoch.internal, ' "lower": "", ', ' "meta_timestamp": "%s", ' % now.internal, ' "name": "a/c", ', ' "object_count": 0, ', ' "state": "sharding", ', ' "state_timestamp": "%s", ' % now.internal, ' "timestamp": "%s", ' % now.internal, ' "upper": ""', '}', 'db_state = sharding', 'Retiring db id: %s' % retiring_db_id, 'Cleaving context: {', ' "cleave_to_row": null, ', ' "cleaving_done": false, ', ' "cursor": "", ', ' "last_cleave_to_row": null, ', ' "max_row": -1, ', ' "misplaced_done": false, ', ' "ranges_done": 0, ', ' "ranges_todo": 0, ', ' "ref": "%s"' % retiring_db_id, '}', 'Metadata:', ' X-Container-Sysmeta-Sharding = True' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self.assertTrue(broker.set_sharded_state()) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now(now): main([broker.db_file, 'info']) expected = [ 'Sharding enabled = True', 'Own shard range: {', ' "bytes_used": 0, ', ' "deleted": 0, ', ' "epoch": "%s", ' % epoch.internal, ' "lower": "", ', ' "meta_timestamp": "%s", ' % now.internal, ' "name": "a/c", ', ' "object_count": 0, ', ' "state": "sharding", ', ' "state_timestamp": "%s", ' % now.internal, ' "timestamp": "%s", ' % now.internal, ' "upper": ""', '}', 'db_state = sharded', 'Metadata:', ' X-Container-Sysmeta-Sharding = True' ] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines())
def test_info(self): broker = self._make_broker() broker.update_metadata({'X-Container-Sysmeta-Sharding': (True, Timestamp.now().internal)}) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): main([broker.db_file, 'info']) expected = ['Sharding enabled = True', 'Own shard range: None', 'db_state = unsharded', 'Metadata:', ' X-Container-Sysmeta-Sharding = True'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) retiring_db_id = broker.get_info()['id'] broker.merge_shard_ranges(ShardRange('.shards/cc', Timestamp.now())) epoch = Timestamp.now() with mock_timestamp_now(epoch) as now: broker.enable_sharding(epoch) self.assertTrue(broker.set_sharding_state()) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now(now): main([broker.db_file, 'info']) expected = ['Sharding enabled = True', 'Own shard range: {', ' "bytes_used": 0, ', ' "deleted": 0, ', ' "epoch": "%s", ' % epoch.internal, ' "lower": "", ', ' "meta_timestamp": "%s", ' % now.internal, ' "name": "a/c", ', ' "object_count": 0, ', ' "state": "sharding", ', ' "state_timestamp": "%s", ' % now.internal, ' "timestamp": "%s", ' % now.internal, ' "upper": ""', '}', 'db_state = sharding', 'Retiring db id: %s' % retiring_db_id, 'Cleaving context: {', ' "cleave_to_row": null, ', ' "cleaving_done": false, ', ' "cursor": "", ', ' "last_cleave_to_row": null, ', ' "max_row": -1, ', ' "misplaced_done": false, ', ' "ranges_done": 0, ', ' "ranges_todo": 0, ', ' "ref": "%s"' % retiring_db_id, '}', 'Metadata:', ' X-Container-Sysmeta-Sharding = True'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self.assertTrue(broker.set_sharded_state()) out = StringIO() err = StringIO() with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): with mock_timestamp_now(now): main([broker.db_file, 'info']) expected = ['Sharding enabled = True', 'Own shard range: {', ' "bytes_used": 0, ', ' "deleted": 0, ', ' "epoch": "%s", ' % epoch.internal, ' "lower": "", ', ' "meta_timestamp": "%s", ' % now.internal, ' "name": "a/c", ', ' "object_count": 0, ', ' "state": "sharding", ', ' "state_timestamp": "%s", ' % now.internal, ' "timestamp": "%s", ' % now.internal, ' "upper": ""', '}', 'db_state = sharded', 'Metadata:', ' X-Container-Sysmeta-Sharding = True'] self.assertEqual(expected, out.getvalue().splitlines()) self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines())