class TestStorageTierer(BaseTestCase): def setUp(self): super(TestStorageTierer, self).setUp() self.namespace = self.conf['namespace'] self.test_account = "test_storage_tiering_%f" % time.time() self.api = ObjectStorageApi(self.namespace) self.gridconf = { "namespace": self.namespace, "container_fetch_limit": 2, "content_fetch_limit": 2, "account": self.test_account, "outdated_threshold": 0, "new_policy": "EC" } self.api.container = ContainerClient(self.gridconf) self._populate() def _populate(self): self.container_0_name = 'container_empty' self.container_0 = self._new_container(self.container_0_name) self.container_1_name = 'container_with_1_content' self.container_1 = self._new_container(self.container_1_name) self.container_1_content_0_name = 'container_1_content_0' self.container_1_content_0 = self._new_object( self.container_1_name, self.container_1_content_0_name, 'SINGLE') self.container_2_name = 'container_with_2_contents' self.container_2 = self._new_container(self.container_1_name) self.container_2_content_0_name = 'container_2_content_0' self.container_2_content_0 = self._new_object( self.container_2_name, self.container_2_content_0_name, 'SINGLE') self.container_2_content_1_name = 'container_2_content_1' self.container_2_content_1 = self._new_object( self.container_2_name, self.container_2_content_1_name, 'TWOCOPIES') def _new_container(self, container): self.api.container_create(self.test_account, container) cnt = self.api.container_get_properties(self.test_account, container) return cnt def _new_object(self, container, obj_name, stgpol): data = random_data(10) self.api.object_create(self.test_account, container, obj_name=obj_name, policy=stgpol, data=data) obj = self.api.object_get_properties(self.test_account, container, obj_name) return obj def tearDown(self): super(TestStorageTierer, self).tearDown() def test_iter_container_list(self): worker = StorageTiererWorker(self.gridconf, Mock()) actual = [x[0] for x in self.api.container_list(self.test_account)] if len(actual) < 3: print("Slow event propagation!") # account events have not yet propagated time.sleep(3.0) actual = [ x[0] for x in self.api.container_list(self.test_account)[0] ] gen = worker._list_containers() self.assertListEqual(list(gen), actual) def test_iter_content_list_outdated_threshold_0(self): self.gridconf["outdated_threshold"] = 0 worker = StorageTiererWorker(self.gridconf, Mock()) gen = worker._list_contents() self.assertEqual((self.test_account, self.container_1_name, self.container_1_content_0_name, int(self.container_1_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_0_name, int(self.container_2_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_1_name, int(self.container_2_content_1['version'])), next(gen)) self.assertRaises(StopIteration, next, gen) def test_iter_content_list_outdated_threshold_9999999999(self): self.gridconf["outdated_threshold"] = 9999999999 worker = StorageTiererWorker(self.gridconf, Mock()) gen = worker._list_contents() self.assertRaises(StopIteration, next, gen) def test_iter_content_list_outdated_threshold_2(self): # add a new content created after the three previous contents now = int(time.time()) time.sleep(2) self._new_object(self.container_2_name, 'titi', 'TWOCOPIES') self.gridconf["outdated_threshold"] = 2 worker = StorageTiererWorker(self.gridconf, Mock()) with mock.patch('oio.crawler.storage_tierer.time.time', mock.MagicMock(return_value=now)): gen = worker._list_contents() self.assertEqual((self.test_account, self.container_1_name, self.container_1_content_0_name, int(self.container_1_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_0_name, int(self.container_2_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_1_name, int(self.container_2_content_1['version'])), next(gen)) self.assertRaises(StopIteration, next, gen) def test_iter_content_list_skip_good_policy(self): self.gridconf["new_policy"] = "SINGLE" worker = StorageTiererWorker(self.gridconf, Mock()) gen = worker._list_contents() self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_1_name, int(self.container_2_content_1['version'])), next(gen)) self.assertRaises(StopIteration, next, gen)
class TestContainerLifecycle(BaseTestCase): CONTAINERS = set() def setUp(self): super(TestContainerLifecycle, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_lifecycle" self.container = "lifecycle-" + random_str(4) self.lifecycle = ContainerLifecycle( self.api, self.account, self.container) @staticmethod def _time_to_date(timestamp=None): if timestamp is None: timestamp = time.time() return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp)) def _upload_something(self, prefix="", path=None, **kwargs): path = path or (prefix + random_str(8)) self.api.object_create(self.account, self.container, obj_name=path, data=path, **kwargs) self.__class__.CONTAINERS.add(self.container) return self.api.object_show(self.account, self.container, path) def _enable_versioning(self): if not self.api.container_create( self.account, self.container, system={'sys.m2.policy.version': '-1'}): self.api.container_set_properties( self.account, self.container, system={'sys.policy.version': '-1'}) def test_load_from_container_property(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>key1</Key> <Value>value1</Value> </Tag> <Tag> <Key>key2</Key> <Value>value2</Value> </Tag> </And> </Filter> <Status>Enabled</Status> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Expiration> <Days>60</Days> </Expiration> <NoncurrentVersionTransition> <NoncurrentDays>1</NoncurrentDays> <StorageClass>THREECOPIES</StorageClass> </NoncurrentVersionTransition> <NoncurrentVersionExpiration> <NoncurrentDays>60</NoncurrentDays> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self.lifecycle.load() self.assertIn('rule1', self.lifecycle._rules) rule = self.lifecycle._rules['rule1'] self.assertEqual('rule1', rule.id) self.assertTrue(rule.enabled) self.assertIsNotNone(rule.filter) self.assertIn('Expiration', rule.actions) self.assertIn('Transition', rule.actions) self.assertIn('NoncurrentVersionExpiration', rule.actions) self.assertIn('NoncurrentVersionTransition', rule.actions) def test_save_to_container_property(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>key1</Key> <Value>value1</Value> </Tag> <Tag> <Key>key2</Key> <Value>value2</Value> </Tag> </And> </Filter> <Status>Enabled</Status> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Expiration> <Days>60</Days> </Expiration> <NoncurrentVersionTransition> <NoncurrentDays>1</NoncurrentDays> <StorageClass>THREECOPIES</StorageClass> </NoncurrentVersionTransition> <NoncurrentVersionExpiration> <NoncurrentDays>60</NoncurrentDays> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """ self.api.container_create(self.account, self.container) self.lifecycle.load_xml(source) self.lifecycle.save() props = self.api.container_get_properties( self.account, self.container)['properties'] self.assertIn(LIFECYCLE_PROPERTY_KEY, props) self.assertEqual(source, props[LIFECYCLE_PROPERTY_KEY]) def test_immediate_expiration_by_date(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() - 86400)) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertNotIn(obj_meta['name'], obj_names) def test_future_expiration_by_date(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() + 86400)) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_expiration_by_delay(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Days>0</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertNotIn(obj_meta['name'], obj_names) def test_future_expiration_by_delay(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) self.lifecycle.apply(obj_meta) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_expiration_filtered_by_prefix(self): obj_meta = self._upload_something(prefix="photos/") obj_meta2 = self._upload_something(prefix="documents/") self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Prefix>documents/</Prefix> </Filter> <Expiration> <Days>0</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta2)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.assertNotIn(obj_meta2['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_expiration_filtered_by_tag(self): obj_meta = self._upload_something() obj_meta2 = self._upload_something(properties={'status': 'deprecated'}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Expiration> <Days>0</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta2)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.assertNotIn(obj_meta2['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_transition_filtered_by_tag(self): obj_meta = self._upload_something(policy='SINGLE') obj_meta2 = self._upload_something(policy='SINGLE', properties={'status': 'deprecated'}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Transition> <Days>0</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta2)) obj_meta_after = self.api.object_show( self.account, self.container, obj_meta['name']) obj_meta_after2 = self.api.object_show( self.account, self.container, obj_meta2['name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual('THREECOPIES', obj_meta_after2['policy']) self.api.object_delete(self.account, self.container, obj_meta['name']) self.api.object_delete(self.account, self.container, obj_meta2['name']) def test_immediate_noncurrent_expiration(self): self._enable_versioning() obj_meta = self._upload_something() obj_meta_v2 = self._upload_something(path=obj_meta['name']) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter></Filter> <NoncurrentVersionExpiration> <NoncurrentDays>0</NoncurrentDays> </NoncurrentVersionExpiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta_v2)) container_descr = self.api.object_list(self.account, self.container, versions=True) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.assertEqual(1, len(obj_names)) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_execute_immediate_expiration(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <Expiration> <Days>0</Days> </Expiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self._upload_something() self._upload_something() self._upload_something() self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(3, len(results)) listing = self.api.object_list(self.account, self.container) self.assertEqual(0, len(listing['objects'])) def test_execute_expiration_on_missing_objects(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <Expiration> <Days>0</Days> </Expiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self.lifecycle.load() fake_listing = { 'objects': [ {'name': 'a', 'ctime': '12'}, {'name': 'b', 'ctime': '12'}, {'name': 'c', 'ctime': '12'}, {'name': 'd', 'ctime': '12'}], 'truncated': False } with patch.object(self.api, 'object_list', side_effect=[fake_listing]): results = [x for x in self.lifecycle.execute()] self.assertEqual(4, len(results)) for res in results: self.assertIsInstance(res[3], Exception)
class TestMeta1RefMapping(BaseTestCase): def setUp(self): super(TestMeta1RefMapping, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_refmapping" self.reference = "refmapping-" + random_str(4) self.logger = logging.getLogger('test') self.mapping = Meta1RefMapping(self.ns, logger=self.logger) def _get_cid(self): data_dir = self.api.directory.get_properties(self.account, self.reference) return data_dir['cid'] def _get_services(self, service_type): data_dir = self.api.directory.list(self.account, self.reference) services = list() for d in data_dir['srv']: if d['type'] == service_type: services.append(d['host']) return services def _test_move(self, service_type, base=None, destination=True): if base is None: base = self._get_cid() raw_services = self._get_services(service_type) expected_services = list(raw_services) src_service = expected_services.pop() dest_service = None all_meta2_services = self.conscience.all_services(service_type, True) if len(all_meta2_services) <= len(raw_services): self.skipTest("need at least %d %s" % (len(raw_services) + 1, service_type)) if destination: for service in all_meta2_services: if service['addr'] not in raw_services: dest_service = service['addr'] expected_services.append(dest_service) moved = self.mapping.move(src_service, dest_service, base, service_type) moved_ok = self.mapping.apply(moved) self.assertEqual(1, len(moved_ok)) data_dir = self.api.directory.list(self.account, self.reference) new_services = list() for d in data_dir['srv']: if d['type'] == service_type: new_services.append(d['host']) if destination: self.assertListEqual(sorted(expected_services), sorted(new_services)) else: for expected_service in expected_services: self.assertIn(expected_service, new_services) self.assertNotIn(src_service, new_services) self.assertEqual(len(expected_services) + 1, len(new_services)) return (src_service, expected_services) def test_move_meta2(self): self.api.container_create(self.account, self.reference) _, expected_services = self._test_move('meta2') properties = self.api.container_get_properties(self.account, self.reference) peers = properties['system']['sys.peers'] new_services = peers.split(',') self.assertListEqual(sorted(expected_services), sorted(new_services)) self.api.object_create(self.account, self.reference, data="move meta2", obj_name="test") for i in range(0, 10): self.api.object_show(self.account, self.reference, "test") self.api.object_delete(self.account, self.reference, "test") sleep(0.5) self.api.container_delete(self.account, self.reference) def test_move_meta2_with_seq(self): self.api.container_create(self.account, self.reference) properties = self.api.container_get_properties(self.account, self.reference) base = properties['system']['sys.name'] _, expected_services = self._test_move('meta2', base=base) properties = self.api.container_get_properties(self.account, self.reference) peers = properties['system']['sys.peers'] new_services = peers.split(',') self.assertListEqual(sorted(expected_services), sorted(new_services)) self.api.object_create(self.account, self.reference, data="move meta2", obj_name="test") for i in range(0, 10): self.api.object_show(self.account, self.reference, "test") self.api.object_delete(self.account, self.reference, "test") sleep(0.5) self.api.container_delete(self.account, self.reference) def test_move_meta2_without_destination(self): self.api.container_create(self.account, self.reference) properties = self.api.container_get_properties(self.account, self.reference) base = properties['system']['sys.name'] src_service, expected_services = self._test_move('meta2', base=base, destination=False) properties = self.api.container_get_properties(self.account, self.reference) peers = properties['system']['sys.peers'] new_services = peers.split(',') for expected_service in expected_services: self.assertIn(expected_service, new_services) self.assertNotIn(src_service, new_services) self.assertEqual(len(expected_services) + 1, len(new_services)) self.api.object_create(self.account, self.reference, data="move meta2", obj_name="test") for i in range(0, 10): self.api.object_show(self.account, self.reference, "test") self.api.object_delete(self.account, self.reference, "test") sleep(0.5) self.api.container_delete(self.account, self.reference) def test_move_sqlx(self): execute('oio-sqlx -O AutoCreate %s/%s/%s ' '"create table foo (a INT, b TEXT)"' % (self.ns, self.account, self.reference)) self._test_move('sqlx') execute('oio-sqlx %s/%s/%s "destroy"' % (self.ns, self.account, self.reference)) def test_move_with_dest_service_already_used(self): self.api.container_create(self.account, self.reference) base = self._get_cid() raw_services = self._get_services('meta2') src_service = raw_services[0] dest_service = raw_services[0] if len(raw_services) > 1: dest_service = raw_services[1] self.assertRaises(ValueError, self.mapping.move, src_service, dest_service, base, 'meta2') self.api.container_delete(self.account, self.reference) def test_move_with_src_service_not_used(self): self.api.container_create(self.account, self.reference) base = self._get_cid() raw_services = self._get_services('meta2') src_service = '127.0.0.1:666' dest_service = None all_meta2_services = self.conscience.all_services('meta2', True) for service in all_meta2_services: if service['addr'] not in raw_services: src_service = service['addr'] dest_service = service['addr'] if dest_service is None: self.skipTest("need at least 2 meta2") self.assertRaises(ValueError, self.mapping.move, src_service, dest_service, base, 'meta2') self.api.container_delete(self.account, self.reference) def test_move_with_wrong_dest(self): self.api.container_create(self.account, self.reference) base = self._get_cid() raw_services = self._get_services('meta2') src_service = raw_services[0] dest_service = '127.0.0.1:666' self.assertRaises(ValueError, self.mapping.move, src_service, dest_service, base, 'meta2') self.api.container_delete(self.account, self.reference)