def setUp(self): super(TestAdmin, self).setUp() self.admin = AdminClient(self.conf) self.api = ObjectStorageApi(self.ns) self.account = "test_admin" self.container = "admin-" + random_str(4) self.api.container_create(self.account, self.container)
class MetaRebuilder(Rebuilder): """ Abstract class for directory rebuilders. """ def __init__(self, conf, logger, volume, **kwargs): super(MetaRebuilder, self).__init__(conf, logger, volume, **kwargs) self.api = ObjectStorageApi(self.namespace, logger=self.logger, **kwargs) def _fill_queue_from_file(self, queue, **kwargs): if self.input_file is None: return False with open(self.input_file, 'r') as ifile: for line in ifile: stripped = line.strip() if stripped and not stripped.startswith('#'): queue.put(stripped) if not self.running: break return True def _full_container_list(self, account, **kwargs): listing = self.api.container_list(account, **kwargs) for element in listing: yield element while listing: kwargs['marker'] = listing[-1][0] listing = self.api.container_list(account, **kwargs) if listing: for element in listing: yield element
def setUp(self): super(TestMeta1RefMapping, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_refmapping" self.reference = "refmapping-" + random_str(4) self.logger = logging.getLogger('test') self.mapping = Meta1RefMapping(self.ns, logger=self.logger)
def setUp(self): super(TestContainerLifecycle, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_lifecycle" self.container = "lifecycle-" + random_str(4) self.lifecycle = ContainerLifecycle( self.api, self.account, self.container)
def test_iter_container_list(self): worker = StorageTiererWorker(self.gridconf, Mock()) api = ObjectStorageApi(self.namespace) actual = [x[0] for x in api.container_list(self.test_account)] if len(actual) < 3: print "Slow event propagation!" # account events have not yet propagated time.sleep(3.0) actual = [x[0] for x in api.container_list(self.test_account)[0]] gen = worker._list_containers() self.assertListEqual(list(gen), actual)
def setUp(self): super(TestStorageTierer, self).setUp() self.namespace = self.conf['namespace'] self.test_account = "test_storage_tiering_%f" % time.time() self.api = ObjectStorageApi(self.namespace) self.gridconf = { "namespace": self.namespace, "container_fetch_limit": 2, "content_fetch_limit": 2, "account": self.test_account, "outdated_threshold": 0, "new_policy": "EC" } self.api.container = ContainerClient(self.gridconf) self._populate()
def __init__(self, conf, logger=None, account_ring=None, container_ring=None, storage=None): for policy, ring_arg in zip(OIO_POLICIES, RING_ARGS): if ring_arg is not None: policy.object_ring = FakeRing(**ring_arg) SwiftApplication.__init__(self, conf, logger=logger, account_ring=account_ring, container_ring=container_ring) if conf is None: conf = dict() sds_conf = {k[4:]: v for k, v in conf.items() if k.startswith("sds_")} self.oio_stgpol = [] if 'auto_storage_policies' in conf: for elem in conf['auto_storage_policies'].split(','): if ':' in elem: name, offset = elem.split(':') self.oio_stgpol.append((name, int(offset))) else: self.oio_stgpol.append((elem, 0)) self.oio_stgpol.sort(key=lambda x: x[1]) policies = [] if 'oio_storage_policies' in conf: for i, pol in enumerate(conf['oio_storage_policies'].split(',')): policies.append( storage_policy.StoragePolicy(i, pol, is_default=i == 0)) else: policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True)) self.POLICIES = storage_policy.StoragePolicyCollection(policies) # Mandatory, raises KeyError sds_namespace = sds_conf['namespace'] sds_conf.pop('namespace') # removed to avoid unpacking conflict # Loaded by ObjectStorageApi if None sds_proxy_url = sds_conf.pop('proxy_url', None) # Fix boolean parameter if 'autocreate' in sds_conf and not ( hasattr(ObjectStorageApi, 'EXTRA_KEYWORDS') or 'autocreate' in ObjectStorageApi.EXTRA_KEYWORDS): logger.warn("'autocreate' parameter is ignored by current version" " of OpenIO SDS. Please update to oio>=4.1.23.") else: sds_conf['autocreate'] = config_true_value( sds_conf.get('autocreate', 'true')) self.storage = storage or \ ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf) self.delete_slo_parts = \ config_true_value(conf.get('delete_slo_parts', False)) self.check_state = \ config_true_value(conf.get('check_state', False))
class AccountRebuilderWorker(ToolWorker): def __init__(self, tool, queue_workers, queue_reply): super(AccountRebuilderWorker, self).__init__(tool, queue_workers, queue_reply) self.api = ObjectStorageApi(self.tool.namespace, logger=self.logger) def _process_item(self, item): namespace, account, container = item if namespace != self.tool.namespace: raise ValueError('Invalid namespace (actual=%s, expected=%s)' % (namespace, self.tool.namespace)) if container is None: self.api.account.account_refresh(account) else: self.api.container_refresh(account, container)
def setUp(self): super(TestContentRebuildFilter, self).setUp() self.namespace = self.conf['namespace'] self.gridconf = {"namespace": self.namespace} self.container = "TestContentRebuildFilter%f" % time.time() self.ref = self.container self.container_client = ContainerClient(self.conf) self.container_client.container_create(self.account, self.container) syst = self.container_client.container_get_properties( self.account, self.container)['system'] self.container_id = syst['sys.name'].split('.', 1)[0] self.object_storage_api = ObjectStorageApi(namespace=self.namespace) self.stgpol = "SINGLE" self.conf['tube'] = 'rebuild' self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300' self.notify_filter = NotifyFilter(app=_App, conf=self.conf) queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300') self.tube = self.conf.get('tube', 'rebuild') self.beanstalk = Beanstalk.from_url(queue_url) self.beanstalk.use(self.tube)
def __init__(self, conf, logger): self.conf = conf self.logger = logger self.account = conf[CONF_ACCOUNT] self.api = ObjectStorageApi(conf['namespace'], logger=self.logger) self.container_client = self.api.container self.account_client = self.api.account self.passes = 0 self.errors = 0 self.last_reported = 0 self.contents_run_time = 0 self.total_contents_processed = 0 self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_contents_per_second = int_value( conf.get('contents_per_second'), 30) self.container_fetch_limit = int_value( conf.get('container_fetch_limit'), 100) self.content_fetch_limit = int_value(conf.get('content_fetch_limit'), 100) self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD), 9999999999) self.new_policy = conf.get(CONF_NEW_POLICY)
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, storage=None): for policy, ring_arg in zip(POLICIES, ring_args): if ring_arg is not None: policy.object_ring = FakeRing(**ring_arg) SwiftApplication.__init__(self, conf, memcache=memcache, logger=logger, account_ring=account_ring, container_ring=container_ring) if conf is None: conf = dict() sds_conf = { k[4:]: v for k, v in conf.iteritems() if k.startswith("sds_") } self.oio_stgpol = [] if 'auto_storage_policies' in conf: for elem in conf['auto_storage_policies'].split(','): if ':' in elem: name, offset = elem.split(':') self.oio_stgpol.append((name, int(offset))) else: self.oio_stgpol.append((elem, 0)) self.oio_stgpol.sort(key=lambda x: x[1]) policies = [] if 'oio_storage_policies' in conf: for i, pol in enumerate(conf['oio_storage_policies'].split(',')): policies.append( storage_policy.StoragePolicy(i, pol, is_default=i == 0)) else: policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True)) self.POLICIES = storage_policy.StoragePolicyCollection(policies) # Mandatory, raises KeyError sds_namespace = sds_conf['namespace'] sds_conf.pop('namespace') # removed to avoid unpacking conflict # Loaded by ObjectStorageApi if None sds_proxy_url = sds_conf.pop('proxy_url', None) self.storage = storage or \ ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
def setUpClass(cls): super(ItemRebuildTest, cls).setUpClass() cls.api = ObjectStorageApi(cls._cls_ns, endpoint=cls._cls_uri)
def setUpClass(cls): super(ObjectStoragePropertiesTest, cls).setUpClass() cls.logger = logging.getLogger('test') cls.api = ObjectStorageApi(cls._cls_ns, cls.logger) cls.obj_cname = "obj_props_" + random_str(8)
print "%d containers in %fs, %f containers per second." % ( counter, now - checkpoint, counter / (now - checkpoint)) counter = 0 checkpoint = now created.append(res) now = time.time() for coro in POOL.coroutines_running: coro.kill() while not RESULTS.empty(): created.append(RESULTS.get(block=False)) end = time.time() rate = len(created) / (end - start) print "End. %d containers created in %fs, %f containers per second." % ( len(created), end - start, rate) print "Cleaning..." for _ in POOL.starmap(API.container_delete, [('benchmark', n) for n in created]): pass POOL.waitall() return rate if __name__ == '__main__': import os import sys THREADS = int(sys.argv[1]) if len(sys.argv) > 1 else 1 API = ObjectStorageApi(os.getenv('OIO_NS', 'OPENIO')) RESULTS = LightQueue(THREADS * 10) POOL = GreenPool(THREADS) main(THREADS)
class TestStorageTierer(BaseTestCase): def setUp(self): super(TestStorageTierer, self).setUp() self.namespace = self.conf['namespace'] self.test_account = "test_storage_tiering_%f" % time.time() self.api = ObjectStorageApi(self.namespace) self.gridconf = { "namespace": self.namespace, "container_fetch_limit": 2, "content_fetch_limit": 2, "account": self.test_account, "outdated_threshold": 0, "new_policy": "EC" } self.api.container = ContainerClient(self.gridconf) self._populate() def _populate(self): self.container_0_name = 'container_empty' self.container_0 = self._new_container(self.container_0_name) self.container_1_name = 'container_with_1_content' self.container_1 = self._new_container(self.container_1_name) self.container_1_content_0_name = 'container_1_content_0' self.container_1_content_0 = self._new_object( self.container_1_name, self.container_1_content_0_name, 'SINGLE') self.container_2_name = 'container_with_2_contents' self.container_2 = self._new_container(self.container_1_name) self.container_2_content_0_name = 'container_2_content_0' self.container_2_content_0 = self._new_object( self.container_2_name, self.container_2_content_0_name, 'SINGLE') self.container_2_content_1_name = 'container_2_content_1' self.container_2_content_1 = self._new_object( self.container_2_name, self.container_2_content_1_name, 'TWOCOPIES') def _new_container(self, container): self.api.container_create(self.test_account, container) cnt = self.api.container_get_properties(self.test_account, container) return cnt def _new_object(self, container, obj_name, stgpol): data = random_data(10) self.api.object_create(self.test_account, container, obj_name=obj_name, policy=stgpol, data=data) obj = self.api.object_get_properties(self.test_account, container, obj_name) return obj def tearDown(self): super(TestStorageTierer, self).tearDown() def test_iter_container_list(self): worker = StorageTiererWorker(self.gridconf, Mock()) actual = [x[0] for x in self.api.container_list(self.test_account)] if len(actual) < 3: print("Slow event propagation!") # account events have not yet propagated time.sleep(3.0) actual = [ x[0] for x in self.api.container_list(self.test_account)[0] ] gen = worker._list_containers() self.assertListEqual(list(gen), actual) def test_iter_content_list_outdated_threshold_0(self): self.gridconf["outdated_threshold"] = 0 worker = StorageTiererWorker(self.gridconf, Mock()) gen = worker._list_contents() self.assertEqual((self.test_account, self.container_1_name, self.container_1_content_0_name, int(self.container_1_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_0_name, int(self.container_2_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_1_name, int(self.container_2_content_1['version'])), next(gen)) self.assertRaises(StopIteration, next, gen) def test_iter_content_list_outdated_threshold_9999999999(self): self.gridconf["outdated_threshold"] = 9999999999 worker = StorageTiererWorker(self.gridconf, Mock()) gen = worker._list_contents() self.assertRaises(StopIteration, next, gen) def test_iter_content_list_outdated_threshold_2(self): # add a new content created after the three previous contents now = int(time.time()) time.sleep(2) self._new_object(self.container_2_name, 'titi', 'TWOCOPIES') self.gridconf["outdated_threshold"] = 2 worker = StorageTiererWorker(self.gridconf, Mock()) with mock.patch('oio.crawler.storage_tierer.time.time', mock.MagicMock(return_value=now)): gen = worker._list_contents() self.assertEqual((self.test_account, self.container_1_name, self.container_1_content_0_name, int(self.container_1_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_0_name, int(self.container_2_content_0['version'])), next(gen)) self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_1_name, int(self.container_2_content_1['version'])), next(gen)) self.assertRaises(StopIteration, next, gen) def test_iter_content_list_skip_good_policy(self): self.gridconf["new_policy"] = "SINGLE" worker = StorageTiererWorker(self.gridconf, Mock()) gen = worker._list_contents() self.assertEqual((self.test_account, self.container_2_name, self.container_2_content_1_name, int(self.container_2_content_1['version'])), next(gen)) self.assertRaises(StopIteration, next, gen)
help='Start a worker connecter to the given addresses') parser.add_argument("--controller", dest="controller", action='store_true', default=False, help='Start a controller bond to the given addresses') parser.add_argument("endpoints", metavar='ENDPOINT', type=str, nargs='+', help='Endpoints to connect/bind to') args = parser.parse_args() zctx = zmq.Context() if args.controller: s = ObjectStorageApi("benchmark") #Creating account ac = AccountClient({"namespace": "benchmark"}) retry = 3 for i in range(retry + 1): try: ac.account_create("benchmark_account") break except ClientException: if i < retry: time.sleep(2) else: raise #Creating Container
def __init__(self, conf, logger, **kwargs): super(MetaRebuilder, self).__init__(conf, logger, **kwargs) self.api = ObjectStorageApi(self.conf['namespace'], logger=self.logger)
'--object-count', type=int, help='Total number of objects to create (duration will be ignored).') args = parser.parse_args() THREADS = args.concurrency POLICY = args.storage_policy DURATION = 60.0 OBJECT_COUNT = 1000 CREATION_MODE = 'duration' if args.duration is not None: CREATION_MODE = 'duration' DURATION = args.duration if args.object_count is not None: CREATION_MODE = 'object count' ACCOUNT = os.getenv('OIO_ACCOUNT', 'benchmark') API = ObjectStorageApi(os.getenv('OIO_NS', 'OPENIO')) API._object_upload = _object_upload RESULTS = LightQueue(THREADS * 10) POOL = GreenPool(THREADS) if CREATION_MODE == 'duration': print("Creating objects during %f seconds." % DURATION) create_objects_in_period(THREADS, duration=DURATION) else: print("Creating %d objects." % args.object_count) create_fixed_number_of_objects(THREADS, obj_count=args.object_count)
def __init__(self, tool, queue_workers, queue_reply): super(AccountRebuilderWorker, self).__init__(tool, queue_workers, queue_reply) self.api = ObjectStorageApi(self.tool.namespace, logger=self.logger)
class TestContainerLifecycle(BaseTestCase): CONTAINERS = set() def setUp(self): super(TestContainerLifecycle, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_lifecycle" self.container = "lifecycle-" + random_str(4) self.lifecycle = ContainerLifecycle( self.api, self.account, self.container) @staticmethod def _time_to_date(timestamp=None): if timestamp is None: timestamp = time.time() return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp)) def _upload_something(self, prefix="", path=None, **kwargs): path = path or (prefix + random_str(8)) self.api.object_create(self.account, self.container, obj_name=path, data=path, **kwargs) self.__class__.CONTAINERS.add(self.container) return self.api.object_show(self.account, self.container, path) def _enable_versioning(self): if not self.api.container_create( self.account, self.container, system={'sys.m2.policy.version': '-1'}): self.api.container_set_properties( self.account, self.container, system={'sys.policy.version': '-1'}) def test_load_from_container_property(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>key1</Key> <Value>value1</Value> </Tag> <Tag> <Key>key2</Key> <Value>value2</Value> </Tag> </And> </Filter> <Status>Enabled</Status> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Expiration> <Days>60</Days> </Expiration> <NoncurrentVersionTransition> <NoncurrentDays>1</NoncurrentDays> <StorageClass>THREECOPIES</StorageClass> </NoncurrentVersionTransition> <NoncurrentVersionExpiration> <NoncurrentDays>60</NoncurrentDays> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self.lifecycle.load() self.assertIn('rule1', self.lifecycle._rules) rule = self.lifecycle._rules['rule1'] self.assertEqual('rule1', rule.id) self.assertTrue(rule.enabled) self.assertIsNotNone(rule.filter) self.assertIn('Expiration', rule.actions) self.assertIn('Transition', rule.actions) self.assertIn('NoncurrentVersionExpiration', rule.actions) self.assertIn('NoncurrentVersionTransition', rule.actions) def test_save_to_container_property(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>key1</Key> <Value>value1</Value> </Tag> <Tag> <Key>key2</Key> <Value>value2</Value> </Tag> </And> </Filter> <Status>Enabled</Status> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Expiration> <Days>60</Days> </Expiration> <NoncurrentVersionTransition> <NoncurrentDays>1</NoncurrentDays> <StorageClass>THREECOPIES</StorageClass> </NoncurrentVersionTransition> <NoncurrentVersionExpiration> <NoncurrentDays>60</NoncurrentDays> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """ self.api.container_create(self.account, self.container) self.lifecycle.load_xml(source) self.lifecycle.save() props = self.api.container_get_properties( self.account, self.container)['properties'] self.assertIn(LIFECYCLE_PROPERTY_KEY, props) self.assertEqual(source, props[LIFECYCLE_PROPERTY_KEY]) def test_immediate_expiration_by_date(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() - 86400)) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertNotIn(obj_meta['name'], obj_names) def test_future_expiration_by_date(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() + 86400)) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_expiration_by_delay(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Days>0</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertNotIn(obj_meta['name'], obj_names) def test_future_expiration_by_delay(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) self.lifecycle.apply(obj_meta) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_expiration_filtered_by_prefix(self): obj_meta = self._upload_something(prefix="photos/") obj_meta2 = self._upload_something(prefix="documents/") self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Prefix>documents/</Prefix> </Filter> <Expiration> <Days>0</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta2)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.assertNotIn(obj_meta2['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_expiration_filtered_by_tag(self): obj_meta = self._upload_something() obj_meta2 = self._upload_something(properties={'status': 'deprecated'}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Expiration> <Days>0</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta2)) container_descr = self.api.object_list(self.account, self.container) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.assertNotIn(obj_meta2['name'], obj_names) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_immediate_transition_filtered_by_tag(self): obj_meta = self._upload_something(policy='SINGLE') obj_meta2 = self._upload_something(policy='SINGLE', properties={'status': 'deprecated'}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Transition> <Days>0</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta2)) obj_meta_after = self.api.object_show( self.account, self.container, obj_meta['name']) obj_meta_after2 = self.api.object_show( self.account, self.container, obj_meta2['name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual('THREECOPIES', obj_meta_after2['policy']) self.api.object_delete(self.account, self.container, obj_meta['name']) self.api.object_delete(self.account, self.container, obj_meta2['name']) def test_immediate_noncurrent_expiration(self): self._enable_versioning() obj_meta = self._upload_something() obj_meta_v2 = self._upload_something(path=obj_meta['name']) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter></Filter> <NoncurrentVersionExpiration> <NoncurrentDays>0</NoncurrentDays> </NoncurrentVersionExpiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) consume(self.lifecycle.apply(obj_meta)) consume(self.lifecycle.apply(obj_meta_v2)) container_descr = self.api.object_list(self.account, self.container, versions=True) obj_names = [obj['name'] for obj in container_descr['objects']] self.assertIn(obj_meta['name'], obj_names) self.assertEqual(1, len(obj_names)) self.api.object_delete(self.account, self.container, obj_meta['name']) def test_execute_immediate_expiration(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <Expiration> <Days>0</Days> </Expiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self._upload_something() self._upload_something() self._upload_something() self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(3, len(results)) listing = self.api.object_list(self.account, self.container) self.assertEqual(0, len(listing['objects'])) def test_execute_expiration_on_missing_objects(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <Expiration> <Days>0</Days> </Expiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self.lifecycle.load() fake_listing = { 'objects': [ {'name': 'a', 'ctime': '12'}, {'name': 'b', 'ctime': '12'}, {'name': 'c', 'ctime': '12'}, {'name': 'd', 'ctime': '12'}], 'truncated': False } with patch.object(self.api, 'object_list', side_effect=[fake_listing]): results = [x for x in self.lifecycle.execute()] self.assertEqual(4, len(results)) for res in results: self.assertIsInstance(res[3], Exception)
class TestAdmin(BaseTestCase): def setUp(self): super(TestAdmin, self).setUp() self.admin = AdminClient(self.conf) self.api = ObjectStorageApi(self.ns) self.account = "test_admin" self.container = "admin-" + random_str(4) self.api.container_create(self.account, self.container) def tearDown(self): super(TestAdmin, self).tearDown() try: self.api.container_delete(self.account, self.container) except Exception: pass def test_election_leave_service_id(self): status = self.admin.election_status("meta2", account=self.account, reference=self.container) peers = status["peers"] service_id = peers.keys()[random.randrange(len(peers))] election = self.admin.election_leave("meta2", account=self.account, reference=self.container, service_id=service_id) self.assertEqual(1, len(election)) self.assertEqual(200, election[service_id]["status"]["status"]) def test_election_leave_serveral_service_ids(self): status = self.admin.election_status("meta2", account=self.account, reference=self.container) peers = status["peers"] if len(peers) < 2: self.skipTest('Can only run in a replicated environment') service_ids = peers.keys()[:2] election = self.admin.election_leave("meta2", account=self.account, reference=self.container, service_id=service_ids) self.assertEquals(2, len(election)) self.assertEquals(200, election[service_ids[0]]["status"]["status"]) self.assertEquals(200, election[service_ids[1]]["status"]["status"]) def test_has_base(self): info = self.admin.has_base('meta2', account=self.account, reference=self.container) for peer, meta in info.iteritems(): self.assertEqual(200, meta['status']['status']) peer = info.keys()[0] peer_loc = info[peer]['body'] self.assertTrue(os.path.isfile(peer_loc)) os.remove(peer_loc) info = self.admin.has_base('meta2', account=self.account, reference=self.container) self.assertNotEquals(200, info[peer]['status']['status']) del info[peer] for peer, meta in info.iteritems(): self.assertEqual(200, meta['status']['status'])
class TestMeta1RefMapping(BaseTestCase): def setUp(self): super(TestMeta1RefMapping, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_refmapping" self.reference = "refmapping-" + random_str(4) self.logger = logging.getLogger('test') self.mapping = Meta1RefMapping(self.ns, logger=self.logger) def _get_cid(self): data_dir = self.api.directory.get_properties(self.account, self.reference) return data_dir['cid'] def _get_services(self, service_type): data_dir = self.api.directory.list(self.account, self.reference) services = list() for d in data_dir['srv']: if d['type'] == service_type: services.append(d['host']) return services def _test_move(self, service_type, base=None, destination=True): if base is None: base = self._get_cid() raw_services = self._get_services(service_type) expected_services = list(raw_services) src_service = expected_services.pop() dest_service = None all_meta2_services = self.conscience.all_services(service_type, True) if len(all_meta2_services) <= len(raw_services): self.skipTest("need at least %d %s" % (len(raw_services) + 1, service_type)) if destination: for service in all_meta2_services: if service['addr'] not in raw_services: dest_service = service['addr'] expected_services.append(dest_service) moved = self.mapping.move(src_service, dest_service, base, service_type) moved_ok = self.mapping.apply(moved) self.assertEqual(1, len(moved_ok)) data_dir = self.api.directory.list(self.account, self.reference) new_services = list() for d in data_dir['srv']: if d['type'] == service_type: new_services.append(d['host']) if destination: self.assertListEqual(sorted(expected_services), sorted(new_services)) else: for expected_service in expected_services: self.assertIn(expected_service, new_services) self.assertNotIn(src_service, new_services) self.assertEqual(len(expected_services) + 1, len(new_services)) return (src_service, expected_services) def test_move_meta2(self): self.api.container_create(self.account, self.reference) _, expected_services = self._test_move('meta2') properties = self.api.container_get_properties(self.account, self.reference) peers = properties['system']['sys.peers'] new_services = peers.split(',') self.assertListEqual(sorted(expected_services), sorted(new_services)) self.api.object_create(self.account, self.reference, data="move meta2", obj_name="test") for i in range(0, 10): self.api.object_show(self.account, self.reference, "test") self.api.object_delete(self.account, self.reference, "test") sleep(0.5) self.api.container_delete(self.account, self.reference) def test_move_meta2_with_seq(self): self.api.container_create(self.account, self.reference) properties = self.api.container_get_properties(self.account, self.reference) base = properties['system']['sys.name'] _, expected_services = self._test_move('meta2', base=base) properties = self.api.container_get_properties(self.account, self.reference) peers = properties['system']['sys.peers'] new_services = peers.split(',') self.assertListEqual(sorted(expected_services), sorted(new_services)) self.api.object_create(self.account, self.reference, data="move meta2", obj_name="test") for i in range(0, 10): self.api.object_show(self.account, self.reference, "test") self.api.object_delete(self.account, self.reference, "test") sleep(0.5) self.api.container_delete(self.account, self.reference) def test_move_meta2_without_destination(self): self.api.container_create(self.account, self.reference) properties = self.api.container_get_properties(self.account, self.reference) base = properties['system']['sys.name'] src_service, expected_services = self._test_move('meta2', base=base, destination=False) properties = self.api.container_get_properties(self.account, self.reference) peers = properties['system']['sys.peers'] new_services = peers.split(',') for expected_service in expected_services: self.assertIn(expected_service, new_services) self.assertNotIn(src_service, new_services) self.assertEqual(len(expected_services) + 1, len(new_services)) self.api.object_create(self.account, self.reference, data="move meta2", obj_name="test") for i in range(0, 10): self.api.object_show(self.account, self.reference, "test") self.api.object_delete(self.account, self.reference, "test") sleep(0.5) self.api.container_delete(self.account, self.reference) def test_move_sqlx(self): execute('oio-sqlx -O AutoCreate %s/%s/%s ' '"create table foo (a INT, b TEXT)"' % (self.ns, self.account, self.reference)) self._test_move('sqlx') execute('oio-sqlx %s/%s/%s "destroy"' % (self.ns, self.account, self.reference)) def test_move_with_dest_service_already_used(self): self.api.container_create(self.account, self.reference) base = self._get_cid() raw_services = self._get_services('meta2') src_service = raw_services[0] dest_service = raw_services[0] if len(raw_services) > 1: dest_service = raw_services[1] self.assertRaises(ValueError, self.mapping.move, src_service, dest_service, base, 'meta2') self.api.container_delete(self.account, self.reference) def test_move_with_src_service_not_used(self): self.api.container_create(self.account, self.reference) base = self._get_cid() raw_services = self._get_services('meta2') src_service = '127.0.0.1:666' dest_service = None all_meta2_services = self.conscience.all_services('meta2', True) for service in all_meta2_services: if service['addr'] not in raw_services: src_service = service['addr'] dest_service = service['addr'] if dest_service is None: self.skipTest("need at least 2 meta2") self.assertRaises(ValueError, self.mapping.move, src_service, dest_service, base, 'meta2') self.api.container_delete(self.account, self.reference) def test_move_with_wrong_dest(self): self.api.container_create(self.account, self.reference) base = self._get_cid() raw_services = self._get_services('meta2') src_service = raw_services[0] dest_service = '127.0.0.1:666' self.assertRaises(ValueError, self.mapping.move, src_service, dest_service, base, 'meta2') self.api.container_delete(self.account, self.reference)
class StorageTiererWorker(object): def __init__(self, conf, logger): self.conf = conf self.logger = logger self.account = conf[CONF_ACCOUNT] self.api = ObjectStorageApi(conf['namespace'], logger=self.logger) self.container_client = self.api.container self.account_client = self.api.account self.passes = 0 self.errors = 0 self.last_reported = 0 self.contents_run_time = 0 self.total_contents_processed = 0 self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_contents_per_second = int_value( conf.get('contents_per_second'), 30) self.container_fetch_limit = int_value( conf.get('container_fetch_limit'), 100) self.content_fetch_limit = int_value(conf.get('content_fetch_limit'), 100) self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD), 9999999999) self.new_policy = conf.get(CONF_NEW_POLICY) def _list_containers(self): container = None while True: resp = self.account_client.container_list( self.account, marker=container, limit=self.container_fetch_limit) if len(resp["listing"]) == 0: break for res in resp["listing"]: container = res[0] yield container def _list_contents(self): for container in self._list_containers(): marker = None while True: try: _, listing = self.container_client.content_list( account=self.account, reference=container, limit=self.content_fetch_limit, marker=marker) except NotFound: self.logger.warn( "Container %s appears in account but doesn't exist", container) break if len(listing["objects"]) == 0: break for obj in listing["objects"]: marker = obj["name"] if obj["mtime"] > time.time() - self.outdated_threshold: continue if obj["policy"] == self.new_policy: continue if true_value(obj['deleted']): continue yield (self.account, container, obj["name"], obj["version"]) def run(self): start_time = report_time = time.time() total_errors = 0 for (account, container, obj, version) in self._list_contents(): self.safe_change_policy(account, container, obj, version) self.contents_run_time = ratelimit(self.contents_run_time, self.max_contents_per_second) self.total_contents_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(total).2f ' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'total': (now - start_time) }) report_time = now total_errors += self.errors self.passes = 0 self.errors = 0 self.last_reported = now elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(content_rate).2f ' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'content_rate': self.total_contents_processed / elapsed }) def safe_change_policy(self, account, container, obj, version): try: self.change_policy(account, container, obj, version) except Exception: self.errors += 1 self.logger.exception( "ERROR while changing policy for content " "%s/%s/%s/%s", account, container, obj, str(version)) self.passes += 1 def change_policy(self, account, container, obj, version): self.logger.info("Changing policy for content %s/%s/%s/%s", account, container, obj, str(version)) self.api.object_change_policy(self, account, container, obj, self.new_policy, version=version)
class TestContentRebuildFilter(BaseTestCase): def setUp(self): super(TestContentRebuildFilter, self).setUp() self.namespace = self.conf['namespace'] self.gridconf = {"namespace": self.namespace} self.container = "TestContentRebuildFilter%f" % time.time() self.ref = self.container self.container_client = ContainerClient(self.conf) self.container_client.container_create(self.account, self.container) syst = self.container_client.container_get_properties( self.account, self.container)['system'] self.container_id = syst['sys.name'].split('.', 1)[0] self.object_storage_api = ObjectStorageApi(namespace=self.namespace) self.stgpol = "SINGLE" self.conf['tube'] = 'rebuild' self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300' self.notify_filter = NotifyFilter(app=_App, conf=self.conf) queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300') self.tube = self.conf.get('tube', 'rebuild') self.beanstalk = Beanstalk.from_url(queue_url) self.beanstalk.use(self.tube) def _create_event(self, content_name, present_chunks, missing_chunks, content_id): event = {} event["when"] = time.time() event["event"] = "storage.content.broken" event["data"] = { "present_chunks": present_chunks, "missing_chunks": missing_chunks } event["url"] = { "ns": self.namespace, "account": self.account, "user": self.container, "path": content_name, "id": self.container_id, "content": content_id } return event def _is_chunks_created(self, previous, after, pos_created): remain = list(after) for p in previous: for r in remain: if p["url"] == r["url"]: remain.remove(r) break if len(remain) != len(pos_created): return False for r in remain: if r["pos"] in pos_created: remain.remove(r) else: return False return True def _rebuild(self, event, job_id=0): self.blob_rebuilder = subprocess.Popen([ 'oio-blob-rebuilder', self.namespace, '--beanstalkd=127.0.0.1:11300' ]) time.sleep(3) self.blob_rebuilder.kill() def _remove_chunks(self, chunks, content_id): if not chunks: return for chunk in chunks: chunk['id'] = chunk['url'] chunk['content'] = content_id chunk['type'] = 'chunk' self.container_client.container_raw_delete(self.account, self.container, data=chunks) def _check_rebuild(self, content_name, chunks, missing_pos, meta, chunks_to_remove, chunk_created=True): self._remove_chunks(chunks_to_remove, meta['id']) event = self._create_event(content_name, chunks, missing_pos, meta['id']) self.notify_filter.process(env=event, cb=None) self._rebuild(event) _, after = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) self.assertIs(chunk_created, self._is_chunks_created(chunks, after, missing_pos)) def test_nothing_missing(self): content_name = "test_nothing_missing" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="THREECOPIES", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] for chunk in chunks: chunk.pop('score', None) missing_pos = [] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove, chunk_created=True) def test_missing_1_chunk(self): content_name = "test_missing_1_chunk" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="THREECOPIES", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_last_chunk(self): content_name = "test_missing_last_chunk" data = random_str(1024 * 1024 * 4) self.object_storage_api.object_create(account=self.account, container=self.container, data=data, policy="THREECOPIES", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["3"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_2_chunks(self): content_name = "test_missing_2_chunks" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="THREECOPIES", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] for i in range(0, 2): chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0", "0"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_all_chunks(self): content_name = "test_missing_all_chunks" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="SINGLE", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove, chunk_created=False) def test_missing_all_chunks_of_a_pos(self): content_name = "test_missing_2_chunks" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="THREECOPIES", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] for i in range(0, 3): chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove, chunk_created=False) def test_missing_multiple_chunks(self): content_name = "test_missing_multiple_chunks" data = random_str(1024 * 1024 * 4) self.object_storage_api.object_create(account=self.account, container=self.container, data=data, policy="THREECOPIES", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(9)) chunks_to_remove.append(chunks.pop(6)) chunks_to_remove.append(chunks.pop(4)) chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0", "1", "2", "3"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_1_chunk_ec(self): if len(self.conf['services']['rawx']) < 9: self.skipTest("Not enough rawx. " "EC tests needs at least 9 rawx to run") content_name = "test_missing_1_chunk_ec" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="EC", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0.1"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_m_chunk_ec(self): if len(self.conf['services']['rawx']) < 9: self.skipTest("Not enough rawx. " "EC tests needs at least 9 rawx to run") content_name = "test_missing_m_chunk_ec" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="EC", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] for i in range(0, 3): chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0.1", "0.2", "0.3"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_m_chunk_ec_2(self): if len(self.conf['services']['rawx']) < 9: self.skipTest("Not enough rawx. " "EC tests needs at least 9 rawx to run") content_name = "test_missing_m_chunk_ec" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="EC", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(0)) chunks_to_remove.append(chunks.pop(3)) chunks_to_remove.append(chunks.pop(5)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0.1", "0.5", "0.8"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove) def test_missing_m1_chunk_ec(self): if len(self.conf['services']['rawx']) < 9: self.skipTest("Not enough rawx. " "EC tests needs at least 9 rawx to run") content_name = "test_missing_m1_chunk_ec" self.object_storage_api.object_create(account=self.account, container=self.container, data="test", policy="EC", obj_name=content_name) meta, chunks = self.object_storage_api.object_locate( container=self.container, obj=content_name, account=self.account) chunks_to_remove = [] chunks_to_remove.append(chunks.pop(0)) chunks_to_remove.append(chunks.pop(0)) chunks_to_remove.append(chunks.pop(0)) chunks_to_remove.append(chunks.pop(0)) for chunk in chunks: chunk.pop('score', None) missing_pos = ["0.1", "0.2", "0.3", "0.4"] self._check_rebuild(content_name, chunks, missing_pos, meta, chunks_to_remove, chunk_created=False)
class TestContainerLifecycle(BaseTestCase): CONTAINERS = set() def setUp(self, recursive=False): super(TestContainerLifecycle, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_lifecycle" self.container = "lifecycle-" + random_str(4) self.lifecycle = ContainerLifecycle( self.api, self.account, self.container, recursive=recursive) @staticmethod def _time_to_date(timestamp=None): if timestamp is None: timestamp = time.time() return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp)) def _upload_something(self, prefix="", path=None, **kwargs): path = path or (prefix + random_str(8)) self.api.object_create(self.account, self.container, obj_name=path, data=path, **kwargs) self.__class__.CONTAINERS.add(self.container) obj_meta = self.api.object_show(self.account, self.container, path) # add fields like LifeCycle do in execute to support CH obj_meta['orig_name'] = obj_meta['name'] obj_meta['container'] = self.container return obj_meta def _enable_versioning(self): if not self.api.container_create( self.account, self.container, system={'sys.m2.policy.version': '-1'}): self.api.container_set_properties( self.account, self.container, system={'sys.policy.version': '-1'}) def test_load_from_container_property(self): source = """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>key1</Key> <Value>value1</Value> </Tag> <Tag> <Key>key2</Key> <Value>value2</Value> </Tag> </And> </Filter> <Status>Enabled</Status> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Expiration> <Days>60</Days> </Expiration> <NoncurrentVersionTransition> <NoncurrentDays>1</NoncurrentDays> <StorageClass>THREECOPIES</StorageClass> </NoncurrentVersionTransition> <NoncurrentVersionExpiration> <NoncurrentDays>60</NoncurrentDays> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """ props = {LIFECYCLE_PROPERTY_KEY: source} self.api.container_create(self.account, self.container, properties=props) self.lifecycle.load() self.assertEqual(1, len(self.lifecycle.rules)) rule = self.lifecycle.rules[0] self.assertIsNotNone(rule) self.assertEqual('rule1', rule.id) self.assertIsNotNone(rule.filter) self.assertEqual('documents/', rule.filter.prefix) self.assertDictEqual({'key1': 'value1', 'key2': 'value2'}, rule.filter.tags) self.assertTrue(rule.enabled) self.assertEqual(4, len(rule.actions)) expiration = rule.actions[0] self.assertEqual(Expiration, type(expiration)) self.assertEqual(60, expiration.filter.days) transition = rule.actions[1] self.assertEqual(Transition, type(transition)) self.assertEqual(1, transition.filter.days) self.assertEqual('THREECOPIES', transition.policy) expiration = rule.actions[2] self.assertEqual(NoncurrentVersionExpiration, type(expiration)) self.assertEqual(60, expiration.filter.days) transition = rule.actions[3] self.assertEqual(NoncurrentVersionTransition, type(transition)) self.assertEqual(1, transition.filter.days) self.assertEqual('THREECOPIES', transition.policy) def test_save_to_container_property(self): source = """ <LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Rule> <ID>rule1</ID> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>key</Key> <Value>value</Value> </Tag> </And> </Filter> <Status>Enabled</Status> <Expiration> <Days>60</Days> </Expiration> <Transition> <StorageClass>SINGLE</StorageClass> <Days>30</Days> </Transition> <Transition> <StorageClass>THREECOPIES</StorageClass> <Days>1</Days> </Transition> <NoncurrentVersionExpiration> <NoncurrentDays>60</NoncurrentDays> </NoncurrentVersionExpiration> <NoncurrentVersionTransition> <StorageClass>THREECOPIES</StorageClass> <NoncurrentDays>1</NoncurrentDays> </NoncurrentVersionTransition> </Rule> </LifecycleConfiguration> """ self.api.container_create(self.account, self.container) self.lifecycle.load_xml(source) self.lifecycle.save() xml = self.lifecycle.get_configuration() self.assertEqual( source.replace(' ', '').replace('\n', ''), xml.replace(' ', '').replace('\n', '')) def test_immediate_expiration_by_date(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() + 86400)) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+172800)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Deleted', status) self.assertRaises(NoSuchObject, self.api.object_show, self.account, obj_meta['container'], obj_meta['name']) def test_immediate_expiration_by_date_after_new_object(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() + 172800)) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) self.api.object_show( self.account, obj_meta['container'], obj_meta['name']) def test_future_expiration_by_date(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Date>%s</Date> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """ % self._time_to_date(time.time() + 86400)) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, obj_meta['container'], obj_meta['name']) self.api.object_delete(self.account, obj_meta['container'], obj_meta['name']) def test_expiration_by_delay(self): obj_meta = self._upload_something() self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, obj_meta['container'], obj_meta['name']) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Deleted', status) self.assertRaises(NoSuchObject, self.api.object_show, self.account, obj_meta['container'], obj_meta['name']) def test_expiration_filtered_by_prefix(self): obj_meta = self._upload_something(prefix="photos/") obj_meta2 = self._upload_something(prefix="documents/") self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Prefix>documents/</Prefix> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta2)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, obj_meta['container'], obj_meta['orig_name']) self.api.object_show(self.account, obj_meta2['container'], obj_meta2['orig_name']) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply( obj_meta2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Deleted', status) self.api.object_show(self.account, obj_meta['container'], obj_meta['orig_name']) self.assertRaises(NoSuchObject, self.api.object_show, self.account, obj_meta2['container'], obj_meta2['orig_name']) self.api.object_delete(self.account, obj_meta['container'], obj_meta['orig_name']) def test_expiration_filtered_by_tag(self): obj_meta = self._upload_something() obj_meta2 = self._upload_something( properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}) obj_meta3 = self._upload_something( properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>status</Key> <Value>approved</Value> </Tag> </TagSet> </Tagging> """}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta2)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta3)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, obj_meta['container'], obj_meta['name']) self.api.object_show(self.account, obj_meta2['container'], obj_meta2['name']) self.api.object_show(self.account, obj_meta3['container'], obj_meta3['name']) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply( obj_meta2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Deleted', status) results = [x for x in self.lifecycle.apply( obj_meta3, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, obj_meta['container'], obj_meta['name']) self.assertRaises(NoSuchObject, self.api.object_show, self.account, obj_meta2['container'], obj_meta2['name']) self.api.object_show(self.account, obj_meta3['container'], obj_meta3['name']) self.api.object_delete(self.account, obj_meta['container'], obj_meta['name']) self.api.object_delete(self.account, obj_meta3['container'], obj_meta3['name']) def test_transition_filtered_by_tag(self): obj_meta = self._upload_something(policy='SINGLE') obj_meta2 = self._upload_something( policy='SINGLE', properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}) obj_meta3 = self._upload_something( policy='SINGLE', properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>status</Key> <Value>approved</Value> </Tag> </TagSet> </Tagging> """}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta2)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta3)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) obj_meta_after, chunks = self.api.object_locate( self.account, obj_meta['container'], obj_meta['name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual(1, len(chunks)) obj_meta2_after, chunks2 = self.api.object_locate( self.account, obj_meta2['container'], obj_meta2['name']) self.assertEqual('SINGLE', obj_meta2_after['policy']) self.assertEqual(1, len(chunks2)) obj_meta3_after, chunks3 = self.api.object_locate( self.account, obj_meta3['container'], obj_meta3['name']) self.assertEqual('SINGLE', obj_meta3_after['policy']) self.assertEqual(1, len(chunks3)) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply( obj_meta2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Policy changed to THREECOPIES', status) results = [x for x in self.lifecycle.apply( obj_meta3, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) obj_meta_after, chunks = self.api.object_locate( self.account, obj_meta['container'], obj_meta['name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual(1, len(chunks)) obj_meta2_after, chunks2 = self.api.object_locate( self.account, obj_meta2['container'], obj_meta2['name']) self.assertEqual('THREECOPIES', obj_meta2_after['policy']) self.assertEqual(3, len(chunks2)) obj_meta3_after, chunks3 = self.api.object_locate( self.account, obj_meta3['container'], obj_meta3['name']) self.assertEqual('SINGLE', obj_meta3_after['policy']) self.assertEqual(1, len(chunks3)) self.api.object_delete(self.account, obj_meta['container'], obj_meta['name']) self.api.object_delete(self.account, obj_meta2['container'], obj_meta2['name']) self.api.object_delete(self.account, obj_meta3['container'], obj_meta3['name']) def test_transition_filtered_by_tags(self): obj_meta = self._upload_something(policy='SINGLE') obj_meta2 = self._upload_something( policy='SINGLE', properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>test1</Key> <Value>test2</Value> </Tag> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}) obj_meta3 = self._upload_something( policy='SINGLE', properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <And> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> <Tag> <Key>test1</Key> <Value>test2</Value> </Tag> </And> </Filter> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta2)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta3)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) obj_meta_after, chunks = self.api.object_locate( self.account, obj_meta['container'], obj_meta['name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual(1, len(chunks)) obj_meta2_after, chunks2 = self.api.object_locate( self.account, obj_meta2['container'], obj_meta2['name']) self.assertEqual('SINGLE', obj_meta2_after['policy']) self.assertEqual(1, len(chunks2)) obj_meta3_after, chunks3 = self.api.object_locate( self.account, obj_meta3['container'], obj_meta3['name']) self.assertEqual('SINGLE', obj_meta3_after['policy']) self.assertEqual(1, len(chunks3)) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply( obj_meta2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Policy changed to THREECOPIES', status) results = [x for x in self.lifecycle.apply( obj_meta3, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) obj_meta_after, chunks = self.api.object_locate( self.account, obj_meta['container'], obj_meta['name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual(1, len(chunks)) obj_meta2_after, chunks2 = self.api.object_locate( self.account, obj_meta2['container'], obj_meta2['name']) self.assertEqual('THREECOPIES', obj_meta2_after['policy']) self.assertEqual(3, len(chunks2)) obj_meta3_after, chunks3 = self.api.object_locate( self.account, obj_meta3['container'], obj_meta3['name']) self.assertEqual('SINGLE', obj_meta3_after['policy']) self.assertEqual(1, len(chunks3)) self.api.object_delete(self.account, obj_meta['container'], obj_meta['name']) self.api.object_delete(self.account, obj_meta2['container'], obj_meta2['name']) self.api.object_delete(self.account, obj_meta3['container'], obj_meta3['name']) def test_transition_filtered_by_prefix_and_tags(self): obj_meta = self._upload_something(policy='SINGLE', prefix="documents/") obj_meta2 = self._upload_something( policy='SINGLE', prefix="documents/", properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>test1</Key> <Value>test2</Value> </Tag> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}) obj_meta3 = self._upload_something( policy='SINGLE', properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>test1</Key> <Value>test2</Value> </Tag> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter> <And> <Prefix>documents/</Prefix> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> <Tag> <Key>test1</Key> <Value>test2</Value> </Tag> </And> </Filter> <Transition> <Days>1</Days> <StorageClass>THREECOPIES</StorageClass> </Transition> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta2)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta3)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) obj_meta_after, chunks = self.api.object_locate( self.account, obj_meta['container'], obj_meta['orig_name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual(1, len(chunks)) obj_meta2_after, chunks2 = self.api.object_locate( self.account, obj_meta2['container'], obj_meta2['orig_name']) self.assertEqual('SINGLE', obj_meta2_after['policy']) self.assertEqual(1, len(chunks2)) obj_meta3_after, chunks3 = self.api.object_locate( self.account, obj_meta3['container'], obj_meta3['orig_name']) self.assertEqual('SINGLE', obj_meta3_after['policy']) self.assertEqual(1, len(chunks3)) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply( obj_meta2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta2_copy, _, _, status = results[0] self.assertEqual(obj_meta2, obj_meta2_copy) self.assertEqual('Policy changed to THREECOPIES', status) results = [x for x in self.lifecycle.apply( obj_meta3, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta3_copy, _, _, status = results[0] self.assertEqual(obj_meta3, obj_meta3_copy) self.assertEqual('Kept', status) obj_meta_after, chunks = self.api.object_locate( self.account, obj_meta['container'], obj_meta['orig_name']) self.assertEqual('SINGLE', obj_meta_after['policy']) self.assertEqual(1, len(chunks)) obj_meta2_after, chunks2 = self.api.object_locate( self.account, obj_meta2['container'], obj_meta2['orig_name']) self.assertEqual('THREECOPIES', obj_meta2_after['policy']) self.assertEqual(3, len(chunks2)) obj_meta3_after, chunks3 = self.api.object_locate( self.account, obj_meta3['container'], obj_meta3['orig_name']) self.assertEqual('SINGLE', obj_meta3_after['policy']) self.assertEqual(1, len(chunks3)) self.api.object_delete(self.account, obj_meta['container'], obj_meta['orig_name']) self.api.object_delete(self.account, obj_meta2['container'], obj_meta2['orig_name']) self.api.object_delete(self.account, obj_meta3['container'], obj_meta3['orig_name']) def test_expiration_with_versioning(self): self._enable_versioning() obj_meta = self._upload_something() obj_meta_v2 = self._upload_something(path=obj_meta['name']) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter></Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta_v2)] self.assertEqual(1, len(results)) obj_meta_v2_copy, _, _, status = results[0] self.assertEqual(obj_meta_v2, obj_meta_v2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, self.container, obj_meta_v2['name'], version=obj_meta_v2['version']) self.api.object_show(self.account, obj_meta['container'], obj_meta['name'], version=obj_meta['version']) results = [x for x in self.lifecycle.apply( obj_meta_v2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_v2_copy, _, _, status = results[0] self.assertEqual(obj_meta_v2, obj_meta_v2_copy) self.assertEqual('Deleted', status) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) self.assertRaises(NoSuchObject, self.api.object_locate, self.account, self.container, obj_meta_v2['name']) self.api.object_show(self.account, self.container, obj_meta_v2['name'], version=obj_meta_v2['version']) self.api.object_show(self.account, obj_meta['container'], obj_meta['name'], version=obj_meta['version']) self.api.object_delete( self.account, obj_meta['container'], obj_meta['name'], version=obj_meta['version']) def test_noncurrent_expiration(self): self._enable_versioning() obj_meta = self._upload_something() obj_meta_v2 = self._upload_something(path=obj_meta['name']) self.lifecycle.load_xml(""" <LifecycleConfiguration> <Rule> <Filter></Filter> <NoncurrentVersionExpiration> <NoncurrentDays>1</NoncurrentDays> </NoncurrentVersionExpiration> <Status>enabled</Status> </Rule> </LifecycleConfiguration> """) results = [x for x in self.lifecycle.apply(obj_meta_v2)] self.assertEqual(1, len(results)) obj_meta_v2_copy, _, _, status = results[0] self.assertEqual(obj_meta_v2, obj_meta_v2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply(obj_meta)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Kept', status) self.api.object_show(self.account, self.container, obj_meta_v2['name'], version=obj_meta_v2['version']) self.api.object_show(self.account, obj_meta['container'], obj_meta['name'], version=obj_meta['version']) results = [x for x in self.lifecycle.apply( obj_meta_v2, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_v2_copy, _, _, status = results[0] self.assertEqual(obj_meta_v2, obj_meta_v2_copy) self.assertEqual('Kept', status) results = [x for x in self.lifecycle.apply( obj_meta, now=time.time()+86400)] self.assertEqual(1, len(results)) obj_meta_copy, _, _, status = results[0] self.assertEqual(obj_meta, obj_meta_copy) self.assertEqual('Deleted', status) self.api.object_show(self.account, self.container, obj_meta_v2['name'], version=obj_meta_v2['version']) self.assertRaises(NoSuchObject, self.api.object_show, self.account, obj_meta['container'], obj_meta['name'], version=obj_meta['version']) self.api.object_delete( self.account, self.container, obj_meta_v2['name'], version=obj_meta_v2['version']) def test_execute_expiration(self): self.api.container_create( self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <Expiration> <Days>1</Days> </Expiration> </Rule> </LifecycleConfiguration> """}) for _ in range(3): self._upload_something() self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(3, len(results)) for res in results: self.assertEqual('Kept', res[3]) listing = self.api.object_list(self.account, self.container) self.assertEqual(3, len(listing['objects'])) results = [x for x in self.lifecycle.execute(now=time.time()+86400)] self.assertEqual(3, len(results)) for res in results: self.assertEqual('Deleted', res[3]) listing = self.api.object_list(self.account, self.container) self.assertEqual(0, len(listing['objects'])) def test_execute_expiration_with_disabled_status(self): self.api.container_create( self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Disabled</Status> <Expiration> <Days>1</Days> </Expiration> </Rule> </LifecycleConfiguration> """}) for _ in range(3): self._upload_something() self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(3, len(results)) for res in results: self.assertEqual('Kept', res[3]) listing = self.api.object_list(self.account, self.container) self.assertEqual(3, len(listing['objects'])) results = [x for x in self.lifecycle.execute(now=time.time()+86400)] self.assertEqual(3, len(results)) for res in results: self.assertEqual('Kept', res[3]) listing = self.api.object_list(self.account, self.container) self.assertEqual(3, len(listing['objects'])) def test_execute_expiration_on_missing_objects(self): self.api.container_create( self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <Expiration> <Days>1</Days> </Expiration> </Rule> </LifecycleConfiguration> """}) fake_listing = { 'objects': [ {'name': 'a', 'version': 1540933092888883, 'mtime': '12', 'deleted': False}, {'name': 'b', 'version': 1540933092888883, 'mtime': '12', 'deleted': False}, {'name': 'c', 'version': 1540933092888883, 'mtime': '12', 'deleted': False}, {'name': 'd', 'version': 1540933092888883, 'mtime': '12', 'deleted': False}], 'truncated': False } with patch.object(self.api, 'object_list', side_effect=[fake_listing]): self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(4, len(results)) for res in results: self.assertIsInstance(res[3], Exception) def test_execute_exceeding_version_expiration_without_versioning(self): self.api.container_create( self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <NoncurrentVersionExpiration> <NoncurrentCount>2</NoncurrentCount> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """}) for _ in range(5): self._upload_something() self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(5, len(results)) for res in results: self.assertEqual('Kept', res[3]) listing = self.api.object_list(self.account, self.container, versions=True) self.assertEqual(5, len(listing['objects'])) def test_execute_exceeding_version_expiration_with_versioning(self): self.api.container_create( self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """ <LifecycleConfiguration> <Rule> <ID>rule1</ID> <Filter> </Filter> <Status>Enabled</Status> <NoncurrentVersionExpiration> <NoncurrentCount>2</NoncurrentCount> </NoncurrentVersionExpiration> </Rule> </LifecycleConfiguration> """}, system={"sys.m2.policy.version": "4"}) for _ in range(5): self._upload_something(path="versioned1") for _ in range(5): self._upload_something(path="versioned2") listing = self.api.object_list(self.account, self.container, versions=True) self.assertEqual(10, len(listing['objects'])) self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(10, len(results)) listing = self.api.object_list(self.account, self.container, versions=True) self.assertEqual(6, len(listing['objects'])) def test_execute_multiple_rules(self): meta = {"sys.m2.policy.version": "4"} self.api.container_create( self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """ <LifecycleConfiguration> <Rule> <Filter> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>enabled</Status> </Rule> <Rule> <Filter> <Prefix>documents/</Prefix> </Filter> <Expiration> <Days>1</Days> </Expiration> <Status>Enabled</Status> </Rule> </LifecycleConfiguration> """}, system=meta) obj_meta = self._upload_something(policy='SINGLE', system=meta) obj_meta2 = self._upload_something( policy='SINGLE', properties={TAGGING_KEY: """ <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <TagSet> <Tag> <Key>status</Key> <Value>deprecated</Value> </Tag> </TagSet> </Tagging> """}, system=meta) obj_meta3 = self._upload_something(path="documents/object3", policy='SINGLE', system=meta) self.lifecycle.load() results = [x for x in self.lifecycle.execute()] self.assertEqual(6, len(results)) for res in results: self.assertEqual('Kept', res[3]) self.api.object_show(self.account, obj_meta['container'], obj_meta['orig_name']) self.api.object_show(self.account, obj_meta2['container'], obj_meta2['orig_name']) self.api.object_show(self.account, obj_meta3['container'], obj_meta3['orig_name']) results = [x for x in self.lifecycle.execute(now=time.time()+86400)] self.assertEqual(5, len(results)) self.api.object_locate( self.account, obj_meta['container'], obj_meta['orig_name']) self.api.object_show( self.account, obj_meta['container'], obj_meta['orig_name'], version=obj_meta['version']) self.assertRaises(NoSuchObject, self.api.object_locate, self.account, obj_meta2['container'], obj_meta2['orig_name']) self.api.object_show( self.account, obj_meta2['container'], obj_meta2['orig_name'], version=obj_meta2['version']) self.assertRaises(NoSuchObject, self.api.object_locate, self.account, obj_meta3['container'], obj_meta3['orig_name']) self.api.object_show( self.account, obj_meta3['container'], obj_meta3['orig_name'], version=obj_meta3['version']) self.api.object_delete( self.account, obj_meta['container'], obj_meta['orig_name'], version=obj_meta['version']) self.api.object_delete( self.account, obj_meta2['container'], obj_meta2['orig_name'], version=obj_meta2['version']) self.api.object_delete( self.account, obj_meta3['container'], obj_meta3['orig_name'], version=obj_meta3['version'])