class TestContainerLifecycle(BaseTestCase):
CONTAINERS = set()
def setUp(self, recursive=False):
super(TestContainerLifecycle, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_lifecycle"
self.container = "lifecycle-" + random_str(4)
self.lifecycle = ContainerLifecycle(
self.api, self.account, self.container, recursive=recursive)
@staticmethod
def _time_to_date(timestamp=None):
if timestamp is None:
timestamp = time.time()
return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp))
def _upload_something(self, prefix="", path=None, **kwargs):
path = path or (prefix + random_str(8))
self.api.object_create(self.account, self.container,
obj_name=path, data=path, **kwargs)
self.__class__.CONTAINERS.add(self.container)
obj_meta = self.api.object_show(self.account, self.container, path)
# add fields like LifeCycle do in execute to support CH
obj_meta['orig_name'] = obj_meta['name']
obj_meta['container'] = self.container
return obj_meta
def _enable_versioning(self):
if not self.api.container_create(
self.account, self.container,
system={'sys.m2.policy.version': '-1'}):
self.api.container_set_properties(
self.account, self.container,
system={'sys.policy.version': '-1'})
def test_load_from_container_property(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Expiration>
<Days>60</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>1</NoncurrentDays>
<StorageClass>THREECOPIES</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self.lifecycle.load()
self.assertEqual(1, len(self.lifecycle.rules))
rule = self.lifecycle.rules[0]
self.assertIsNotNone(rule)
self.assertEqual('rule1', rule.id)
self.assertIsNotNone(rule.filter)
self.assertEqual('documents/', rule.filter.prefix)
self.assertDictEqual({'key1': 'value1', 'key2': 'value2'},
rule.filter.tags)
self.assertTrue(rule.enabled)
self.assertEqual(4, len(rule.actions))
expiration = rule.actions[0]
self.assertEqual(Expiration, type(expiration))
self.assertEqual(60, expiration.filter.days)
transition = rule.actions[1]
self.assertEqual(Transition, type(transition))
self.assertEqual(1, transition.filter.days)
self.assertEqual('THREECOPIES', transition.policy)
expiration = rule.actions[2]
self.assertEqual(NoncurrentVersionExpiration, type(expiration))
self.assertEqual(60, expiration.filter.days)
transition = rule.actions[3]
self.assertEqual(NoncurrentVersionTransition, type(transition))
self.assertEqual(1, transition.filter.days)
self.assertEqual('THREECOPIES', transition.policy)
def test_save_to_container_property(self):
source = """
<LifecycleConfiguration
xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key</Key>
<Value>value</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>60</Days>
</Expiration>
<Transition>
<StorageClass>SINGLE</StorageClass>
<Days>30</Days>
</Transition>
<Transition>
<StorageClass>THREECOPIES</StorageClass>
<Days>1</Days>
</Transition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
<NoncurrentVersionTransition>
<StorageClass>THREECOPIES</StorageClass>
<NoncurrentDays>1</NoncurrentDays>
</NoncurrentVersionTransition>
</Rule>
</LifecycleConfiguration>
"""
self.api.container_create(self.account, self.container)
self.lifecycle.load_xml(source)
self.lifecycle.save()
xml = self.lifecycle.get_configuration()
self.assertEqual(
source.replace(' ', '').replace('\n', ''),
xml.replace(' ', '').replace('\n', ''))
def test_immediate_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 86400))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+172800)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Deleted', status)
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta['container'],
obj_meta['name'])
def test_immediate_expiration_by_date_after_new_object(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 172800))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(
self.account, obj_meta['container'], obj_meta['name'])
def test_future_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 86400))
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
def test_expiration_by_delay(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Deleted', status)
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta['container'],
obj_meta['name'])
def test_expiration_filtered_by_prefix(self):
obj_meta = self._upload_something(prefix="photos/")
obj_meta2 = self._upload_something(prefix="documents/")
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Prefix>documents/</Prefix>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.api.object_show(self.account, obj_meta2['container'],
obj_meta2['orig_name'])
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Deleted', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['orig_name'])
def test_expiration_filtered_by_tag(self):
obj_meta = self._upload_something()
obj_meta2 = self._upload_something(
properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>approved</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_show(self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_show(self.account, obj_meta3['container'],
obj_meta3['name'])
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Deleted', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_show(self.account, obj_meta3['container'],
obj_meta3['name'])
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['name'])
def test_transition_filtered_by_tag(self):
obj_meta = self._upload_something(policy='SINGLE')
obj_meta2 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>approved</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('SINGLE', obj_meta2_after['policy'])
self.assertEqual(1, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Policy changed to THREECOPIES', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('THREECOPIES', obj_meta2_after['policy'])
self.assertEqual(3, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['name'])
def test_transition_filtered_by_tags(self):
obj_meta = self._upload_something(policy='SINGLE')
obj_meta2 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<And>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
</And>
</Filter>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('SINGLE', obj_meta2_after['policy'])
self.assertEqual(1, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Policy changed to THREECOPIES', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('THREECOPIES', obj_meta2_after['policy'])
self.assertEqual(3, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['name'])
def test_transition_filtered_by_prefix_and_tags(self):
obj_meta = self._upload_something(policy='SINGLE', prefix="documents/")
obj_meta2 = self._upload_something(
policy='SINGLE', prefix="documents/", properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
</And>
</Filter>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['orig_name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['orig_name'])
self.assertEqual('SINGLE', obj_meta2_after['policy'])
self.assertEqual(1, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['orig_name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Policy changed to THREECOPIES', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['orig_name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['orig_name'])
self.assertEqual('THREECOPIES', obj_meta2_after['policy'])
self.assertEqual(3, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['orig_name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.api.object_delete(self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['orig_name'])
def test_expiration_with_versioning(self):
self._enable_versioning()
obj_meta = self._upload_something()
obj_meta_v2 = self._upload_something(path=obj_meta['name'])
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter></Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta_v2)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
results = [x for x in self.lifecycle.apply(
obj_meta_v2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Deleted', status)
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.assertRaises(NoSuchObject, self.api.object_locate,
self.account, self.container, obj_meta_v2['name'])
self.api.object_show(self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
self.api.object_delete(
self.account, obj_meta['container'], obj_meta['name'],
version=obj_meta['version'])
def test_noncurrent_expiration(self):
self._enable_versioning()
obj_meta = self._upload_something()
obj_meta_v2 = self._upload_something(path=obj_meta['name'])
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter></Filter>
<NoncurrentVersionExpiration>
<NoncurrentDays>1</NoncurrentDays>
</NoncurrentVersionExpiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta_v2)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
results = [x for x in self.lifecycle.apply(
obj_meta_v2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Deleted', status)
self.api.object_show(self.account, self.container,
obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
self.api.object_delete(
self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
def test_execute_expiration(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""})
for _ in range(3):
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(3, len(listing['objects']))
results = [x for x in self.lifecycle.execute(now=time.time()+86400)]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Deleted', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(0, len(listing['objects']))
def test_execute_expiration_with_disabled_status(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Disabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""})
for _ in range(3):
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(3, len(listing['objects']))
results = [x for x in self.lifecycle.execute(now=time.time()+86400)]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(3, len(listing['objects']))
def test_execute_expiration_on_missing_objects(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""})
fake_listing = {
'objects': [
{'name': 'a', 'version': 1540933092888883, 'mtime': '12',
'deleted': False},
{'name': 'b', 'version': 1540933092888883, 'mtime': '12',
'deleted': False},
{'name': 'c', 'version': 1540933092888883, 'mtime': '12',
'deleted': False},
{'name': 'd', 'version': 1540933092888883, 'mtime': '12',
'deleted': False}],
'truncated': False
}
with patch.object(self.api, 'object_list',
side_effect=[fake_listing]):
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(4, len(results))
for res in results:
self.assertIsInstance(res[3], Exception)
def test_execute_exceeding_version_expiration_without_versioning(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<NoncurrentVersionExpiration>
<NoncurrentCount>2</NoncurrentCount>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""})
for _ in range(5):
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(5, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container,
versions=True)
self.assertEqual(5, len(listing['objects']))
def test_execute_exceeding_version_expiration_with_versioning(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<NoncurrentVersionExpiration>
<NoncurrentCount>2</NoncurrentCount>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""}, system={"sys.m2.policy.version": "4"})
for _ in range(5):
self._upload_something(path="versioned1")
for _ in range(5):
self._upload_something(path="versioned2")
listing = self.api.object_list(self.account, self.container,
versions=True)
self.assertEqual(10, len(listing['objects']))
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(10, len(results))
listing = self.api.object_list(self.account, self.container,
versions=True)
self.assertEqual(6, len(listing['objects']))
def test_execute_multiple_rules(self):
meta = {"sys.m2.policy.version": "4"}
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
<Rule>
<Filter>
<Prefix>documents/</Prefix>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>
</LifecycleConfiguration>
"""}, system=meta)
obj_meta = self._upload_something(policy='SINGLE', system=meta)
obj_meta2 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""}, system=meta)
obj_meta3 = self._upload_something(path="documents/object3",
policy='SINGLE',
system=meta)
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(6, len(results))
for res in results:
self.assertEqual('Kept', res[3])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.api.object_show(self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_show(self.account, obj_meta3['container'],
obj_meta3['orig_name'])
results = [x for x in self.lifecycle.execute(now=time.time()+86400)]
self.assertEqual(5, len(results))
self.api.object_locate(
self.account, obj_meta['container'], obj_meta['orig_name'])
self.api.object_show(
self.account, obj_meta['container'], obj_meta['orig_name'],
version=obj_meta['version'])
self.assertRaises(NoSuchObject, self.api.object_locate,
self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_show(
self.account, obj_meta2['container'], obj_meta2['orig_name'],
version=obj_meta2['version'])
self.assertRaises(NoSuchObject, self.api.object_locate,
self.account, obj_meta3['container'],
obj_meta3['orig_name'])
self.api.object_show(
self.account, obj_meta3['container'], obj_meta3['orig_name'],
version=obj_meta3['version'])
self.api.object_delete(
self.account, obj_meta['container'], obj_meta['orig_name'],
version=obj_meta['version'])
self.api.object_delete(
self.account, obj_meta2['container'], obj_meta2['orig_name'],
version=obj_meta2['version'])
self.api.object_delete(
self.account, obj_meta3['container'], obj_meta3['orig_name'],
version=obj_meta3['version'])
class TestContainerLifecycle(BaseTestCase):
CONTAINERS = set()
def setUp(self):
super(TestContainerLifecycle, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_lifecycle"
self.container = "lifecycle-" + random_str(4)
self.lifecycle = ContainerLifecycle(
self.api, self.account, self.container)
@staticmethod
def _time_to_date(timestamp=None):
if timestamp is None:
timestamp = time.time()
return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp))
def _upload_something(self, prefix="", path=None, **kwargs):
path = path or (prefix + random_str(8))
self.api.object_create(self.account, self.container,
obj_name=path, data=path, **kwargs)
self.__class__.CONTAINERS.add(self.container)
return self.api.object_show(self.account, self.container, path)
def _enable_versioning(self):
if not self.api.container_create(
self.account, self.container,
system={'sys.m2.policy.version': '-1'}):
self.api.container_set_properties(
self.account, self.container,
system={'sys.policy.version': '-1'})
def test_load_from_container_property(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Expiration>
<Days>60</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>1</NoncurrentDays>
<StorageClass>THREECOPIES</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self.lifecycle.load()
self.assertIn('rule1', self.lifecycle._rules)
rule = self.lifecycle._rules['rule1']
self.assertEqual('rule1', rule.id)
self.assertTrue(rule.enabled)
self.assertIsNotNone(rule.filter)
self.assertIn('Expiration', rule.actions)
self.assertIn('Transition', rule.actions)
self.assertIn('NoncurrentVersionExpiration', rule.actions)
self.assertIn('NoncurrentVersionTransition', rule.actions)
def test_save_to_container_property(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Expiration>
<Days>60</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>1</NoncurrentDays>
<StorageClass>THREECOPIES</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""
self.api.container_create(self.account, self.container)
self.lifecycle.load_xml(source)
self.lifecycle.save()
props = self.api.container_get_properties(
self.account, self.container)['properties']
self.assertIn(LIFECYCLE_PROPERTY_KEY, props)
self.assertEqual(source, props[LIFECYCLE_PROPERTY_KEY])
def test_immediate_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() - 86400))
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertNotIn(obj_meta['name'], obj_names)
def test_future_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 86400))
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_expiration_by_delay(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertNotIn(obj_meta['name'], obj_names)
def test_future_expiration_by_delay(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
self.lifecycle.apply(obj_meta)
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_expiration_filtered_by_prefix(self):
obj_meta = self._upload_something(prefix="photos/")
obj_meta2 = self._upload_something(prefix="documents/")
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Prefix>documents/</Prefix>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta2))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.assertNotIn(obj_meta2['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_expiration_filtered_by_tag(self):
obj_meta = self._upload_something()
obj_meta2 = self._upload_something(properties={'status': 'deprecated'})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta2))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.assertNotIn(obj_meta2['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_transition_filtered_by_tag(self):
obj_meta = self._upload_something(policy='SINGLE')
obj_meta2 = self._upload_something(policy='SINGLE',
properties={'status': 'deprecated'})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Transition>
<Days>0</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta2))
obj_meta_after = self.api.object_show(
self.account, self.container, obj_meta['name'])
obj_meta_after2 = self.api.object_show(
self.account, self.container, obj_meta2['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual('THREECOPIES', obj_meta_after2['policy'])
self.api.object_delete(self.account, self.container, obj_meta['name'])
self.api.object_delete(self.account, self.container, obj_meta2['name'])
def test_immediate_noncurrent_expiration(self):
self._enable_versioning()
obj_meta = self._upload_something()
obj_meta_v2 = self._upload_something(path=obj_meta['name'])
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter></Filter>
<NoncurrentVersionExpiration>
<NoncurrentDays>0</NoncurrentDays>
</NoncurrentVersionExpiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta_v2))
container_descr = self.api.object_list(self.account, self.container,
versions=True)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.assertEqual(1, len(obj_names))
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_execute_immediate_expiration(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>0</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self._upload_something()
self._upload_something()
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(3, len(results))
listing = self.api.object_list(self.account, self.container)
self.assertEqual(0, len(listing['objects']))
def test_execute_expiration_on_missing_objects(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>0</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self.lifecycle.load()
fake_listing = {
'objects': [
{'name': 'a', 'ctime': '12'},
{'name': 'b', 'ctime': '12'},
{'name': 'c', 'ctime': '12'},
{'name': 'd', 'ctime': '12'}],
'truncated': False
}
with patch.object(self.api, 'object_list',
side_effect=[fake_listing]):
results = [x for x in self.lifecycle.execute()]
self.assertEqual(4, len(results))
for res in results:
self.assertIsInstance(res[3], Exception)
class TestMeta1RefMapping(BaseTestCase):
def setUp(self):
super(TestMeta1RefMapping, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_refmapping"
self.reference = "refmapping-" + random_str(4)
self.logger = logging.getLogger('test')
self.mapping = Meta1RefMapping(self.ns, logger=self.logger)
def _get_cid(self):
data_dir = self.api.directory.get_properties(self.account,
self.reference)
return data_dir['cid']
def _get_services(self, service_type):
data_dir = self.api.directory.list(self.account, self.reference)
services = list()
for d in data_dir['srv']:
if d['type'] == service_type:
services.append(d['host'])
return services
def _test_move(self, service_type, base=None, destination=True):
if base is None:
base = self._get_cid()
raw_services = self._get_services(service_type)
expected_services = list(raw_services)
src_service = expected_services.pop()
dest_service = None
all_meta2_services = self.conscience.all_services(service_type, True)
if len(all_meta2_services) <= len(raw_services):
self.skipTest("need at least %d %s" %
(len(raw_services) + 1, service_type))
if destination:
for service in all_meta2_services:
if service['addr'] not in raw_services:
dest_service = service['addr']
expected_services.append(dest_service)
moved = self.mapping.move(src_service, dest_service, base,
service_type)
moved_ok = self.mapping.apply(moved)
self.assertEqual(1, len(moved_ok))
data_dir = self.api.directory.list(self.account, self.reference)
new_services = list()
for d in data_dir['srv']:
if d['type'] == service_type:
new_services.append(d['host'])
if destination:
self.assertListEqual(sorted(expected_services),
sorted(new_services))
else:
for expected_service in expected_services:
self.assertIn(expected_service, new_services)
self.assertNotIn(src_service, new_services)
self.assertEqual(len(expected_services) + 1, len(new_services))
return (src_service, expected_services)
def test_move_meta2(self):
self.api.container_create(self.account, self.reference)
_, expected_services = self._test_move('meta2')
properties = self.api.container_get_properties(self.account,
self.reference)
peers = properties['system']['sys.peers']
new_services = peers.split(',')
self.assertListEqual(sorted(expected_services), sorted(new_services))
self.api.object_create(self.account,
self.reference,
data="move meta2",
obj_name="test")
for i in range(0, 10):
self.api.object_show(self.account, self.reference, "test")
self.api.object_delete(self.account, self.reference, "test")
sleep(0.5)
self.api.container_delete(self.account, self.reference)
def test_move_meta2_with_seq(self):
self.api.container_create(self.account, self.reference)
properties = self.api.container_get_properties(self.account,
self.reference)
base = properties['system']['sys.name']
_, expected_services = self._test_move('meta2', base=base)
properties = self.api.container_get_properties(self.account,
self.reference)
peers = properties['system']['sys.peers']
new_services = peers.split(',')
self.assertListEqual(sorted(expected_services), sorted(new_services))
self.api.object_create(self.account,
self.reference,
data="move meta2",
obj_name="test")
for i in range(0, 10):
self.api.object_show(self.account, self.reference, "test")
self.api.object_delete(self.account, self.reference, "test")
sleep(0.5)
self.api.container_delete(self.account, self.reference)
def test_move_meta2_without_destination(self):
self.api.container_create(self.account, self.reference)
properties = self.api.container_get_properties(self.account,
self.reference)
base = properties['system']['sys.name']
src_service, expected_services = self._test_move('meta2',
base=base,
destination=False)
properties = self.api.container_get_properties(self.account,
self.reference)
peers = properties['system']['sys.peers']
new_services = peers.split(',')
for expected_service in expected_services:
self.assertIn(expected_service, new_services)
self.assertNotIn(src_service, new_services)
self.assertEqual(len(expected_services) + 1, len(new_services))
self.api.object_create(self.account,
self.reference,
data="move meta2",
obj_name="test")
for i in range(0, 10):
self.api.object_show(self.account, self.reference, "test")
self.api.object_delete(self.account, self.reference, "test")
sleep(0.5)
self.api.container_delete(self.account, self.reference)
def test_move_sqlx(self):
execute('oio-sqlx -O AutoCreate %s/%s/%s '
'"create table foo (a INT, b TEXT)"' %
(self.ns, self.account, self.reference))
self._test_move('sqlx')
execute('oio-sqlx %s/%s/%s "destroy"' %
(self.ns, self.account, self.reference))
def test_move_with_dest_service_already_used(self):
self.api.container_create(self.account, self.reference)
base = self._get_cid()
raw_services = self._get_services('meta2')
src_service = raw_services[0]
dest_service = raw_services[0]
if len(raw_services) > 1:
dest_service = raw_services[1]
self.assertRaises(ValueError, self.mapping.move, src_service,
dest_service, base, 'meta2')
self.api.container_delete(self.account, self.reference)
def test_move_with_src_service_not_used(self):
self.api.container_create(self.account, self.reference)
base = self._get_cid()
raw_services = self._get_services('meta2')
src_service = '127.0.0.1:666'
dest_service = None
all_meta2_services = self.conscience.all_services('meta2', True)
for service in all_meta2_services:
if service['addr'] not in raw_services:
src_service = service['addr']
dest_service = service['addr']
if dest_service is None:
self.skipTest("need at least 2 meta2")
self.assertRaises(ValueError, self.mapping.move, src_service,
dest_service, base, 'meta2')
self.api.container_delete(self.account, self.reference)
def test_move_with_wrong_dest(self):
self.api.container_create(self.account, self.reference)
base = self._get_cid()
raw_services = self._get_services('meta2')
src_service = raw_services[0]
dest_service = '127.0.0.1:666'
self.assertRaises(ValueError, self.mapping.move, src_service,
dest_service, base, 'meta2')
self.api.container_delete(self.account, self.reference)