def setUp(self):
super(TestAdmin, self).setUp()
self.admin = AdminClient(self.conf)
self.api = ObjectStorageApi(self.ns)
self.account = "test_admin"
self.container = "admin-" + random_str(4)
self.api.container_create(self.account, self.container)
class MetaRebuilder(Rebuilder):
"""
Abstract class for directory rebuilders.
"""
def __init__(self, conf, logger, volume, **kwargs):
super(MetaRebuilder, self).__init__(conf, logger, volume, **kwargs)
self.api = ObjectStorageApi(self.namespace,
logger=self.logger,
**kwargs)
def _fill_queue_from_file(self, queue, **kwargs):
if self.input_file is None:
return False
with open(self.input_file, 'r') as ifile:
for line in ifile:
stripped = line.strip()
if stripped and not stripped.startswith('#'):
queue.put(stripped)
if not self.running:
break
return True
def _full_container_list(self, account, **kwargs):
listing = self.api.container_list(account, **kwargs)
for element in listing:
yield element
while listing:
kwargs['marker'] = listing[-1][0]
listing = self.api.container_list(account, **kwargs)
if listing:
for element in listing:
yield element
def setUp(self):
super(TestMeta1RefMapping, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_refmapping"
self.reference = "refmapping-" + random_str(4)
self.logger = logging.getLogger('test')
self.mapping = Meta1RefMapping(self.ns, logger=self.logger)
def setUp(self):
super(TestContainerLifecycle, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_lifecycle"
self.container = "lifecycle-" + random_str(4)
self.lifecycle = ContainerLifecycle(
self.api, self.account, self.container)
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
api = ObjectStorageApi(self.namespace)
actual = [x[0] for x in api.container_list(self.test_account)]
if len(actual) < 3:
print "Slow event propagation!"
# account events have not yet propagated
time.sleep(3.0)
actual = [x[0] for x in api.container_list(self.test_account)[0]]
gen = worker._list_containers()
self.assertListEqual(list(gen), actual)
def setUp(self):
super(TestStorageTierer, self).setUp()
self.namespace = self.conf['namespace']
self.test_account = "test_storage_tiering_%f" % time.time()
self.api = ObjectStorageApi(self.namespace)
self.gridconf = {
"namespace": self.namespace,
"container_fetch_limit": 2,
"content_fetch_limit": 2,
"account": self.test_account,
"outdated_threshold": 0,
"new_policy": "EC"
}
self.api.container = ContainerClient(self.gridconf)
self._populate()
def __init__(self,
conf,
logger=None,
account_ring=None,
container_ring=None,
storage=None):
for policy, ring_arg in zip(OIO_POLICIES, RING_ARGS):
if ring_arg is not None:
policy.object_ring = FakeRing(**ring_arg)
SwiftApplication.__init__(self,
conf,
logger=logger,
account_ring=account_ring,
container_ring=container_ring)
if conf is None:
conf = dict()
sds_conf = {k[4:]: v for k, v in conf.items() if k.startswith("sds_")}
self.oio_stgpol = []
if 'auto_storage_policies' in conf:
for elem in conf['auto_storage_policies'].split(','):
if ':' in elem:
name, offset = elem.split(':')
self.oio_stgpol.append((name, int(offset)))
else:
self.oio_stgpol.append((elem, 0))
self.oio_stgpol.sort(key=lambda x: x[1])
policies = []
if 'oio_storage_policies' in conf:
for i, pol in enumerate(conf['oio_storage_policies'].split(',')):
policies.append(
storage_policy.StoragePolicy(i, pol, is_default=i == 0))
else:
policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True))
self.POLICIES = storage_policy.StoragePolicyCollection(policies)
# Mandatory, raises KeyError
sds_namespace = sds_conf['namespace']
sds_conf.pop('namespace') # removed to avoid unpacking conflict
# Loaded by ObjectStorageApi if None
sds_proxy_url = sds_conf.pop('proxy_url', None)
# Fix boolean parameter
if 'autocreate' in sds_conf and not (
hasattr(ObjectStorageApi, 'EXTRA_KEYWORDS')
or 'autocreate' in ObjectStorageApi.EXTRA_KEYWORDS):
logger.warn("'autocreate' parameter is ignored by current version"
" of OpenIO SDS. Please update to oio>=4.1.23.")
else:
sds_conf['autocreate'] = config_true_value(
sds_conf.get('autocreate', 'true'))
self.storage = storage or \
ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
self.delete_slo_parts = \
config_true_value(conf.get('delete_slo_parts', False))
self.check_state = \
config_true_value(conf.get('check_state', False))
class AccountRebuilderWorker(ToolWorker):
def __init__(self, tool, queue_workers, queue_reply):
super(AccountRebuilderWorker, self).__init__(tool, queue_workers,
queue_reply)
self.api = ObjectStorageApi(self.tool.namespace, logger=self.logger)
def _process_item(self, item):
namespace, account, container = item
if namespace != self.tool.namespace:
raise ValueError('Invalid namespace (actual=%s, expected=%s)' %
(namespace, self.tool.namespace))
if container is None:
self.api.account.account_refresh(account)
else:
self.api.container_refresh(account, container)
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.namespace = self.conf['namespace']
self.gridconf = {"namespace": self.namespace}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = ContainerClient(self.conf)
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = ObjectStorageApi(namespace=self.namespace)
self.stgpol = "SINGLE"
self.conf['tube'] = 'rebuild'
self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300'
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.tube = self.conf.get('tube', 'rebuild')
self.beanstalk = Beanstalk.from_url(queue_url)
self.beanstalk.use(self.tube)
def __init__(self, conf, logger):
self.conf = conf
self.logger = logger
self.account = conf[CONF_ACCOUNT]
self.api = ObjectStorageApi(conf['namespace'], logger=self.logger)
self.container_client = self.api.container
self.account_client = self.api.account
self.passes = 0
self.errors = 0
self.last_reported = 0
self.contents_run_time = 0
self.total_contents_processed = 0
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_contents_per_second = int_value(
conf.get('contents_per_second'), 30)
self.container_fetch_limit = int_value(
conf.get('container_fetch_limit'), 100)
self.content_fetch_limit = int_value(conf.get('content_fetch_limit'),
100)
self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD),
9999999999)
self.new_policy = conf.get(CONF_NEW_POLICY)
def __init__(self,
conf,
memcache=None,
logger=None,
account_ring=None,
container_ring=None,
storage=None):
for policy, ring_arg in zip(POLICIES, ring_args):
if ring_arg is not None:
policy.object_ring = FakeRing(**ring_arg)
SwiftApplication.__init__(self,
conf,
memcache=memcache,
logger=logger,
account_ring=account_ring,
container_ring=container_ring)
if conf is None:
conf = dict()
sds_conf = {
k[4:]: v
for k, v in conf.iteritems() if k.startswith("sds_")
}
self.oio_stgpol = []
if 'auto_storage_policies' in conf:
for elem in conf['auto_storage_policies'].split(','):
if ':' in elem:
name, offset = elem.split(':')
self.oio_stgpol.append((name, int(offset)))
else:
self.oio_stgpol.append((elem, 0))
self.oio_stgpol.sort(key=lambda x: x[1])
policies = []
if 'oio_storage_policies' in conf:
for i, pol in enumerate(conf['oio_storage_policies'].split(',')):
policies.append(
storage_policy.StoragePolicy(i, pol, is_default=i == 0))
else:
policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True))
self.POLICIES = storage_policy.StoragePolicyCollection(policies)
# Mandatory, raises KeyError
sds_namespace = sds_conf['namespace']
sds_conf.pop('namespace') # removed to avoid unpacking conflict
# Loaded by ObjectStorageApi if None
sds_proxy_url = sds_conf.pop('proxy_url', None)
self.storage = storage or \
ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
def setUpClass(cls):
super(ItemRebuildTest, cls).setUpClass()
cls.api = ObjectStorageApi(cls._cls_ns, endpoint=cls._cls_uri)
def setUpClass(cls):
super(ObjectStoragePropertiesTest, cls).setUpClass()
cls.logger = logging.getLogger('test')
cls.api = ObjectStorageApi(cls._cls_ns, cls.logger)
cls.obj_cname = "obj_props_" + random_str(8)
print "%d containers in %fs, %f containers per second." % (
counter, now - checkpoint, counter / (now - checkpoint))
counter = 0
checkpoint = now
created.append(res)
now = time.time()
for coro in POOL.coroutines_running:
coro.kill()
while not RESULTS.empty():
created.append(RESULTS.get(block=False))
end = time.time()
rate = len(created) / (end - start)
print "End. %d containers created in %fs, %f containers per second." % (
len(created), end - start, rate)
print "Cleaning..."
for _ in POOL.starmap(API.container_delete,
[('benchmark', n) for n in created]):
pass
POOL.waitall()
return rate
if __name__ == '__main__':
import os
import sys
THREADS = int(sys.argv[1]) if len(sys.argv) > 1 else 1
API = ObjectStorageApi(os.getenv('OIO_NS', 'OPENIO'))
RESULTS = LightQueue(THREADS * 10)
POOL = GreenPool(THREADS)
main(THREADS)
class TestStorageTierer(BaseTestCase):
def setUp(self):
super(TestStorageTierer, self).setUp()
self.namespace = self.conf['namespace']
self.test_account = "test_storage_tiering_%f" % time.time()
self.api = ObjectStorageApi(self.namespace)
self.gridconf = {
"namespace": self.namespace,
"container_fetch_limit": 2,
"content_fetch_limit": 2,
"account": self.test_account,
"outdated_threshold": 0,
"new_policy": "EC"
}
self.api.container = ContainerClient(self.gridconf)
self._populate()
def _populate(self):
self.container_0_name = 'container_empty'
self.container_0 = self._new_container(self.container_0_name)
self.container_1_name = 'container_with_1_content'
self.container_1 = self._new_container(self.container_1_name)
self.container_1_content_0_name = 'container_1_content_0'
self.container_1_content_0 = self._new_object(
self.container_1_name, self.container_1_content_0_name, 'SINGLE')
self.container_2_name = 'container_with_2_contents'
self.container_2 = self._new_container(self.container_1_name)
self.container_2_content_0_name = 'container_2_content_0'
self.container_2_content_0 = self._new_object(
self.container_2_name, self.container_2_content_0_name, 'SINGLE')
self.container_2_content_1_name = 'container_2_content_1'
self.container_2_content_1 = self._new_object(
self.container_2_name, self.container_2_content_1_name,
'TWOCOPIES')
def _new_container(self, container):
self.api.container_create(self.test_account, container)
cnt = self.api.container_get_properties(self.test_account, container)
return cnt
def _new_object(self, container, obj_name, stgpol):
data = random_data(10)
self.api.object_create(self.test_account,
container,
obj_name=obj_name,
policy=stgpol,
data=data)
obj = self.api.object_get_properties(self.test_account, container,
obj_name)
return obj
def tearDown(self):
super(TestStorageTierer, self).tearDown()
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
actual = [x[0] for x in self.api.container_list(self.test_account)]
if len(actual) < 3:
print("Slow event propagation!")
# account events have not yet propagated
time.sleep(3.0)
actual = [
x[0] for x in self.api.container_list(self.test_account)[0]
]
gen = worker._list_containers()
self.assertListEqual(list(gen), actual)
def test_iter_content_list_outdated_threshold_0(self):
self.gridconf["outdated_threshold"] = 0
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_1_name,
self.container_1_content_0_name,
int(self.container_1_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_0_name,
int(self.container_2_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_9999999999(self):
self.gridconf["outdated_threshold"] = 9999999999
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_2(self):
# add a new content created after the three previous contents
now = int(time.time())
time.sleep(2)
self._new_object(self.container_2_name, 'titi', 'TWOCOPIES')
self.gridconf["outdated_threshold"] = 2
worker = StorageTiererWorker(self.gridconf, Mock())
with mock.patch('oio.crawler.storage_tierer.time.time',
mock.MagicMock(return_value=now)):
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_1_name,
self.container_1_content_0_name,
int(self.container_1_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_0_name,
int(self.container_2_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_skip_good_policy(self):
self.gridconf["new_policy"] = "SINGLE"
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
help='Start a worker connecter to the given addresses')
parser.add_argument("--controller",
dest="controller",
action='store_true',
default=False,
help='Start a controller bond to the given addresses')
parser.add_argument("endpoints",
metavar='ENDPOINT',
type=str,
nargs='+',
help='Endpoints to connect/bind to')
args = parser.parse_args()
zctx = zmq.Context()
if args.controller:
s = ObjectStorageApi("benchmark")
#Creating account
ac = AccountClient({"namespace": "benchmark"})
retry = 3
for i in range(retry + 1):
try:
ac.account_create("benchmark_account")
break
except ClientException:
if i < retry:
time.sleep(2)
else:
raise
#Creating Container
def __init__(self, conf, logger, **kwargs):
super(MetaRebuilder, self).__init__(conf, logger, **kwargs)
self.api = ObjectStorageApi(self.conf['namespace'], logger=self.logger)
'--object-count',
type=int,
help='Total number of objects to create (duration will be ignored).')
args = parser.parse_args()
THREADS = args.concurrency
POLICY = args.storage_policy
DURATION = 60.0
OBJECT_COUNT = 1000
CREATION_MODE = 'duration'
if args.duration is not None:
CREATION_MODE = 'duration'
DURATION = args.duration
if args.object_count is not None:
CREATION_MODE = 'object count'
ACCOUNT = os.getenv('OIO_ACCOUNT', 'benchmark')
API = ObjectStorageApi(os.getenv('OIO_NS', 'OPENIO'))
API._object_upload = _object_upload
RESULTS = LightQueue(THREADS * 10)
POOL = GreenPool(THREADS)
if CREATION_MODE == 'duration':
print("Creating objects during %f seconds." % DURATION)
create_objects_in_period(THREADS, duration=DURATION)
else:
print("Creating %d objects." % args.object_count)
create_fixed_number_of_objects(THREADS, obj_count=args.object_count)
def __init__(self, tool, queue_workers, queue_reply):
super(AccountRebuilderWorker, self).__init__(tool, queue_workers,
queue_reply)
self.api = ObjectStorageApi(self.tool.namespace, logger=self.logger)
class TestContainerLifecycle(BaseTestCase):
CONTAINERS = set()
def setUp(self):
super(TestContainerLifecycle, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_lifecycle"
self.container = "lifecycle-" + random_str(4)
self.lifecycle = ContainerLifecycle(
self.api, self.account, self.container)
@staticmethod
def _time_to_date(timestamp=None):
if timestamp is None:
timestamp = time.time()
return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp))
def _upload_something(self, prefix="", path=None, **kwargs):
path = path or (prefix + random_str(8))
self.api.object_create(self.account, self.container,
obj_name=path, data=path, **kwargs)
self.__class__.CONTAINERS.add(self.container)
return self.api.object_show(self.account, self.container, path)
def _enable_versioning(self):
if not self.api.container_create(
self.account, self.container,
system={'sys.m2.policy.version': '-1'}):
self.api.container_set_properties(
self.account, self.container,
system={'sys.policy.version': '-1'})
def test_load_from_container_property(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Expiration>
<Days>60</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>1</NoncurrentDays>
<StorageClass>THREECOPIES</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self.lifecycle.load()
self.assertIn('rule1', self.lifecycle._rules)
rule = self.lifecycle._rules['rule1']
self.assertEqual('rule1', rule.id)
self.assertTrue(rule.enabled)
self.assertIsNotNone(rule.filter)
self.assertIn('Expiration', rule.actions)
self.assertIn('Transition', rule.actions)
self.assertIn('NoncurrentVersionExpiration', rule.actions)
self.assertIn('NoncurrentVersionTransition', rule.actions)
def test_save_to_container_property(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Expiration>
<Days>60</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>1</NoncurrentDays>
<StorageClass>THREECOPIES</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""
self.api.container_create(self.account, self.container)
self.lifecycle.load_xml(source)
self.lifecycle.save()
props = self.api.container_get_properties(
self.account, self.container)['properties']
self.assertIn(LIFECYCLE_PROPERTY_KEY, props)
self.assertEqual(source, props[LIFECYCLE_PROPERTY_KEY])
def test_immediate_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() - 86400))
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertNotIn(obj_meta['name'], obj_names)
def test_future_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 86400))
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_expiration_by_delay(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertNotIn(obj_meta['name'], obj_names)
def test_future_expiration_by_delay(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
self.lifecycle.apply(obj_meta)
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_expiration_filtered_by_prefix(self):
obj_meta = self._upload_something(prefix="photos/")
obj_meta2 = self._upload_something(prefix="documents/")
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Prefix>documents/</Prefix>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta2))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.assertNotIn(obj_meta2['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_expiration_filtered_by_tag(self):
obj_meta = self._upload_something()
obj_meta2 = self._upload_something(properties={'status': 'deprecated'})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Expiration>
<Days>0</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta2))
container_descr = self.api.object_list(self.account, self.container)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.assertNotIn(obj_meta2['name'], obj_names)
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_immediate_transition_filtered_by_tag(self):
obj_meta = self._upload_something(policy='SINGLE')
obj_meta2 = self._upload_something(policy='SINGLE',
properties={'status': 'deprecated'})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Transition>
<Days>0</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta2))
obj_meta_after = self.api.object_show(
self.account, self.container, obj_meta['name'])
obj_meta_after2 = self.api.object_show(
self.account, self.container, obj_meta2['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual('THREECOPIES', obj_meta_after2['policy'])
self.api.object_delete(self.account, self.container, obj_meta['name'])
self.api.object_delete(self.account, self.container, obj_meta2['name'])
def test_immediate_noncurrent_expiration(self):
self._enable_versioning()
obj_meta = self._upload_something()
obj_meta_v2 = self._upload_something(path=obj_meta['name'])
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter></Filter>
<NoncurrentVersionExpiration>
<NoncurrentDays>0</NoncurrentDays>
</NoncurrentVersionExpiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
consume(self.lifecycle.apply(obj_meta))
consume(self.lifecycle.apply(obj_meta_v2))
container_descr = self.api.object_list(self.account, self.container,
versions=True)
obj_names = [obj['name'] for obj in container_descr['objects']]
self.assertIn(obj_meta['name'], obj_names)
self.assertEqual(1, len(obj_names))
self.api.object_delete(self.account, self.container, obj_meta['name'])
def test_execute_immediate_expiration(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>0</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self._upload_something()
self._upload_something()
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(3, len(results))
listing = self.api.object_list(self.account, self.container)
self.assertEqual(0, len(listing['objects']))
def test_execute_expiration_on_missing_objects(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>0</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self.lifecycle.load()
fake_listing = {
'objects': [
{'name': 'a', 'ctime': '12'},
{'name': 'b', 'ctime': '12'},
{'name': 'c', 'ctime': '12'},
{'name': 'd', 'ctime': '12'}],
'truncated': False
}
with patch.object(self.api, 'object_list',
side_effect=[fake_listing]):
results = [x for x in self.lifecycle.execute()]
self.assertEqual(4, len(results))
for res in results:
self.assertIsInstance(res[3], Exception)
class TestAdmin(BaseTestCase):
def setUp(self):
super(TestAdmin, self).setUp()
self.admin = AdminClient(self.conf)
self.api = ObjectStorageApi(self.ns)
self.account = "test_admin"
self.container = "admin-" + random_str(4)
self.api.container_create(self.account, self.container)
def tearDown(self):
super(TestAdmin, self).tearDown()
try:
self.api.container_delete(self.account, self.container)
except Exception:
pass
def test_election_leave_service_id(self):
status = self.admin.election_status("meta2",
account=self.account,
reference=self.container)
peers = status["peers"]
service_id = peers.keys()[random.randrange(len(peers))]
election = self.admin.election_leave("meta2",
account=self.account,
reference=self.container,
service_id=service_id)
self.assertEqual(1, len(election))
self.assertEqual(200, election[service_id]["status"]["status"])
def test_election_leave_serveral_service_ids(self):
status = self.admin.election_status("meta2",
account=self.account,
reference=self.container)
peers = status["peers"]
if len(peers) < 2:
self.skipTest('Can only run in a replicated environment')
service_ids = peers.keys()[:2]
election = self.admin.election_leave("meta2",
account=self.account,
reference=self.container,
service_id=service_ids)
self.assertEquals(2, len(election))
self.assertEquals(200, election[service_ids[0]]["status"]["status"])
self.assertEquals(200, election[service_ids[1]]["status"]["status"])
def test_has_base(self):
info = self.admin.has_base('meta2',
account=self.account,
reference=self.container)
for peer, meta in info.iteritems():
self.assertEqual(200, meta['status']['status'])
peer = info.keys()[0]
peer_loc = info[peer]['body']
self.assertTrue(os.path.isfile(peer_loc))
os.remove(peer_loc)
info = self.admin.has_base('meta2',
account=self.account,
reference=self.container)
self.assertNotEquals(200, info[peer]['status']['status'])
del info[peer]
for peer, meta in info.iteritems():
self.assertEqual(200, meta['status']['status'])
class TestMeta1RefMapping(BaseTestCase):
def setUp(self):
super(TestMeta1RefMapping, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_refmapping"
self.reference = "refmapping-" + random_str(4)
self.logger = logging.getLogger('test')
self.mapping = Meta1RefMapping(self.ns, logger=self.logger)
def _get_cid(self):
data_dir = self.api.directory.get_properties(self.account,
self.reference)
return data_dir['cid']
def _get_services(self, service_type):
data_dir = self.api.directory.list(self.account, self.reference)
services = list()
for d in data_dir['srv']:
if d['type'] == service_type:
services.append(d['host'])
return services
def _test_move(self, service_type, base=None, destination=True):
if base is None:
base = self._get_cid()
raw_services = self._get_services(service_type)
expected_services = list(raw_services)
src_service = expected_services.pop()
dest_service = None
all_meta2_services = self.conscience.all_services(service_type, True)
if len(all_meta2_services) <= len(raw_services):
self.skipTest("need at least %d %s" %
(len(raw_services) + 1, service_type))
if destination:
for service in all_meta2_services:
if service['addr'] not in raw_services:
dest_service = service['addr']
expected_services.append(dest_service)
moved = self.mapping.move(src_service, dest_service, base,
service_type)
moved_ok = self.mapping.apply(moved)
self.assertEqual(1, len(moved_ok))
data_dir = self.api.directory.list(self.account, self.reference)
new_services = list()
for d in data_dir['srv']:
if d['type'] == service_type:
new_services.append(d['host'])
if destination:
self.assertListEqual(sorted(expected_services),
sorted(new_services))
else:
for expected_service in expected_services:
self.assertIn(expected_service, new_services)
self.assertNotIn(src_service, new_services)
self.assertEqual(len(expected_services) + 1, len(new_services))
return (src_service, expected_services)
def test_move_meta2(self):
self.api.container_create(self.account, self.reference)
_, expected_services = self._test_move('meta2')
properties = self.api.container_get_properties(self.account,
self.reference)
peers = properties['system']['sys.peers']
new_services = peers.split(',')
self.assertListEqual(sorted(expected_services), sorted(new_services))
self.api.object_create(self.account,
self.reference,
data="move meta2",
obj_name="test")
for i in range(0, 10):
self.api.object_show(self.account, self.reference, "test")
self.api.object_delete(self.account, self.reference, "test")
sleep(0.5)
self.api.container_delete(self.account, self.reference)
def test_move_meta2_with_seq(self):
self.api.container_create(self.account, self.reference)
properties = self.api.container_get_properties(self.account,
self.reference)
base = properties['system']['sys.name']
_, expected_services = self._test_move('meta2', base=base)
properties = self.api.container_get_properties(self.account,
self.reference)
peers = properties['system']['sys.peers']
new_services = peers.split(',')
self.assertListEqual(sorted(expected_services), sorted(new_services))
self.api.object_create(self.account,
self.reference,
data="move meta2",
obj_name="test")
for i in range(0, 10):
self.api.object_show(self.account, self.reference, "test")
self.api.object_delete(self.account, self.reference, "test")
sleep(0.5)
self.api.container_delete(self.account, self.reference)
def test_move_meta2_without_destination(self):
self.api.container_create(self.account, self.reference)
properties = self.api.container_get_properties(self.account,
self.reference)
base = properties['system']['sys.name']
src_service, expected_services = self._test_move('meta2',
base=base,
destination=False)
properties = self.api.container_get_properties(self.account,
self.reference)
peers = properties['system']['sys.peers']
new_services = peers.split(',')
for expected_service in expected_services:
self.assertIn(expected_service, new_services)
self.assertNotIn(src_service, new_services)
self.assertEqual(len(expected_services) + 1, len(new_services))
self.api.object_create(self.account,
self.reference,
data="move meta2",
obj_name="test")
for i in range(0, 10):
self.api.object_show(self.account, self.reference, "test")
self.api.object_delete(self.account, self.reference, "test")
sleep(0.5)
self.api.container_delete(self.account, self.reference)
def test_move_sqlx(self):
execute('oio-sqlx -O AutoCreate %s/%s/%s '
'"create table foo (a INT, b TEXT)"' %
(self.ns, self.account, self.reference))
self._test_move('sqlx')
execute('oio-sqlx %s/%s/%s "destroy"' %
(self.ns, self.account, self.reference))
def test_move_with_dest_service_already_used(self):
self.api.container_create(self.account, self.reference)
base = self._get_cid()
raw_services = self._get_services('meta2')
src_service = raw_services[0]
dest_service = raw_services[0]
if len(raw_services) > 1:
dest_service = raw_services[1]
self.assertRaises(ValueError, self.mapping.move, src_service,
dest_service, base, 'meta2')
self.api.container_delete(self.account, self.reference)
def test_move_with_src_service_not_used(self):
self.api.container_create(self.account, self.reference)
base = self._get_cid()
raw_services = self._get_services('meta2')
src_service = '127.0.0.1:666'
dest_service = None
all_meta2_services = self.conscience.all_services('meta2', True)
for service in all_meta2_services:
if service['addr'] not in raw_services:
src_service = service['addr']
dest_service = service['addr']
if dest_service is None:
self.skipTest("need at least 2 meta2")
self.assertRaises(ValueError, self.mapping.move, src_service,
dest_service, base, 'meta2')
self.api.container_delete(self.account, self.reference)
def test_move_with_wrong_dest(self):
self.api.container_create(self.account, self.reference)
base = self._get_cid()
raw_services = self._get_services('meta2')
src_service = raw_services[0]
dest_service = '127.0.0.1:666'
self.assertRaises(ValueError, self.mapping.move, src_service,
dest_service, base, 'meta2')
self.api.container_delete(self.account, self.reference)
class StorageTiererWorker(object):
def __init__(self, conf, logger):
self.conf = conf
self.logger = logger
self.account = conf[CONF_ACCOUNT]
self.api = ObjectStorageApi(conf['namespace'], logger=self.logger)
self.container_client = self.api.container
self.account_client = self.api.account
self.passes = 0
self.errors = 0
self.last_reported = 0
self.contents_run_time = 0
self.total_contents_processed = 0
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_contents_per_second = int_value(
conf.get('contents_per_second'), 30)
self.container_fetch_limit = int_value(
conf.get('container_fetch_limit'), 100)
self.content_fetch_limit = int_value(conf.get('content_fetch_limit'),
100)
self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD),
9999999999)
self.new_policy = conf.get(CONF_NEW_POLICY)
def _list_containers(self):
container = None
while True:
resp = self.account_client.container_list(
self.account,
marker=container,
limit=self.container_fetch_limit)
if len(resp["listing"]) == 0:
break
for res in resp["listing"]:
container = res[0]
yield container
def _list_contents(self):
for container in self._list_containers():
marker = None
while True:
try:
_, listing = self.container_client.content_list(
account=self.account,
reference=container,
limit=self.content_fetch_limit,
marker=marker)
except NotFound:
self.logger.warn(
"Container %s appears in account but doesn't exist",
container)
break
if len(listing["objects"]) == 0:
break
for obj in listing["objects"]:
marker = obj["name"]
if obj["mtime"] > time.time() - self.outdated_threshold:
continue
if obj["policy"] == self.new_policy:
continue
if true_value(obj['deleted']):
continue
yield (self.account, container, obj["name"],
obj["version"])
def run(self):
start_time = report_time = time.time()
total_errors = 0
for (account, container, obj, version) in self._list_contents():
self.safe_change_policy(account, container, obj, version)
self.contents_run_time = ratelimit(self.contents_run_time,
self.max_contents_per_second)
self.total_contents_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(total).2f ' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'total': (now - start_time)
})
report_time = now
total_errors += self.errors
self.passes = 0
self.errors = 0
self.last_reported = now
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(content_rate).2f ' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'content_rate': self.total_contents_processed / elapsed
})
def safe_change_policy(self, account, container, obj, version):
try:
self.change_policy(account, container, obj, version)
except Exception:
self.errors += 1
self.logger.exception(
"ERROR while changing policy for content "
"%s/%s/%s/%s", account, container, obj, str(version))
self.passes += 1
def change_policy(self, account, container, obj, version):
self.logger.info("Changing policy for content %s/%s/%s/%s", account,
container, obj, str(version))
self.api.object_change_policy(self,
account,
container,
obj,
self.new_policy,
version=version)
class TestContentRebuildFilter(BaseTestCase):
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.namespace = self.conf['namespace']
self.gridconf = {"namespace": self.namespace}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = ContainerClient(self.conf)
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = ObjectStorageApi(namespace=self.namespace)
self.stgpol = "SINGLE"
self.conf['tube'] = 'rebuild'
self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300'
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.tube = self.conf.get('tube', 'rebuild')
self.beanstalk = Beanstalk.from_url(queue_url)
self.beanstalk.use(self.tube)
def _create_event(self, content_name, present_chunks, missing_chunks,
content_id):
event = {}
event["when"] = time.time()
event["event"] = "storage.content.broken"
event["data"] = {
"present_chunks": present_chunks,
"missing_chunks": missing_chunks
}
event["url"] = {
"ns": self.namespace,
"account": self.account,
"user": self.container,
"path": content_name,
"id": self.container_id,
"content": content_id
}
return event
def _is_chunks_created(self, previous, after, pos_created):
remain = list(after)
for p in previous:
for r in remain:
if p["url"] == r["url"]:
remain.remove(r)
break
if len(remain) != len(pos_created):
return False
for r in remain:
if r["pos"] in pos_created:
remain.remove(r)
else:
return False
return True
def _rebuild(self, event, job_id=0):
self.blob_rebuilder = subprocess.Popen([
'oio-blob-rebuilder', self.namespace,
'--beanstalkd=127.0.0.1:11300'
])
time.sleep(3)
self.blob_rebuilder.kill()
def _remove_chunks(self, chunks, content_id):
if not chunks:
return
for chunk in chunks:
chunk['id'] = chunk['url']
chunk['content'] = content_id
chunk['type'] = 'chunk'
self.container_client.container_raw_delete(self.account,
self.container,
data=chunks)
def _check_rebuild(self,
content_name,
chunks,
missing_pos,
meta,
chunks_to_remove,
chunk_created=True):
self._remove_chunks(chunks_to_remove, meta['id'])
event = self._create_event(content_name, chunks, missing_pos,
meta['id'])
self.notify_filter.process(env=event, cb=None)
self._rebuild(event)
_, after = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
self.assertIs(chunk_created,
self._is_chunks_created(chunks, after, missing_pos))
def test_nothing_missing(self):
content_name = "test_nothing_missing"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
for chunk in chunks:
chunk.pop('score', None)
missing_pos = []
self._check_rebuild(content_name,
chunks,
missing_pos,
meta,
chunks_to_remove,
chunk_created=True)
def test_missing_1_chunk(self):
content_name = "test_missing_1_chunk"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_last_chunk(self):
content_name = "test_missing_last_chunk"
data = random_str(1024 * 1024 * 4)
self.object_storage_api.object_create(account=self.account,
container=self.container,
data=data,
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["3"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_2_chunks(self):
content_name = "test_missing_2_chunks"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
for i in range(0, 2):
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0", "0"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_all_chunks(self):
content_name = "test_missing_all_chunks"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="SINGLE",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0"]
self._check_rebuild(content_name,
chunks,
missing_pos,
meta,
chunks_to_remove,
chunk_created=False)
def test_missing_all_chunks_of_a_pos(self):
content_name = "test_missing_2_chunks"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
for i in range(0, 3):
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0"]
self._check_rebuild(content_name,
chunks,
missing_pos,
meta,
chunks_to_remove,
chunk_created=False)
def test_missing_multiple_chunks(self):
content_name = "test_missing_multiple_chunks"
data = random_str(1024 * 1024 * 4)
self.object_storage_api.object_create(account=self.account,
container=self.container,
data=data,
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(9))
chunks_to_remove.append(chunks.pop(6))
chunks_to_remove.append(chunks.pop(4))
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0", "1", "2", "3"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_1_chunk_ec(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_1_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_m_chunk_ec(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_m_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
for i in range(0, 3):
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1", "0.2", "0.3"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_m_chunk_ec_2(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_m_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(3))
chunks_to_remove.append(chunks.pop(5))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1", "0.5", "0.8"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_m1_chunk_ec(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_m1_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name, account=self.account)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1", "0.2", "0.3", "0.4"]
self._check_rebuild(content_name,
chunks,
missing_pos,
meta,
chunks_to_remove,
chunk_created=False)
class TestContainerLifecycle(BaseTestCase):
CONTAINERS = set()
def setUp(self, recursive=False):
super(TestContainerLifecycle, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_lifecycle"
self.container = "lifecycle-" + random_str(4)
self.lifecycle = ContainerLifecycle(
self.api, self.account, self.container, recursive=recursive)
@staticmethod
def _time_to_date(timestamp=None):
if timestamp is None:
timestamp = time.time()
return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(timestamp))
def _upload_something(self, prefix="", path=None, **kwargs):
path = path or (prefix + random_str(8))
self.api.object_create(self.account, self.container,
obj_name=path, data=path, **kwargs)
self.__class__.CONTAINERS.add(self.container)
obj_meta = self.api.object_show(self.account, self.container, path)
# add fields like LifeCycle do in execute to support CH
obj_meta['orig_name'] = obj_meta['name']
obj_meta['container'] = self.container
return obj_meta
def _enable_versioning(self):
if not self.api.container_create(
self.account, self.container,
system={'sys.m2.policy.version': '-1'}):
self.api.container_set_properties(
self.account, self.container,
system={'sys.policy.version': '-1'})
def test_load_from_container_property(self):
source = """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key1</Key>
<Value>value1</Value>
</Tag>
<Tag>
<Key>key2</Key>
<Value>value2</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Expiration>
<Days>60</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>1</NoncurrentDays>
<StorageClass>THREECOPIES</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""
props = {LIFECYCLE_PROPERTY_KEY: source}
self.api.container_create(self.account, self.container,
properties=props)
self.lifecycle.load()
self.assertEqual(1, len(self.lifecycle.rules))
rule = self.lifecycle.rules[0]
self.assertIsNotNone(rule)
self.assertEqual('rule1', rule.id)
self.assertIsNotNone(rule.filter)
self.assertEqual('documents/', rule.filter.prefix)
self.assertDictEqual({'key1': 'value1', 'key2': 'value2'},
rule.filter.tags)
self.assertTrue(rule.enabled)
self.assertEqual(4, len(rule.actions))
expiration = rule.actions[0]
self.assertEqual(Expiration, type(expiration))
self.assertEqual(60, expiration.filter.days)
transition = rule.actions[1]
self.assertEqual(Transition, type(transition))
self.assertEqual(1, transition.filter.days)
self.assertEqual('THREECOPIES', transition.policy)
expiration = rule.actions[2]
self.assertEqual(NoncurrentVersionExpiration, type(expiration))
self.assertEqual(60, expiration.filter.days)
transition = rule.actions[3]
self.assertEqual(NoncurrentVersionTransition, type(transition))
self.assertEqual(1, transition.filter.days)
self.assertEqual('THREECOPIES', transition.policy)
def test_save_to_container_property(self):
source = """
<LifecycleConfiguration
xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ID>rule1</ID>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>key</Key>
<Value>value</Value>
</Tag>
</And>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>60</Days>
</Expiration>
<Transition>
<StorageClass>SINGLE</StorageClass>
<Days>30</Days>
</Transition>
<Transition>
<StorageClass>THREECOPIES</StorageClass>
<Days>1</Days>
</Transition>
<NoncurrentVersionExpiration>
<NoncurrentDays>60</NoncurrentDays>
</NoncurrentVersionExpiration>
<NoncurrentVersionTransition>
<StorageClass>THREECOPIES</StorageClass>
<NoncurrentDays>1</NoncurrentDays>
</NoncurrentVersionTransition>
</Rule>
</LifecycleConfiguration>
"""
self.api.container_create(self.account, self.container)
self.lifecycle.load_xml(source)
self.lifecycle.save()
xml = self.lifecycle.get_configuration()
self.assertEqual(
source.replace(' ', '').replace('\n', ''),
xml.replace(' ', '').replace('\n', ''))
def test_immediate_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 86400))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+172800)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Deleted', status)
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta['container'],
obj_meta['name'])
def test_immediate_expiration_by_date_after_new_object(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 172800))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(
self.account, obj_meta['container'], obj_meta['name'])
def test_future_expiration_by_date(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Date>%s</Date>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""" % self._time_to_date(time.time() + 86400))
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
def test_expiration_by_delay(self):
obj_meta = self._upload_something()
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Deleted', status)
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta['container'],
obj_meta['name'])
def test_expiration_filtered_by_prefix(self):
obj_meta = self._upload_something(prefix="photos/")
obj_meta2 = self._upload_something(prefix="documents/")
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Prefix>documents/</Prefix>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.api.object_show(self.account, obj_meta2['container'],
obj_meta2['orig_name'])
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Deleted', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['orig_name'])
def test_expiration_filtered_by_tag(self):
obj_meta = self._upload_something()
obj_meta2 = self._upload_something(
properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>approved</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_show(self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_show(self.account, obj_meta3['container'],
obj_meta3['name'])
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Deleted', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'])
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_show(self.account, obj_meta3['container'],
obj_meta3['name'])
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['name'])
def test_transition_filtered_by_tag(self):
obj_meta = self._upload_something(policy='SINGLE')
obj_meta2 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>approved</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('SINGLE', obj_meta2_after['policy'])
self.assertEqual(1, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Policy changed to THREECOPIES', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('THREECOPIES', obj_meta2_after['policy'])
self.assertEqual(3, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['name'])
def test_transition_filtered_by_tags(self):
obj_meta = self._upload_something(policy='SINGLE')
obj_meta2 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<And>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
</And>
</Filter>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('SINGLE', obj_meta2_after['policy'])
self.assertEqual(1, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Policy changed to THREECOPIES', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['name'])
self.assertEqual('THREECOPIES', obj_meta2_after['policy'])
self.assertEqual(3, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['name'])
self.api.object_delete(self.account, obj_meta2['container'],
obj_meta2['name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['name'])
def test_transition_filtered_by_prefix_and_tags(self):
obj_meta = self._upload_something(policy='SINGLE', prefix="documents/")
obj_meta2 = self._upload_something(
policy='SINGLE', prefix="documents/", properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
obj_meta3 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""})
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter>
<And>
<Prefix>documents/</Prefix>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
<Tag>
<Key>test1</Key>
<Value>test2</Value>
</Tag>
</And>
</Filter>
<Transition>
<Days>1</Days>
<StorageClass>THREECOPIES</StorageClass>
</Transition>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta2)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta3)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['orig_name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['orig_name'])
self.assertEqual('SINGLE', obj_meta2_after['policy'])
self.assertEqual(1, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['orig_name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta2_copy, _, _, status = results[0]
self.assertEqual(obj_meta2, obj_meta2_copy)
self.assertEqual('Policy changed to THREECOPIES', status)
results = [x for x in self.lifecycle.apply(
obj_meta3, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta3_copy, _, _, status = results[0]
self.assertEqual(obj_meta3, obj_meta3_copy)
self.assertEqual('Kept', status)
obj_meta_after, chunks = self.api.object_locate(
self.account, obj_meta['container'], obj_meta['orig_name'])
self.assertEqual('SINGLE', obj_meta_after['policy'])
self.assertEqual(1, len(chunks))
obj_meta2_after, chunks2 = self.api.object_locate(
self.account, obj_meta2['container'], obj_meta2['orig_name'])
self.assertEqual('THREECOPIES', obj_meta2_after['policy'])
self.assertEqual(3, len(chunks2))
obj_meta3_after, chunks3 = self.api.object_locate(
self.account, obj_meta3['container'], obj_meta3['orig_name'])
self.assertEqual('SINGLE', obj_meta3_after['policy'])
self.assertEqual(1, len(chunks3))
self.api.object_delete(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.api.object_delete(self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_delete(self.account, obj_meta3['container'],
obj_meta3['orig_name'])
def test_expiration_with_versioning(self):
self._enable_versioning()
obj_meta = self._upload_something()
obj_meta_v2 = self._upload_something(path=obj_meta['name'])
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter></Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta_v2)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
results = [x for x in self.lifecycle.apply(
obj_meta_v2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Deleted', status)
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.assertRaises(NoSuchObject, self.api.object_locate,
self.account, self.container, obj_meta_v2['name'])
self.api.object_show(self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
self.api.object_delete(
self.account, obj_meta['container'], obj_meta['name'],
version=obj_meta['version'])
def test_noncurrent_expiration(self):
self._enable_versioning()
obj_meta = self._upload_something()
obj_meta_v2 = self._upload_something(path=obj_meta['name'])
self.lifecycle.load_xml("""
<LifecycleConfiguration>
<Rule>
<Filter></Filter>
<NoncurrentVersionExpiration>
<NoncurrentDays>1</NoncurrentDays>
</NoncurrentVersionExpiration>
<Status>enabled</Status>
</Rule>
</LifecycleConfiguration>
""")
results = [x for x in self.lifecycle.apply(obj_meta_v2)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(obj_meta)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Kept', status)
self.api.object_show(self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
results = [x for x in self.lifecycle.apply(
obj_meta_v2, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_v2_copy, _, _, status = results[0]
self.assertEqual(obj_meta_v2, obj_meta_v2_copy)
self.assertEqual('Kept', status)
results = [x for x in self.lifecycle.apply(
obj_meta, now=time.time()+86400)]
self.assertEqual(1, len(results))
obj_meta_copy, _, _, status = results[0]
self.assertEqual(obj_meta, obj_meta_copy)
self.assertEqual('Deleted', status)
self.api.object_show(self.account, self.container,
obj_meta_v2['name'],
version=obj_meta_v2['version'])
self.assertRaises(NoSuchObject, self.api.object_show,
self.account, obj_meta['container'],
obj_meta['name'], version=obj_meta['version'])
self.api.object_delete(
self.account, self.container, obj_meta_v2['name'],
version=obj_meta_v2['version'])
def test_execute_expiration(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""})
for _ in range(3):
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(3, len(listing['objects']))
results = [x for x in self.lifecycle.execute(now=time.time()+86400)]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Deleted', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(0, len(listing['objects']))
def test_execute_expiration_with_disabled_status(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Disabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""})
for _ in range(3):
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(3, len(listing['objects']))
results = [x for x in self.lifecycle.execute(now=time.time()+86400)]
self.assertEqual(3, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container)
self.assertEqual(3, len(listing['objects']))
def test_execute_expiration_on_missing_objects(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
</LifecycleConfiguration>
"""})
fake_listing = {
'objects': [
{'name': 'a', 'version': 1540933092888883, 'mtime': '12',
'deleted': False},
{'name': 'b', 'version': 1540933092888883, 'mtime': '12',
'deleted': False},
{'name': 'c', 'version': 1540933092888883, 'mtime': '12',
'deleted': False},
{'name': 'd', 'version': 1540933092888883, 'mtime': '12',
'deleted': False}],
'truncated': False
}
with patch.object(self.api, 'object_list',
side_effect=[fake_listing]):
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(4, len(results))
for res in results:
self.assertIsInstance(res[3], Exception)
def test_execute_exceeding_version_expiration_without_versioning(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<NoncurrentVersionExpiration>
<NoncurrentCount>2</NoncurrentCount>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""})
for _ in range(5):
self._upload_something()
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(5, len(results))
for res in results:
self.assertEqual('Kept', res[3])
listing = self.api.object_list(self.account, self.container,
versions=True)
self.assertEqual(5, len(listing['objects']))
def test_execute_exceeding_version_expiration_with_versioning(self):
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<ID>rule1</ID>
<Filter>
</Filter>
<Status>Enabled</Status>
<NoncurrentVersionExpiration>
<NoncurrentCount>2</NoncurrentCount>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>
"""}, system={"sys.m2.policy.version": "4"})
for _ in range(5):
self._upload_something(path="versioned1")
for _ in range(5):
self._upload_something(path="versioned2")
listing = self.api.object_list(self.account, self.container,
versions=True)
self.assertEqual(10, len(listing['objects']))
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(10, len(results))
listing = self.api.object_list(self.account, self.container,
versions=True)
self.assertEqual(6, len(listing['objects']))
def test_execute_multiple_rules(self):
meta = {"sys.m2.policy.version": "4"}
self.api.container_create(
self.account, self.container, properties={LIFECYCLE_PROPERTY_KEY: """
<LifecycleConfiguration>
<Rule>
<Filter>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>enabled</Status>
</Rule>
<Rule>
<Filter>
<Prefix>documents/</Prefix>
</Filter>
<Expiration>
<Days>1</Days>
</Expiration>
<Status>Enabled</Status>
</Rule>
</LifecycleConfiguration>
"""}, system=meta)
obj_meta = self._upload_something(policy='SINGLE', system=meta)
obj_meta2 = self._upload_something(
policy='SINGLE', properties={TAGGING_KEY: """
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>status</Key>
<Value>deprecated</Value>
</Tag>
</TagSet>
</Tagging>
"""}, system=meta)
obj_meta3 = self._upload_something(path="documents/object3",
policy='SINGLE',
system=meta)
self.lifecycle.load()
results = [x for x in self.lifecycle.execute()]
self.assertEqual(6, len(results))
for res in results:
self.assertEqual('Kept', res[3])
self.api.object_show(self.account, obj_meta['container'],
obj_meta['orig_name'])
self.api.object_show(self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_show(self.account, obj_meta3['container'],
obj_meta3['orig_name'])
results = [x for x in self.lifecycle.execute(now=time.time()+86400)]
self.assertEqual(5, len(results))
self.api.object_locate(
self.account, obj_meta['container'], obj_meta['orig_name'])
self.api.object_show(
self.account, obj_meta['container'], obj_meta['orig_name'],
version=obj_meta['version'])
self.assertRaises(NoSuchObject, self.api.object_locate,
self.account, obj_meta2['container'],
obj_meta2['orig_name'])
self.api.object_show(
self.account, obj_meta2['container'], obj_meta2['orig_name'],
version=obj_meta2['version'])
self.assertRaises(NoSuchObject, self.api.object_locate,
self.account, obj_meta3['container'],
obj_meta3['orig_name'])
self.api.object_show(
self.account, obj_meta3['container'], obj_meta3['orig_name'],
version=obj_meta3['version'])
self.api.object_delete(
self.account, obj_meta['container'], obj_meta['orig_name'],
version=obj_meta['version'])
self.api.object_delete(
self.account, obj_meta2['container'], obj_meta2['orig_name'],
version=obj_meta2['version'])
self.api.object_delete(
self.account, obj_meta3['container'], obj_meta3['orig_name'],
version=obj_meta3['version'])
