def __init__(self,
conf,
logger=None,
account_ring=None,
container_ring=None,
storage=None):
for policy, ring_arg in zip(OIO_POLICIES, RING_ARGS):
if ring_arg is not None:
policy.object_ring = FakeRing(**ring_arg)
SwiftApplication.__init__(self,
conf,
logger=logger,
account_ring=account_ring,
container_ring=container_ring)
if conf is None:
conf = dict()
sds_conf = {k[4:]: v for k, v in conf.items() if k.startswith("sds_")}
self.oio_stgpol = []
if 'auto_storage_policies' in conf:
for elem in conf['auto_storage_policies'].split(','):
if ':' in elem:
name, offset = elem.split(':')
self.oio_stgpol.append((name, int(offset)))
else:
self.oio_stgpol.append((elem, 0))
self.oio_stgpol.sort(key=lambda x: x[1])
policies = []
if 'oio_storage_policies' in conf:
for i, pol in enumerate(conf['oio_storage_policies'].split(',')):
policies.append(
storage_policy.StoragePolicy(i, pol, is_default=i == 0))
else:
policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True))
self.POLICIES = storage_policy.StoragePolicyCollection(policies)
# Mandatory, raises KeyError
sds_namespace = sds_conf['namespace']
sds_conf.pop('namespace') # removed to avoid unpacking conflict
# Loaded by ObjectStorageApi if None
sds_proxy_url = sds_conf.pop('proxy_url', None)
# Fix boolean parameter
if 'autocreate' in sds_conf and not (
hasattr(ObjectStorageApi, 'EXTRA_KEYWORDS')
or 'autocreate' in ObjectStorageApi.EXTRA_KEYWORDS):
logger.warn("'autocreate' parameter is ignored by current version"
" of OpenIO SDS. Please update to oio>=4.1.23.")
else:
sds_conf['autocreate'] = config_true_value(
sds_conf.get('autocreate', 'true'))
self.storage = storage or \
ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
self.delete_slo_parts = \
config_true_value(conf.get('delete_slo_parts', False))
self.check_state = \
config_true_value(conf.get('check_state', False))
def setUp(self):
super(TestContainerLifecycle, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_lifecycle"
self.container = "lifecycle-" + random_str(4)
self.lifecycle = ContainerLifecycle(
self.api, self.account, self.container)
def setUp(self):
super(TestAdmin, self).setUp()
self.admin = AdminClient(self.conf)
self.api = ObjectStorageApi(self.ns)
self.account = "test_admin"
self.container = "admin-" + random_str(4)
self.api.container_create(self.account, self.container)
def setUp(self):
super(TestMeta1RefMapping, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_refmapping"
self.reference = "refmapping-" + random_str(4)
self.logger = logging.getLogger('test')
self.mapping = Meta1RefMapping(self.ns, logger=self.logger)
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
api = ObjectStorageApi(self.namespace)
actual = [x[0] for x in api.container_list(self.test_account)]
if len(actual) < 3:
print "Slow event propagation!"
# account events have not yet propagated
time.sleep(3.0)
actual = [x[0] for x in api.container_list(self.test_account)[0]]
gen = worker._list_containers()
self.assertListEqual(list(gen), actual)
def __init__(self,
conf,
memcache=None,
logger=None,
account_ring=None,
container_ring=None,
storage=None):
for policy, ring_arg in zip(POLICIES, ring_args):
if ring_arg is not None:
policy.object_ring = FakeRing(**ring_arg)
SwiftApplication.__init__(self,
conf,
memcache=memcache,
logger=logger,
account_ring=account_ring,
container_ring=container_ring)
if conf is None:
conf = dict()
sds_conf = {
k[4:]: v
for k, v in conf.iteritems() if k.startswith("sds_")
}
self.oio_stgpol = []
if 'auto_storage_policies' in conf:
for elem in conf['auto_storage_policies'].split(','):
if ':' in elem:
name, offset = elem.split(':')
self.oio_stgpol.append((name, int(offset)))
else:
self.oio_stgpol.append((elem, 0))
self.oio_stgpol.sort(key=lambda x: x[1])
policies = []
if 'oio_storage_policies' in conf:
for i, pol in enumerate(conf['oio_storage_policies'].split(',')):
policies.append(
storage_policy.StoragePolicy(i, pol, is_default=i == 0))
else:
policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True))
self.POLICIES = storage_policy.StoragePolicyCollection(policies)
# Mandatory, raises KeyError
sds_namespace = sds_conf['namespace']
sds_conf.pop('namespace') # removed to avoid unpacking conflict
# Loaded by ObjectStorageApi if None
sds_proxy_url = sds_conf.pop('proxy_url', None)
self.storage = storage or \
ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
def setUp(self):
super(TestStorageTierer, self).setUp()
self.namespace = self.conf['namespace']
self.test_account = "test_storage_tiering_%f" % time.time()
self.api = ObjectStorageApi(self.namespace)
self.gridconf = {
"namespace": self.namespace,
"container_fetch_limit": 2,
"content_fetch_limit": 2,
"account": self.test_account,
"outdated_threshold": 0,
"new_policy": "EC"
}
self.api.container = ContainerClient(self.gridconf)
self._populate()
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.namespace = self.conf['namespace']
self.gridconf = {"namespace": self.namespace}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = ContainerClient(self.conf)
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = ObjectStorageApi(namespace=self.namespace)
self.stgpol = "SINGLE"
self.conf['tube'] = 'rebuild'
self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300'
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.tube = self.conf.get('tube', 'rebuild')
self.beanstalk = Beanstalk.from_url(queue_url)
self.beanstalk.use(self.tube)
def __init__(self, conf, logger):
self.conf = conf
self.logger = logger
self.account = conf[CONF_ACCOUNT]
self.api = ObjectStorageApi(conf['namespace'], logger=self.logger)
self.container_client = self.api.container
self.account_client = self.api.account
self.passes = 0
self.errors = 0
self.last_reported = 0
self.contents_run_time = 0
self.total_contents_processed = 0
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_contents_per_second = int_value(
conf.get('contents_per_second'), 30)
self.container_fetch_limit = int_value(
conf.get('container_fetch_limit'), 100)
self.content_fetch_limit = int_value(conf.get('content_fetch_limit'),
100)
self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD),
9999999999)
self.new_policy = conf.get(CONF_NEW_POLICY)
def setUpClass(cls):
super(ItemRebuildTest, cls).setUpClass()
cls.api = ObjectStorageApi(cls._cls_ns, endpoint=cls._cls_uri)
print "%d containers in %fs, %f containers per second." % (
counter, now - checkpoint, counter / (now - checkpoint))
counter = 0
checkpoint = now
created.append(res)
now = time.time()
for coro in POOL.coroutines_running:
coro.kill()
while not RESULTS.empty():
created.append(RESULTS.get(block=False))
end = time.time()
rate = len(created) / (end - start)
print "End. %d containers created in %fs, %f containers per second." % (
len(created), end - start, rate)
print "Cleaning..."
for _ in POOL.starmap(API.container_delete,
[('benchmark', n) for n in created]):
pass
POOL.waitall()
return rate
if __name__ == '__main__':
import os
import sys
THREADS = int(sys.argv[1]) if len(sys.argv) > 1 else 1
API = ObjectStorageApi(os.getenv('OIO_NS', 'OPENIO'))
RESULTS = LightQueue(THREADS * 10)
POOL = GreenPool(THREADS)
main(THREADS)
def setUpClass(cls):
super(ObjectStoragePropertiesTest, cls).setUpClass()
cls.logger = logging.getLogger('test')
cls.api = ObjectStorageApi(cls._cls_ns, cls.logger)
cls.obj_cname = "obj_props_" + random_str(8)
def __init__(self, tool, queue_workers, queue_reply):
super(AccountRebuilderWorker, self).__init__(tool, queue_workers,
queue_reply)
self.api = ObjectStorageApi(self.tool.namespace, logger=self.logger)
def __init__(self, conf, logger, **kwargs):
super(MetaRebuilder, self).__init__(conf, logger, **kwargs)
self.api = ObjectStorageApi(self.conf['namespace'], logger=self.logger)
help='Start a worker connecter to the given addresses')
parser.add_argument("--controller",
dest="controller",
action='store_true',
default=False,
help='Start a controller bond to the given addresses')
parser.add_argument("endpoints",
metavar='ENDPOINT',
type=str,
nargs='+',
help='Endpoints to connect/bind to')
args = parser.parse_args()
zctx = zmq.Context()
if args.controller:
s = ObjectStorageApi("benchmark")
#Creating account
ac = AccountClient({"namespace": "benchmark"})
retry = 3
for i in range(retry + 1):
try:
ac.account_create("benchmark_account")
break
except ClientException:
if i < retry:
time.sleep(2)
else:
raise
#Creating Container