def __init__(self, conf, logger=None, account_ring=None, container_ring=None, storage=None): for policy, ring_arg in zip(OIO_POLICIES, RING_ARGS): if ring_arg is not None: policy.object_ring = FakeRing(**ring_arg) SwiftApplication.__init__(self, conf, logger=logger, account_ring=account_ring, container_ring=container_ring) if conf is None: conf = dict() sds_conf = {k[4:]: v for k, v in conf.items() if k.startswith("sds_")} self.oio_stgpol = [] if 'auto_storage_policies' in conf: for elem in conf['auto_storage_policies'].split(','): if ':' in elem: name, offset = elem.split(':') self.oio_stgpol.append((name, int(offset))) else: self.oio_stgpol.append((elem, 0)) self.oio_stgpol.sort(key=lambda x: x[1]) policies = [] if 'oio_storage_policies' in conf: for i, pol in enumerate(conf['oio_storage_policies'].split(',')): policies.append( storage_policy.StoragePolicy(i, pol, is_default=i == 0)) else: policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True)) self.POLICIES = storage_policy.StoragePolicyCollection(policies) # Mandatory, raises KeyError sds_namespace = sds_conf['namespace'] sds_conf.pop('namespace') # removed to avoid unpacking conflict # Loaded by ObjectStorageApi if None sds_proxy_url = sds_conf.pop('proxy_url', None) # Fix boolean parameter if 'autocreate' in sds_conf and not ( hasattr(ObjectStorageApi, 'EXTRA_KEYWORDS') or 'autocreate' in ObjectStorageApi.EXTRA_KEYWORDS): logger.warn("'autocreate' parameter is ignored by current version" " of OpenIO SDS. Please update to oio>=4.1.23.") else: sds_conf['autocreate'] = config_true_value( sds_conf.get('autocreate', 'true')) self.storage = storage or \ ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf) self.delete_slo_parts = \ config_true_value(conf.get('delete_slo_parts', False)) self.check_state = \ config_true_value(conf.get('check_state', False))
def setUp(self): super(TestContainerLifecycle, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_lifecycle" self.container = "lifecycle-" + random_str(4) self.lifecycle = ContainerLifecycle( self.api, self.account, self.container)
def setUp(self): super(TestAdmin, self).setUp() self.admin = AdminClient(self.conf) self.api = ObjectStorageApi(self.ns) self.account = "test_admin" self.container = "admin-" + random_str(4) self.api.container_create(self.account, self.container)
def setUp(self): super(TestMeta1RefMapping, self).setUp() self.api = ObjectStorageApi(self.ns) self.account = "test_refmapping" self.reference = "refmapping-" + random_str(4) self.logger = logging.getLogger('test') self.mapping = Meta1RefMapping(self.ns, logger=self.logger)
def test_iter_container_list(self): worker = StorageTiererWorker(self.gridconf, Mock()) api = ObjectStorageApi(self.namespace) actual = [x[0] for x in api.container_list(self.test_account)] if len(actual) < 3: print "Slow event propagation!" # account events have not yet propagated time.sleep(3.0) actual = [x[0] for x in api.container_list(self.test_account)[0]] gen = worker._list_containers() self.assertListEqual(list(gen), actual)
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, storage=None): for policy, ring_arg in zip(POLICIES, ring_args): if ring_arg is not None: policy.object_ring = FakeRing(**ring_arg) SwiftApplication.__init__(self, conf, memcache=memcache, logger=logger, account_ring=account_ring, container_ring=container_ring) if conf is None: conf = dict() sds_conf = { k[4:]: v for k, v in conf.iteritems() if k.startswith("sds_") } self.oio_stgpol = [] if 'auto_storage_policies' in conf: for elem in conf['auto_storage_policies'].split(','): if ':' in elem: name, offset = elem.split(':') self.oio_stgpol.append((name, int(offset))) else: self.oio_stgpol.append((elem, 0)) self.oio_stgpol.sort(key=lambda x: x[1]) policies = [] if 'oio_storage_policies' in conf: for i, pol in enumerate(conf['oio_storage_policies'].split(',')): policies.append( storage_policy.StoragePolicy(i, pol, is_default=i == 0)) else: policies.append(storage_policy.StoragePolicy(0, 'SINGLE', True)) self.POLICIES = storage_policy.StoragePolicyCollection(policies) # Mandatory, raises KeyError sds_namespace = sds_conf['namespace'] sds_conf.pop('namespace') # removed to avoid unpacking conflict # Loaded by ObjectStorageApi if None sds_proxy_url = sds_conf.pop('proxy_url', None) self.storage = storage or \ ObjectStorageApi(sds_namespace, endpoint=sds_proxy_url, **sds_conf)
def setUp(self): super(TestStorageTierer, self).setUp() self.namespace = self.conf['namespace'] self.test_account = "test_storage_tiering_%f" % time.time() self.api = ObjectStorageApi(self.namespace) self.gridconf = { "namespace": self.namespace, "container_fetch_limit": 2, "content_fetch_limit": 2, "account": self.test_account, "outdated_threshold": 0, "new_policy": "EC" } self.api.container = ContainerClient(self.gridconf) self._populate()
def setUp(self): super(TestContentRebuildFilter, self).setUp() self.namespace = self.conf['namespace'] self.gridconf = {"namespace": self.namespace} self.container = "TestContentRebuildFilter%f" % time.time() self.ref = self.container self.container_client = ContainerClient(self.conf) self.container_client.container_create(self.account, self.container) syst = self.container_client.container_get_properties( self.account, self.container)['system'] self.container_id = syst['sys.name'].split('.', 1)[0] self.object_storage_api = ObjectStorageApi(namespace=self.namespace) self.stgpol = "SINGLE" self.conf['tube'] = 'rebuild' self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300' self.notify_filter = NotifyFilter(app=_App, conf=self.conf) queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300') self.tube = self.conf.get('tube', 'rebuild') self.beanstalk = Beanstalk.from_url(queue_url) self.beanstalk.use(self.tube)
def __init__(self, conf, logger): self.conf = conf self.logger = logger self.account = conf[CONF_ACCOUNT] self.api = ObjectStorageApi(conf['namespace'], logger=self.logger) self.container_client = self.api.container self.account_client = self.api.account self.passes = 0 self.errors = 0 self.last_reported = 0 self.contents_run_time = 0 self.total_contents_processed = 0 self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_contents_per_second = int_value( conf.get('contents_per_second'), 30) self.container_fetch_limit = int_value( conf.get('container_fetch_limit'), 100) self.content_fetch_limit = int_value(conf.get('content_fetch_limit'), 100) self.outdated_threshold = int_value(conf.get(CONF_OUTDATED_THRESHOLD), 9999999999) self.new_policy = conf.get(CONF_NEW_POLICY)
def setUpClass(cls): super(ItemRebuildTest, cls).setUpClass() cls.api = ObjectStorageApi(cls._cls_ns, endpoint=cls._cls_uri)
print "%d containers in %fs, %f containers per second." % ( counter, now - checkpoint, counter / (now - checkpoint)) counter = 0 checkpoint = now created.append(res) now = time.time() for coro in POOL.coroutines_running: coro.kill() while not RESULTS.empty(): created.append(RESULTS.get(block=False)) end = time.time() rate = len(created) / (end - start) print "End. %d containers created in %fs, %f containers per second." % ( len(created), end - start, rate) print "Cleaning..." for _ in POOL.starmap(API.container_delete, [('benchmark', n) for n in created]): pass POOL.waitall() return rate if __name__ == '__main__': import os import sys THREADS = int(sys.argv[1]) if len(sys.argv) > 1 else 1 API = ObjectStorageApi(os.getenv('OIO_NS', 'OPENIO')) RESULTS = LightQueue(THREADS * 10) POOL = GreenPool(THREADS) main(THREADS)
def setUpClass(cls): super(ObjectStoragePropertiesTest, cls).setUpClass() cls.logger = logging.getLogger('test') cls.api = ObjectStorageApi(cls._cls_ns, cls.logger) cls.obj_cname = "obj_props_" + random_str(8)
def __init__(self, tool, queue_workers, queue_reply): super(AccountRebuilderWorker, self).__init__(tool, queue_workers, queue_reply) self.api = ObjectStorageApi(self.tool.namespace, logger=self.logger)
def __init__(self, conf, logger, **kwargs): super(MetaRebuilder, self).__init__(conf, logger, **kwargs) self.api = ObjectStorageApi(self.conf['namespace'], logger=self.logger)
help='Start a worker connecter to the given addresses') parser.add_argument("--controller", dest="controller", action='store_true', default=False, help='Start a controller bond to the given addresses') parser.add_argument("endpoints", metavar='ENDPOINT', type=str, nargs='+', help='Endpoints to connect/bind to') args = parser.parse_args() zctx = zmq.Context() if args.controller: s = ObjectStorageApi("benchmark") #Creating account ac = AccountClient({"namespace": "benchmark"}) retry = 3 for i in range(retry + 1): try: ac.account_create("benchmark_account") break except ClientException: if i < retry: time.sleep(2) else: raise #Creating Container