def _autocontainer_loop(self,
account,
marker=None,
limit=None,
concurrency=1,
**kwargs):
from functools import partial
autocontainer = self.app.client_manager.get_flatns_manager()
container_marker = autocontainer(marker) if marker else None
count = 0
kwargs['pool_manager'] = get_pool_manager(pool_maxsize=concurrency * 2)
# Start to list contents at 'marker' inside the last visited container
if container_marker:
for element in self._list_loop(account,
container_marker,
marker=marker,
**kwargs):
count += 1
yield element
if limit and count >= limit:
return
pool = GreenPool(concurrency)
self.account = account
self.autocontainer = autocontainer
for object_list in pool.imap(
partial(self._list_autocontainer_objects, **kwargs),
self._container_provider(account, marker=container_marker)):
for element in object_list:
count += 1
yield element
if limit and count >= limit:
return
def setUp(self):
super(BaseTestCase, self).setUp()
self.conf = get_config()
self.uri = 'http://' + self.conf['proxy']
self.ns = self.conf['namespace']
self.account = self.conf['account']
self.http_pool = get_pool_manager()
self._flush_cs('echo')
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception(
'Missing field "%s" in service configuration' % k)
self.name = '%s|%s|%s' % \
(service['type'], service['host'], service['port'])
self.service = service
self.rise = int_value(self._load_item_config('rise'), 1)
self.fall = int_value(self._load_item_config('fall'), 1)
self.check_interval = float_value(
self._load_item_config('check_interval'), 1)
self.deregister_on_exit = true_value(
self._load_item_config('deregister_on_exit', False))
self.logger = get_logger(self.conf)
self.pool_manager = get_pool_manager()
self.cs = ConscienceClient(self.conf, pool_manager=self.pool_manager,
logger=self.logger)
# FIXME: explain that
self.client = ProxyClient(self.conf, pool_manager=self.pool_manager,
no_ns_in_url=True, logger=self.logger)
self.last_status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}}
if self.service.get('location', None):
self.service_definition['tags']['tag.loc'] = \
self.service['location']
if self.service.get('slots', None):
self.service_definition['tags']['tag.slots'] = \
','.join(self.service['slots'])
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
def get_chunks_info(chunks):
pool_manager = get_pool_manager()
chunk_hash = ""
chunk_size = ""
for c in chunks:
resp = pool_manager.request('HEAD', c['url'])
if resp.status != 200:
chunk_size = "%d %s" % (resp.status, resp.reason)
chunk_hash = "%d %s" % (resp.status, resp.reason)
else:
chunk_size = resp.headers.get(
'X-oio-chunk-meta-chunk-size',
'Missing chunk size header')
chunk_hash = resp.headers.get(
'X-oio-chunk-meta-chunk-hash',
'Missing chunk hash header')
yield (c['pos'], c['url'], c['size'], c['hash'], chunk_size,
chunk_hash)
def __init__(self, endpoint=None, pool_manager=None, **kwargs):
"""
:param pool_manager: an optional pool manager that will be reused
:type pool_manager: `urllib3.PoolManager`
:param endpoint: base of the URL that will requested
:type endpoint: `str`
:keyword admin_mode: allow talking to a slave/worm namespace
:type admin_mode: `bool`
"""
super(HttpApi, self).__init__()
self.endpoint = endpoint
if not pool_manager:
pool_manager_conf = {
k: int(v)
for k, v in kwargs.iteritems()
if k in _POOL_MANAGER_OPTIONS_KEYS
}
pool_manager = get_pool_manager(**pool_manager_conf)
self.pool_manager = pool_manager
self.admin_mode = kwargs.get('admin_mode', False)
def run(args):
pool = get_pool_manager()
v = vars(args)
dirclient = DirectoryClient(v)
backend = AccountBackend(v)
for entry, _, _, partial in full_list(backend, prefix=args.prefix):
if partial:
if args.verbose:
print(":%s: partial, skip" % entry)
continue
try:
dirclient.show(account=ACCOUNT, reference=entry)
if args.verbose:
print("%s: OK" % entry)
continue
except NotFound:
pass
except Exception as exc:
print("Exception not managed for %s: %s" % (entry, str(exc)))
continue
print("%s: meta2 not found" % entry)
if args.dry_run:
continue
data = {"dtime": time(), "name": entry}
# post event to Account service
res = pool.request('POST',
HOST +
'/v1.0/account/container/update?id=%s' % ACCOUNT,
headers={'Content-Type': 'application/json'},
body=json.dumps(data))
if res.status_int / 100 != 2:
print(res.status)
def setUp(self):
super(TestRdirServer, self).setUp()
self.http_pool = get_pool_manager(max_retries=10)
self.num, self.db_path, self.host, self.port = self.get_service('rdir')
self.vol = self._volume()
def __init__(self):
self.http_pool = get_pool_manager()
def __init__(self, conf, log):
from oio.common.http import get_pool_manager
self.log = log
self.pool = get_pool_manager()
self.url_prefix = 'http://%s/v3.0/%s/admin/status?type=meta1&cid=' % (
conf['proxyd_url'], conf['namespace'])