def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
namespace = self.app.client_manager.admin.conf['namespace']
sds_conf = load_namespace_conf(namespace)
output = list()
for k in sds_conf:
output.append(("%s/%s" % (namespace, k), sds_conf[k]))
return zip(*output)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
namespace = self.app.client_manager.admin.conf['namespace']
sds_conf = load_namespace_conf(namespace)
output = list()
for k in sds_conf:
output.append(("%s/%s" % (namespace, k), sds_conf[k]))
return zip(*output)
def __init__(self, namespace, endpoint=None, **kwargs):
if not endpoint:
endpoint = utils.load_namespace_conf(namespace)["proxy"]
if not endpoint.startswith("http://"):
endpoint = "http://" + endpoint
endpoint_v3 = "/".join([endpoint.rstrip("/"), "v3.0"])
super(ObjectStorageAPI, self).__init__(endpoint=endpoint_v3, **kwargs)
self.directory = DirectoryAPI(namespace, endpoint, session=self.session, admin_mode=self.admin_mode)
self.namespace = namespace
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option(
'-v', '--verbose',
action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-s', '--smart',
action="store_true", dest="SMART", default=False,
help="Delete onle the members belong to services with multiple" \
" members")
parser.add_option(
'-d', '--dry-run',
action="store_true", dest="DRY", default=False,
help="Do not delete, just print")
parser.add_option(
'-n', '--min-services',
type=int, action="store", dest="NUM", default=4,
help="Do not delete election if less the NUM")
parser.add_option(
'-1', '--alone',
action="store_true", dest="ALONE", default=False,
help="Also consider members alone in their group")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
logging.debug(">DIR %s", group)
for node in list_problematic_nodes(zh, group, options):
delete_node(zh, node, options)
zookeeper.close(zh)
def __init__(self, conf, **kwargs):
super(Client, self).__init__()
validate_service_conf(conf)
self.ns = conf.get("namespace")
ns_conf = load_namespace_conf(self.ns)
self.conf = conf
self.ns_conf = ns_conf
self.logger = get_logger(conf)
self.session = requests.Session()
self.endpoint = "http://%s" % ns_conf.get("proxy")
def __init__(self, conf, session=None, **kwargs):
super(Client, self).__init__()
validate_service_conf(conf)
self.ns = conf.get('namespace')
ns_conf = load_namespace_conf(self.ns)
self.conf = conf
self.ns_conf = ns_conf
self.logger = get_logger(conf)
self.session = session or requests.Session()
self.endpoint = 'http://%s' % ns_conf.get('proxy')
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-c',
'--min-children',
type=int,
action="store",
dest="CHILDREN",
default=15,
help="Do not print the children number if less than that value")
parser.add_option('-s',
'--min-size',
type=int,
action="store",
dest="SIZE",
default=0,
help="Do not print unless the size is over that value")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
for child, meta in list_nodes(zh, group, options):
if meta['dataLength'] > options.SIZE:
print "NODE", meta['dataLength'], child
zookeeper.close(zh)
def __init__(self, namespace, endpoint=None, **kwargs):
if not endpoint:
endpoint = utils.load_namespace_conf(namespace)['proxy']
if not endpoint.startswith('http://'):
endpoint = 'http://' + endpoint
endpoint_v3 = '/'.join([endpoint.rstrip('/'), 'v3.0'])
super(ObjectStorageAPI, self).__init__(endpoint=endpoint_v3, **kwargs)
self.directory = DirectoryAPI(
namespace,
endpoint,
session=self.session
)
self.namespace = namespace
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option(
'-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
help='Only play with small batches to avoid timeouts on slow hosts.')
parser.add_option(
'--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Do not populate entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def __init__(self, namespace, endpoint=None, **kwargs):
if not endpoint:
endpoint = utils.load_namespace_conf(namespace)['proxy']
if not endpoint.startswith('http://'):
endpoint = 'http://' + endpoint
endpoint_v3 = '/'.join([endpoint.rstrip('/'), 'v3.0'])
super(ObjectStorageAPI, self).__init__(endpoint=endpoint_v3, **kwargs)
self.directory = DirectoryAPI(
namespace,
endpoint,
session=self.session,
admin_mode=self.admin_mode
)
self.namespace = namespace
def setup(self):
if not self.setup_done:
if not self._options.get('namespace', None):
msg = 'Set a namespace with --oio-ns, OIO_NS\n'
raise exceptions.CommandError('Missing parameter: \n%s' % msg)
self.namespace = self._options['namespace']
sds_conf = load_namespace_conf(self.namespace) or {}
if not self._options.get('proxyd_url') and 'proxy' in sds_conf:
proxyd_url = 'http://%s' % sds_conf.get('proxy')
self._options['proxyd_url'] = proxyd_url
validate_options(self._options)
LOG.info('Using parameters %s' % self._options)
self.session = requests.Session()
self.setup_done = True
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option('-d',
'--data',
action="store",
dest="VALUE",
default='',
help="Data to force")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
logging.debug(">DIR %s", group)
for mom, key, first, last in list_groups(zh, group, options):
tail = str(1 + int(last)).rjust(10, '0')
path = mom + '/' + key + '-' + tail
create_node(zh, options, path)
logging.debug("Please send a signal to remove the ephemeral nodes")
logging.warn("PAUSED ...")
signal.pause()
zookeeper.close(zh)
def __init__(self,
conf,
pool_manager=None,
request_prefix="",
no_ns_in_url=False,
endpoint=None,
request_attempts=REQUEST_ATTEMPTS,
logger=None,
**kwargs):
"""
:param pool_manager: an optional pool manager that will be reused
:type pool_manager: `urllib3.PoolManager`
:param request_prefix: text to insert in between endpoint and
requested URL
:type request_prefix: `str`
:param no_ns_in_url: do not insert namespace name between endpoint
and `request_prefix`
:type no_ns_in_url: `bool`
:param request_attempts: number of attempts for the request in case of
error 503
:raise oio.common.exceptions.ServiceBusy: if all attempts fail
"""
assert (request_attempts > 0)
validate_service_conf(conf)
self.ns = conf.get('namespace')
self.conf = conf
self.logger = logger or get_logger(conf)
ep_parts = list()
if endpoint:
self.proxy_netloc = endpoint.lstrip("http://")
else:
ns_conf = load_namespace_conf(self.ns)
self.proxy_netloc = ns_conf.get('proxy')
ep_parts.append("http:/")
ep_parts.append(self.proxy_netloc)
ep_parts.append("v3.0")
if not no_ns_in_url:
ep_parts.append(self.ns)
if request_prefix:
ep_parts.append(request_prefix.lstrip('/'))
self._request_attempts = request_attempts
super(ProxyClient, self).__init__(endpoint='/'.join(ep_parts),
**kwargs)
def setup(self):
if not self.setup_done:
if not self._options.get('namespace', None):
msg = 'Set a namespace with --oio-ns, OIO_NS\n'
raise exceptions.CommandError('Missing parameter: \n%s' % msg)
self.namespace = self._options['namespace']
sds_conf = load_namespace_conf(self.namespace) or {}
if not self._options.get('proxyd_url') and 'proxy' in sds_conf:
proxyd_url = 'http://%s' % sds_conf.get('proxy')
self._options['proxyd_url'] = proxyd_url
validate_options(self._options)
LOG.debug('Using parameters %s' % self._options)
self.session = requests.Session()
self.setup_done = True
self._admin_mode = self._options.get('admin_mode')
def collect(self):
namespaces = self.namespaces
# Convert a string config value to be an array
if isinstance(namespaces, basestring):
namespaces = [namespaces]
http = urllib3.PoolManager()
for ns in namespaces:
config = utils.load_namespace_conf(ns)
if not config:
self.log.error('No configuration found for namespace ' + ns)
continue
proxy = config['proxy']
self.get_stats(http, ns, proxy)
def collect(self):
namespaces = self.namespaces
# Convert a string config value to be an array
if isinstance(namespaces, basestring):
namespaces = [namespaces]
http = urllib3.PoolManager()
for ns in namespaces:
config = load_namespace_conf(ns)
if not config:
self.log.error('No configuration found for namespace ' + ns)
continue
proxy = config['proxy']
self.get_stats(http, ns, proxy)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
help='Only play with small batches to avoid timeouts on slow hosts.')
parser.add_option(
'--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Do not populate entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def setup(self):
if not self.setup_done:
if not self._options.get('namespace', None):
msg = 'Set a namespace with --oio-ns, OIO_NS\n'
raise CommandError('Missing parameter: \n%s' % msg)
self.namespace = self._options['namespace']
sds_conf = load_namespace_conf(self.namespace) or {}
if not self._options.get('proxyd_url') and 'proxy' in sds_conf:
proxyd_url = 'http://%s' % sds_conf.get('proxy')
self._options['proxyd_url'] = proxyd_url
validate_options(self._options)
LOG.debug('Using parameters %s' % self._options)
self.setup_done = True
self._admin_mode = self._options.get('admin_mode')
if 'meta1_digits' in sds_conf:
self._meta1_digits = int(sds_conf["meta1_digits"])
self._options['log_level'] = \
getLevelName(LOG.getEffectiveLevel())
def main():
usage = "usage: %prog [options] NS"
from optparse import OptionParser as OptionParser
parser = OptionParser(usage=usage)
parser.add_option(
'-v', '--verbose',
action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-a', '--all',
action="store_true", dest="flag_all",
help='Remove all oio-sds nodes (not only meta0)')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
for shard in cnxstr.split(";"):
logging.info("ZK=%s", shard)
zh = zookeeper.init(shard)
if options.flag_all:
logging.warn("FLUSHING all the oio-sds entries in the ZK server")
delete_children(zh, "/hc")
else:
logging.info("Cleaning only the meta0 registrations in ZK server")
delete_children(zh, "/hc/ns/"+ns+"/srv/meta0")
zookeeper.close(zh)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS SRVTYPE [SRVTYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
children = list(list_nodes(zh, group, options))
if len(children) > 0:
logging.info("> %s", group)
for k in children:
data, meta = zookeeper.get(zh, group + '/' + k)
logging.info(" %s : %s", k, data)
zookeeper.close(zh)
def __init__(self,
conf,
session=None,
request_prefix="",
no_ns_in_url=False,
endpoint=None,
**kwargs):
"""
:param session: an optional session that will be reused
:type session: `requests.Session`
:param request_prefix: text to insert in between endpoint and
requested URL
:type request_prefix: `str`
:param no_ns_in_url: do not insert namespace name between endpoint
and `request_prefix`
:type no_ns_in_url: `bool`
"""
validate_service_conf(conf)
self.ns = conf.get('namespace')
self.conf = conf
self.logger = get_logger(conf)
ep_parts = list()
if endpoint:
self.proxy_netloc = endpoint[7:] # skip "http://"
ep_parts.append(endpoint)
else:
ns_conf = load_namespace_conf(self.ns)
self.proxy_netloc = ns_conf.get('proxy')
ep_parts.append("http:/")
ep_parts.append(self.proxy_netloc)
ep_parts.append("v3.0")
if not no_ns_in_url:
ep_parts.append(self.ns)
if request_prefix:
ep_parts.append(request_prefix.lstrip('/'))
super(ProxyClient, self).__init__(endpoint='/'.join(ep_parts),
**kwargs)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v', '--verbose', action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option('-a', '--all', action="store_true", dest="flag_all",
help='Triggers debugging traces')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
if options.flag_all:
logging.warn("FLUSHING all the oio-sds entries in the ZK server");
delete_children(zh, "/hc")
else:
logging.info("Cleaning only the meta0 registrations in ZK server");
delete_children(zh, "/hc/ns/"+ns+"/srv/meta0")
zookeeper.close(zh)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
default=False,
help='Triggers debugging traces')
parser.add_option('--lazy',
action="store_true",
dest="LAZY",
default=False,
help='Quickly check if things seem OK.')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
default=False,
help='Send small batches to avoid timeouts on slow hosts.')
parser.add_option('--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Avoid entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
for shard in cnxstr.split(";"):
logging.info("ZK=%s", shard)
zh = zookeeper.init(shard)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
missing = True
if options.LAZY:
_m = False
for t, _, _ in SRVTYPES:
try:
_, _ = zookeeper.get(zh, PREFIX_NS + '/' + ns + '/el/' + t)
except:
_m = True
missing = _m
if missing:
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def __init__(self, conf, **kwargs):
self.ns_conf = load_namespace_conf(conf["namespace"])
self.queue_url = self.ns_conf['event-agent']
self._beanstalk = None