def parse_get_node_args(options, args):
"""
Parse the get_nodes commandline args
:returns: a tuple, (ring_path, args)
"""
ring_path = None
if options.policy_name:
if POLICIES.get_by_name(options.policy_name) is None:
raise InfoSystemExit('No policy named %r' % options.policy_name)
elif args and args[0].endswith('.ring.gz'):
if os.path.exists(args[0]):
ring_path = args.pop(0)
else:
raise InfoSystemExit('Ring file does not exist')
if len(args) == 1:
args = args[0].strip('/').split('/', 2)
if not ring_path and not options.policy_name:
raise InfoSystemExit('Need to specify policy_name or <ring.gz>')
if not (args or options.partition):
raise InfoSystemExit('No target specified')
if len(args) > 3:
raise InfoSystemExit('Invalid arguments')
return ring_path, args
def parse_get_node_args(options, args):
"""
Parse the get_nodes commandline args
:returns: a tuple, (ring_path, args)
"""
ring_path = None
if options.policy_name:
if POLICIES.get_by_name(options.policy_name) is None:
raise InfoSystemExit('No policy named %r' % options.policy_name)
elif args and args[0].endswith('ring.gz'):
if os.path.exists(args[0]):
ring_path = args.pop(0)
else:
raise InfoSystemExit('Ring file does not exist')
if len(args) == 1:
args = args[0].strip('/').split('/', 2)
if not ring_path and not options.policy_name:
raise InfoSystemExit('Need to specify policy_name or <ring.gz>')
if not (args or options.partition):
raise InfoSystemExit('No target specified')
if len(args) > 3:
raise InfoSystemExit('Invalid arguments')
return ring_path, args
def test_defaults(self):
self.assertTrue(len(POLICIES) > 0)
# test class functions
default_policy = POLICIES.default
self.assert_(default_policy.is_default)
zero_policy = POLICIES.get_by_index(0)
self.assert_(zero_policy.idx == 0)
zero_policy_by_name = POLICIES.get_by_name(zero_policy.name)
self.assert_(zero_policy_by_name.idx == 0)
def test_defaults(self):
self.assertTrue(len(POLICIES) > 0)
# test class functions
default_policy = POLICIES.default
self.assertTrue(default_policy.is_default)
zero_policy = POLICIES.get_by_index(0)
self.assertTrue(zero_policy.idx == 0)
zero_policy_by_name = POLICIES.get_by_name(zero_policy.name)
self.assertTrue(zero_policy_by_name.idx == 0)
def _convert_policy_to_index(self, req):
"""
Helper method to convert a policy name (from a request from a client)
to a policy index (for a request to a backend).
:param req: incoming request
"""
policy_name = req.headers.get(POLICY)
if policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
return policy.idx
else:
raise HTTPBadRequest(request=req,
content_type="text/plain",
body=("Invalid X-Storage-Policy '%s'"
% policy_name))
def _convert_policy_to_index(self, req):
"""
Helper method to convert a policy name (from a request from a client)
to a policy index (for a request to a backend).
:param req: incoming request
"""
policy_name = req.headers.get("X-Storage-Policy")
if not policy_name:
return
policy = POLICIES.get_by_name(policy_name)
if not policy:
raise HTTPBadRequest(
request=req, content_type="text/plain", body=("Invalid %s '%s'" % ("X-Storage-Policy", policy_name))
)
if policy.is_deprecated:
body = "Storage Policy %r is deprecated" % (policy.name)
raise HTTPBadRequest(request=req, body=body)
return int(policy)
def _convert_policy_to_index(self, req):
"""
Helper method to convert a policy name (from a request from a client)
to a policy index (for a request to a backend).
:param req: incoming request
"""
policy_name = req.headers.get('X-Storage-Policy')
if not policy_name:
return
policy = POLICIES.get_by_name(policy_name)
if not policy:
raise HTTPBadRequest(request=req,
content_type="text/plain",
body=("Invalid %s '%s'"
% ('X-Storage-Policy', policy_name)))
if policy.is_deprecated:
body = 'Storage Policy %r is deprecated' % (policy.name)
raise HTTPBadRequest(request=req, body=body)
return int(policy)
def main():
options, commands = parser.parse_args()
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester',
'testing')
if options.server_type == 'object' and not options.policy_name:
options.policy_name = POLICIES.default.name
if options.policy_name:
options.server_type = 'object'
policy = POLICIES.get_by_name(options.policy_name)
if not policy:
return 'ERROR: unknown policy %r' % options.policy_name
else:
policy = None
brain = BrainSplitter(url,
token,
options.container,
options.object,
options.server_type,
policy=policy)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print '**WARNING**: %s raised %s' % (command, e)
print 'STATUS'.join(['*' * 25] * 2)
brain.servers.status()
sys.exit()
def main():
options, commands = parser.parse_args()
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
if options.server_type == 'object' and not options.policy_name:
options.policy_name = POLICIES.default.name
if options.policy_name:
options.server_type = 'object'
policy = POLICIES.get_by_name(options.policy_name)
if not policy:
return 'ERROR: unknown policy %r' % options.policy_name
else:
policy = None
brain = BrainSplitter(url, token, options.container, options.object,
options.server_type, policy=policy)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print '**WARNING**: %s raised %s' % (command, e)
print 'STATUS'.join(['*' * 25] * 2)
brain.servers.status()
sys.exit()
def print_item_locations(ring, ring_name=None, account=None, container=None,
obj=None, **kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get('policy_name', None)
part = kwargs.get('partition', None)
all_nodes = kwargs.get('all', False)
swift_dir = kwargs.get('swift_dir', '/etc/swift')
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print 'Attention! mismatch between ring and policy detected!'
else:
print 'Attention! Policy %s is not valid' % policy_name
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print 'Need a ring or policy'
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print 'No policy named %r' % policy_name
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print 'No account specified'
raise InfoSystemExit()
if container is None and obj is not None:
print 'No container specified'
raise InfoSystemExit()
if account is None and part is None:
print 'No target specified'
raise InfoSystemExit()
loc = '<type>'
if part and ring_name:
if '-' in ring_name and ring_name.startswith('object'):
loc = 'objects-' + ring_name.split('-', 1)[1]
else:
loc = ring_name + 's'
if account and container and obj:
loc = 'objects'
if '-' in ring_name and ring_name.startswith('object'):
policy_index = int(ring_name.rsplit('-', 1)[1])
loc = 'objects-%d' % policy_index
if account and container and not obj:
loc = 'containers'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='container')
else:
if ring_name != 'container':
print 'Attention! mismatch between ring and item detected!'
if account and not container and not obj:
loc = 'accounts'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='account')
else:
if ring_name != 'account':
print 'Attention! mismatch between ring and item detected!'
print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object \t%s\n\n' % obj
print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
policy_index=policy_index)
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
policy_name=''):
"""
Display information about an object read from the datafile.
Optionally verify the datafile content matches the ETag metadata.
:param datafile: path on disk to object file
:param check_etag: boolean, will read datafile content and verify
computed checksum matches value stored in
metadata.
:param swift_dir: the path on disk to rings
:param policy_name: optionally the name to use when finding the ring
"""
if not os.path.exists(datafile):
print "Data file doesn't exist"
raise InfoSystemExit()
if not datafile.startswith(('/', './')):
datafile = './' + datafile
policy_index = None
ring = None
datadir = DATADIR_BASE
# try to extract policy index from datafile disk path
policy_index = int(extract_policy(datafile) or POLICIES.legacy)
try:
if policy_index:
datadir += '-' + str(policy_index)
ring = Ring(swift_dir, ring_name='object-' + str(policy_index))
elif policy_index == 0:
ring = Ring(swift_dir, ring_name='object')
except IOError:
# no such ring
pass
if policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
policy_index_for_name = policy.idx
if (policy_index is not None and
policy_index_for_name is not None and
policy_index != policy_index_for_name):
print 'Attention: Ring does not match policy!'
print 'Double check your policy name!'
if not ring and policy_index_for_name:
ring = POLICIES.get_object_ring(policy_index_for_name,
swift_dir)
datadir = get_data_dir(policy_index_for_name)
with open(datafile, 'rb') as fp:
try:
metadata = read_metadata(fp)
except EOFError:
print "Invalid metadata"
raise InfoSystemExit()
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
path = metadata.get('name', '')
print_obj_metadata(metadata)
# Optional integrity check; it's useful, but slow.
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print 'ETag: %s (valid)' % etag
else:
print ("ETag: %s doesn't match file hash of %s!" %
(etag, h))
else:
print 'ETag: Not found in metadata'
else:
print 'ETag: %s (not checked)' % etag
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print 'Content-Length: %s (valid)' % length
else:
print ("Content-Length: %s doesn't match file length of %s"
% (length, file_len))
else:
print 'Content-Length: Not found in metadata'
account, container, obj = path.split('/', 3)[1:]
if ring:
print_ring_locations(ring, datadir, account, container, obj,
policy_index=policy_index)
def generate_report(conf, policy_name=None):
global json_output
json_output = config_true_value(conf.get('dump_json', 'no'))
if policy_name is None:
policy = POLICIES.default
else:
policy = POLICIES.get_by_name(policy_name)
if policy is None:
exit('Unable to find policy: %s' % policy_name)
if not json_output:
print('Using storage policy: %s ' % policy.name)
swift_dir = conf.get('swift_dir', '/etc/swift')
retries = int(conf.get('retries', 5))
concurrency = int(conf.get('concurrency', 25))
endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
region_name = str(conf.get('region_name', ''))
container_report = config_true_value(conf.get('container_report', 'yes'))
object_report = config_true_value(conf.get('object_report', 'yes'))
if not (object_report or container_report):
exit("Neither container or object report is set to run")
user_domain_name = str(conf.get('user_domain_name', ''))
project_domain_name = str(conf.get('project_domain_name', ''))
project_name = str(conf.get('project_name', ''))
insecure = config_true_value(conf.get('keystone_api_insecure', 'no'))
coropool = GreenPool(size=concurrency)
os_options = {'endpoint_type': endpoint_type}
if user_domain_name:
os_options['user_domain_name'] = user_domain_name
if project_domain_name:
os_options['project_domain_name'] = project_domain_name
if project_name:
os_options['project_name'] = project_name
if region_name:
os_options['region_name'] = region_name
url, token = get_auth(conf['auth_url'], conf['auth_user'],
conf['auth_key'],
auth_version=conf.get('auth_version', '1.0'),
os_options=os_options,
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name=policy.ring_name)
output = {}
if container_report:
output['container'] = container_dispersion_report(
coropool, connpool, account, container_ring, retries,
conf.get('partitions'), policy)
if object_report:
output['object'] = object_dispersion_report(
coropool, connpool, account, object_ring, retries,
conf.get('partitions'), policy)
return output
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get("policy_name", None)
part = kwargs.get("partition", None)
all_nodes = kwargs.get("all", False)
swift_dir = kwargs.get("swift_dir", "/etc/swift")
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print "Attention! mismatch between ring and policy detected!"
else:
print "Attention! Policy %s is not valid" % policy_name
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print "Need a ring or policy"
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print "No policy named %r" % policy_name
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print "No account specified"
raise InfoSystemExit()
if container is None and obj is not None:
print "No container specified"
raise InfoSystemExit()
if account is None and part is None:
print "No target specified"
raise InfoSystemExit()
loc = "<type>"
if part and ring_name:
if "-" in ring_name and ring_name.startswith("object"):
loc = "objects-" + ring_name.split("-", 1)[1]
else:
loc = ring_name + "s"
if account and container and obj:
loc = "objects"
if "-" in ring_name and ring_name.startswith("object"):
policy_index = int(ring_name.rsplit("-", 1)[1])
loc = "objects-%d" % policy_index
if account and container and not obj:
loc = "containers"
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name="container")
else:
if ring_name != "container":
print "Attention! mismatch between ring and item detected!"
if account and not container and not obj:
loc = "accounts"
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name="account")
else:
if ring_name != "account":
print "Attention! mismatch between ring and item detected!"
print "\nAccount \t%s" % account
print "Container\t%s" % container
print "Object \t%s\n\n" % obj
print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def generate_report(conf, policy_name=None):
global json_output
json_output = config_true_value(conf.get('dump_json', 'no'))
if policy_name is None:
policy = POLICIES.default
else:
policy = POLICIES.get_by_name(policy_name)
if policy is None:
exit('Unable to find policy: %s' % policy_name)
if not json_output:
print('Using storage policy: %s ' % policy.name)
swift_dir = conf.get('swift_dir', '/etc/swift')
retries = int(conf.get('retries', 5))
concurrency = int(conf.get('concurrency', 25))
endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
region_name = str(conf.get('region_name', ''))
container_report = config_true_value(conf.get('container_report', 'yes'))
object_report = config_true_value(conf.get('object_report', 'yes'))
if not (object_report or container_report):
exit("Neither container or object report is set to run")
user_domain_name = str(conf.get('user_domain_name', ''))
project_domain_name = str(conf.get('project_domain_name', ''))
project_name = str(conf.get('project_name', ''))
insecure = config_true_value(conf.get('keystone_api_insecure', 'no'))
coropool = GreenPool(size=concurrency)
os_options = {'endpoint_type': endpoint_type}
if user_domain_name:
os_options['user_domain_name'] = user_domain_name
if project_domain_name:
os_options['project_domain_name'] = project_domain_name
if project_name:
os_options['project_name'] = project_name
if region_name:
os_options['region_name'] = region_name
url, token = get_auth(conf['auth_url'],
conf['auth_user'],
conf['auth_key'],
auth_version=conf.get('auth_version', '1.0'),
os_options=os_options,
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name=policy.ring_name)
output = {}
if container_report:
output['container'] = container_dispersion_report(
coropool, connpool, account, container_ring, retries,
conf.get('partitions'), policy)
if object_report:
output['object'] = object_dispersion_report(coropool, connpool,
account, object_ring,
retries,
conf.get('partitions'),
policy)
return output
