def parse_get_node_args(options, args): """ Parse the get_nodes commandline args :returns: a tuple, (ring_path, args) """ ring_path = None if options.policy_name: if POLICIES.get_by_name(options.policy_name) is None: raise InfoSystemExit('No policy named %r' % options.policy_name) elif args and args[0].endswith('.ring.gz'): if os.path.exists(args[0]): ring_path = args.pop(0) else: raise InfoSystemExit('Ring file does not exist') if len(args) == 1: args = args[0].strip('/').split('/', 2) if not ring_path and not options.policy_name: raise InfoSystemExit('Need to specify policy_name or <ring.gz>') if not (args or options.partition): raise InfoSystemExit('No target specified') if len(args) > 3: raise InfoSystemExit('Invalid arguments') return ring_path, args
def parse_get_node_args(options, args): """ Parse the get_nodes commandline args :returns: a tuple, (ring_path, args) """ ring_path = None if options.policy_name: if POLICIES.get_by_name(options.policy_name) is None: raise InfoSystemExit('No policy named %r' % options.policy_name) elif args and args[0].endswith('ring.gz'): if os.path.exists(args[0]): ring_path = args.pop(0) else: raise InfoSystemExit('Ring file does not exist') if len(args) == 1: args = args[0].strip('/').split('/', 2) if not ring_path and not options.policy_name: raise InfoSystemExit('Need to specify policy_name or <ring.gz>') if not (args or options.partition): raise InfoSystemExit('No target specified') if len(args) > 3: raise InfoSystemExit('Invalid arguments') return ring_path, args
def test_defaults(self): self.assertTrue(len(POLICIES) > 0) # test class functions default_policy = POLICIES.default self.assert_(default_policy.is_default) zero_policy = POLICIES.get_by_index(0) self.assert_(zero_policy.idx == 0) zero_policy_by_name = POLICIES.get_by_name(zero_policy.name) self.assert_(zero_policy_by_name.idx == 0)
def test_defaults(self): self.assertTrue(len(POLICIES) > 0) # test class functions default_policy = POLICIES.default self.assertTrue(default_policy.is_default) zero_policy = POLICIES.get_by_index(0) self.assertTrue(zero_policy.idx == 0) zero_policy_by_name = POLICIES.get_by_name(zero_policy.name) self.assertTrue(zero_policy_by_name.idx == 0)
def _convert_policy_to_index(self, req): """ Helper method to convert a policy name (from a request from a client) to a policy index (for a request to a backend). :param req: incoming request """ policy_name = req.headers.get(POLICY) if policy_name: policy = POLICIES.get_by_name(policy_name) if policy: return policy.idx else: raise HTTPBadRequest(request=req, content_type="text/plain", body=("Invalid X-Storage-Policy '%s'" % policy_name))
def _convert_policy_to_index(self, req): """ Helper method to convert a policy name (from a request from a client) to a policy index (for a request to a backend). :param req: incoming request """ policy_name = req.headers.get("X-Storage-Policy") if not policy_name: return policy = POLICIES.get_by_name(policy_name) if not policy: raise HTTPBadRequest( request=req, content_type="text/plain", body=("Invalid %s '%s'" % ("X-Storage-Policy", policy_name)) ) if policy.is_deprecated: body = "Storage Policy %r is deprecated" % (policy.name) raise HTTPBadRequest(request=req, body=body) return int(policy)
def _convert_policy_to_index(self, req): """ Helper method to convert a policy name (from a request from a client) to a policy index (for a request to a backend). :param req: incoming request """ policy_name = req.headers.get('X-Storage-Policy') if not policy_name: return policy = POLICIES.get_by_name(policy_name) if not policy: raise HTTPBadRequest(request=req, content_type="text/plain", body=("Invalid %s '%s'" % ('X-Storage-Policy', policy_name))) if policy.is_deprecated: body = 'Storage Policy %r is deprecated' % (policy.name) raise HTTPBadRequest(request=req, body=body) return int(policy)
def main(): options, commands = parser.parse_args() if not commands: parser.print_help() return 'ERROR: must specify at least one command' for cmd_args in commands: cmd = cmd_args.split(':', 1)[0] if cmd not in BrainSplitter.__commands__: parser.print_help() return 'ERROR: unknown command %s' % cmd url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') if options.server_type == 'object' and not options.policy_name: options.policy_name = POLICIES.default.name if options.policy_name: options.server_type = 'object' policy = POLICIES.get_by_name(options.policy_name) if not policy: return 'ERROR: unknown policy %r' % options.policy_name else: policy = None brain = BrainSplitter(url, token, options.container, options.object, options.server_type, policy=policy) for cmd_args in commands: parts = cmd_args.split(':', 1) command = parts[0] if len(parts) > 1: args = utils.list_from_csv(parts[1]) else: args = () try: brain.run(command, *args) except ClientException as e: print '**WARNING**: %s raised %s' % (command, e) print 'STATUS'.join(['*' * 25] * 2) brain.servers.status() sys.exit()
def main(): options, commands = parser.parse_args() if not commands: parser.print_help() return 'ERROR: must specify at least one command' for cmd_args in commands: cmd = cmd_args.split(':', 1)[0] if cmd not in BrainSplitter.__commands__: parser.print_help() return 'ERROR: unknown command %s' % cmd url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') if options.server_type == 'object' and not options.policy_name: options.policy_name = POLICIES.default.name if options.policy_name: options.server_type = 'object' policy = POLICIES.get_by_name(options.policy_name) if not policy: return 'ERROR: unknown policy %r' % options.policy_name else: policy = None brain = BrainSplitter(url, token, options.container, options.object, options.server_type, policy=policy) for cmd_args in commands: parts = cmd_args.split(':', 1) command = parts[0] if len(parts) > 1: args = utils.list_from_csv(parts[1]) else: args = () try: brain.run(command, *args) except ClientException as e: print '**WARNING**: %s raised %s' % (command, e) print 'STATUS'.join(['*' * 25] * 2) brain.servers.status() sys.exit()
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs): """ Display placement information for an item based on ring lookup. If a ring is provided it always takes precedence, but warnings will be emitted if it doesn't match other optional arguments like the policy_name or ring_name. If no ring is provided the ring_name and/or policy_name will be used to lookup the ring. :param ring: a ring instance :param ring_name: server type, or storage policy ring name if object ring :param account: account name :param container: container name :param obj: object name :param partition: part number for non path lookups :param policy_name: name of storage policy to use to lookup the ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. """ policy_name = kwargs.get('policy_name', None) part = kwargs.get('partition', None) all_nodes = kwargs.get('all', False) swift_dir = kwargs.get('swift_dir', '/etc/swift') if ring and policy_name: policy = POLICIES.get_by_name(policy_name) if policy: if ring_name != policy.ring_name: print 'Attention! mismatch between ring and policy detected!' else: print 'Attention! Policy %s is not valid' % policy_name policy_index = None if ring is None and (obj or part): if not policy_name: print 'Need a ring or policy' raise InfoSystemExit() policy = POLICIES.get_by_name(policy_name) if not policy: print 'No policy named %r' % policy_name raise InfoSystemExit() policy_index = int(policy) ring = POLICIES.get_object_ring(policy_index, swift_dir) ring_name = (POLICIES.get_by_name(policy_name)).ring_name if account is None and (container is not None or obj is not None): print 'No account specified' raise InfoSystemExit() if container is None and obj is not None: print 'No container specified' raise InfoSystemExit() if account is None and part is None: print 'No target specified' raise InfoSystemExit() loc = '<type>' if part and ring_name: if '-' in ring_name and ring_name.startswith('object'): loc = 'objects-' + ring_name.split('-', 1)[1] else: loc = ring_name + 's' if account and container and obj: loc = 'objects' if '-' in ring_name and ring_name.startswith('object'): policy_index = int(ring_name.rsplit('-', 1)[1]) loc = 'objects-%d' % policy_index if account and container and not obj: loc = 'containers' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='container') else: if ring_name != 'container': print 'Attention! mismatch between ring and item detected!' if account and not container and not obj: loc = 'accounts' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='account') else: if ring_name != 'account': print 'Attention! mismatch between ring and item detected!' print '\nAccount \t%s' % account print 'Container\t%s' % container print 'Object \t%s\n\n' % obj print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', policy_name=''): """ Display information about an object read from the datafile. Optionally verify the datafile content matches the ETag metadata. :param datafile: path on disk to object file :param check_etag: boolean, will read datafile content and verify computed checksum matches value stored in metadata. :param swift_dir: the path on disk to rings :param policy_name: optionally the name to use when finding the ring """ if not os.path.exists(datafile): print "Data file doesn't exist" raise InfoSystemExit() if not datafile.startswith(('/', './')): datafile = './' + datafile policy_index = None ring = None datadir = DATADIR_BASE # try to extract policy index from datafile disk path policy_index = int(extract_policy(datafile) or POLICIES.legacy) try: if policy_index: datadir += '-' + str(policy_index) ring = Ring(swift_dir, ring_name='object-' + str(policy_index)) elif policy_index == 0: ring = Ring(swift_dir, ring_name='object') except IOError: # no such ring pass if policy_name: policy = POLICIES.get_by_name(policy_name) if policy: policy_index_for_name = policy.idx if (policy_index is not None and policy_index_for_name is not None and policy_index != policy_index_for_name): print 'Attention: Ring does not match policy!' print 'Double check your policy name!' if not ring and policy_index_for_name: ring = POLICIES.get_object_ring(policy_index_for_name, swift_dir) datadir = get_data_dir(policy_index_for_name) with open(datafile, 'rb') as fp: try: metadata = read_metadata(fp) except EOFError: print "Invalid metadata" raise InfoSystemExit() etag = metadata.pop('ETag', '') length = metadata.pop('Content-Length', '') path = metadata.get('name', '') print_obj_metadata(metadata) # Optional integrity check; it's useful, but slow. file_len = None if check_etag: h = md5() file_len = 0 while True: data = fp.read(64 * 1024) if not data: break h.update(data) file_len += len(data) h = h.hexdigest() if etag: if h == etag: print 'ETag: %s (valid)' % etag else: print ("ETag: %s doesn't match file hash of %s!" % (etag, h)) else: print 'ETag: Not found in metadata' else: print 'ETag: %s (not checked)' % etag file_len = os.fstat(fp.fileno()).st_size if length: if file_len == int(length): print 'Content-Length: %s (valid)' % length else: print ("Content-Length: %s doesn't match file length of %s" % (length, file_len)) else: print 'Content-Length: Not found in metadata' account, container, obj = path.split('/', 3)[1:] if ring: print_ring_locations(ring, datadir, account, container, obj, policy_index=policy_index)
def generate_report(conf, policy_name=None): global json_output json_output = config_true_value(conf.get('dump_json', 'no')) if policy_name is None: policy = POLICIES.default else: policy = POLICIES.get_by_name(policy_name) if policy is None: exit('Unable to find policy: %s' % policy_name) if not json_output: print('Using storage policy: %s ' % policy.name) swift_dir = conf.get('swift_dir', '/etc/swift') retries = int(conf.get('retries', 5)) concurrency = int(conf.get('concurrency', 25)) endpoint_type = str(conf.get('endpoint_type', 'publicURL')) region_name = str(conf.get('region_name', '')) container_report = config_true_value(conf.get('container_report', 'yes')) object_report = config_true_value(conf.get('object_report', 'yes')) if not (object_report or container_report): exit("Neither container or object report is set to run") user_domain_name = str(conf.get('user_domain_name', '')) project_domain_name = str(conf.get('project_domain_name', '')) project_name = str(conf.get('project_name', '')) insecure = config_true_value(conf.get('keystone_api_insecure', 'no')) coropool = GreenPool(size=concurrency) os_options = {'endpoint_type': endpoint_type} if user_domain_name: os_options['user_domain_name'] = user_domain_name if project_domain_name: os_options['project_domain_name'] = project_domain_name if project_name: os_options['project_name'] = project_name if region_name: os_options['region_name'] = region_name url, token = get_auth(conf['auth_url'], conf['auth_user'], conf['auth_key'], auth_version=conf.get('auth_version', '1.0'), os_options=os_options, insecure=insecure) account = url.rsplit('/', 1)[1] connpool = Pool(max_size=concurrency) connpool.create = lambda: SimpleClient( url=url, token=token, retries=retries) container_ring = Ring(swift_dir, ring_name='container') object_ring = Ring(swift_dir, ring_name=policy.ring_name) output = {} if container_report: output['container'] = container_dispersion_report( coropool, connpool, account, container_ring, retries, conf.get('partitions'), policy) if object_report: output['object'] = object_dispersion_report( coropool, connpool, account, object_ring, retries, conf.get('partitions'), policy) return output
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs): """ Display placement information for an item based on ring lookup. If a ring is provided it always takes precedence, but warnings will be emitted if it doesn't match other optional arguments like the policy_name or ring_name. If no ring is provided the ring_name and/or policy_name will be used to lookup the ring. :param ring: a ring instance :param ring_name: server type, or storage policy ring name if object ring :param account: account name :param container: container name :param obj: object name :param partition: part number for non path lookups :param policy_name: name of storage policy to use to lookup the ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. """ policy_name = kwargs.get("policy_name", None) part = kwargs.get("partition", None) all_nodes = kwargs.get("all", False) swift_dir = kwargs.get("swift_dir", "/etc/swift") if ring and policy_name: policy = POLICIES.get_by_name(policy_name) if policy: if ring_name != policy.ring_name: print "Attention! mismatch between ring and policy detected!" else: print "Attention! Policy %s is not valid" % policy_name policy_index = None if ring is None and (obj or part): if not policy_name: print "Need a ring or policy" raise InfoSystemExit() policy = POLICIES.get_by_name(policy_name) if not policy: print "No policy named %r" % policy_name raise InfoSystemExit() policy_index = int(policy) ring = POLICIES.get_object_ring(policy_index, swift_dir) ring_name = (POLICIES.get_by_name(policy_name)).ring_name if account is None and (container is not None or obj is not None): print "No account specified" raise InfoSystemExit() if container is None and obj is not None: print "No container specified" raise InfoSystemExit() if account is None and part is None: print "No target specified" raise InfoSystemExit() loc = "<type>" if part and ring_name: if "-" in ring_name and ring_name.startswith("object"): loc = "objects-" + ring_name.split("-", 1)[1] else: loc = ring_name + "s" if account and container and obj: loc = "objects" if "-" in ring_name and ring_name.startswith("object"): policy_index = int(ring_name.rsplit("-", 1)[1]) loc = "objects-%d" % policy_index if account and container and not obj: loc = "containers" if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name="container") else: if ring_name != "container": print "Attention! mismatch between ring and item detected!" if account and not container and not obj: loc = "accounts" if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name="account") else: if ring_name != "account": print "Attention! mismatch between ring and item detected!" print "\nAccount \t%s" % account print "Container\t%s" % container print "Object \t%s\n\n" % obj print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def generate_report(conf, policy_name=None): global json_output json_output = config_true_value(conf.get('dump_json', 'no')) if policy_name is None: policy = POLICIES.default else: policy = POLICIES.get_by_name(policy_name) if policy is None: exit('Unable to find policy: %s' % policy_name) if not json_output: print('Using storage policy: %s ' % policy.name) swift_dir = conf.get('swift_dir', '/etc/swift') retries = int(conf.get('retries', 5)) concurrency = int(conf.get('concurrency', 25)) endpoint_type = str(conf.get('endpoint_type', 'publicURL')) region_name = str(conf.get('region_name', '')) container_report = config_true_value(conf.get('container_report', 'yes')) object_report = config_true_value(conf.get('object_report', 'yes')) if not (object_report or container_report): exit("Neither container or object report is set to run") user_domain_name = str(conf.get('user_domain_name', '')) project_domain_name = str(conf.get('project_domain_name', '')) project_name = str(conf.get('project_name', '')) insecure = config_true_value(conf.get('keystone_api_insecure', 'no')) coropool = GreenPool(size=concurrency) os_options = {'endpoint_type': endpoint_type} if user_domain_name: os_options['user_domain_name'] = user_domain_name if project_domain_name: os_options['project_domain_name'] = project_domain_name if project_name: os_options['project_name'] = project_name if region_name: os_options['region_name'] = region_name url, token = get_auth(conf['auth_url'], conf['auth_user'], conf['auth_key'], auth_version=conf.get('auth_version', '1.0'), os_options=os_options, insecure=insecure) account = url.rsplit('/', 1)[1] connpool = Pool(max_size=concurrency) connpool.create = lambda: SimpleClient( url=url, token=token, retries=retries) container_ring = Ring(swift_dir, ring_name='container') object_ring = Ring(swift_dir, ring_name=policy.ring_name) output = {} if container_report: output['container'] = container_dispersion_report( coropool, connpool, account, container_ring, retries, conf.get('partitions'), policy) if object_report: output['object'] = object_dispersion_report(coropool, connpool, account, object_ring, retries, conf.get('partitions'), policy) return output