def get_ring(ring_name, required_replicas, required_devices, server=None, force_validate=None, ipport2server=None, config_paths=None): if not server: server = ring_name ring = Ring('/etc/swift', ring_name=ring_name) if ipport2server is None: ipport2server = {} # used internally, even if not passed in if config_paths is None: config_paths = defaultdict(dict) store_config_paths(server, config_paths) repl_name = '%s-replicator' % server repl_configs = { i: readconf(c, section_name=repl_name) for i, c in config_paths[repl_name].items() } servers_per_port = any( int(c.get('servers_per_port', '0')) for c in repl_configs.values()) add_ring_devs_to_ipport2server(ring, server, ipport2server, servers_per_port=servers_per_port) if not VALIDATE_RSYNC and not force_validate: return ring # easy sanity checks if ring.replica_count != required_replicas: raise unittest.SkipTest( '%s has %s replicas instead of %s' % (ring.serialized_path, ring.replica_count, required_replicas)) devs = [dev for dev in ring.devs if dev is not None] if len(devs) != required_devices: raise unittest.SkipTest( '%s has %s devices instead of %s' % (ring.serialized_path, len(devs), required_devices)) for dev in devs: # verify server is exposing mounted device ipport = (dev['ip'], dev['port']) _, server_number = get_server_number(ipport, ipport2server) conf = repl_configs[server_number] for device in os.listdir(conf['devices']): if device == dev['device']: dev_path = os.path.join(conf['devices'], device) full_path = os.path.realpath(dev_path) if not os.path.exists(full_path): raise unittest.SkipTest( 'device %s in %s was not found (%s)' % (device, conf['devices'], full_path)) break else: raise unittest.SkipTest( "unable to find ring device %s under %s's devices (%s)" % (dev['device'], server, conf['devices'])) # verify server is exposing rsync device rsync_export = conf.get('rsync_module', '').rstrip('/') if not rsync_export: rsync_export = '{replication_ip}::%s' % server cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev) p = Popen(cmd, shell=True, stdout=PIPE) stdout, _stderr = p.communicate() if p.returncode: raise unittest.SkipTest('unable to connect to rsync ' 'export %s (%s)' % (rsync_export, cmd)) for line in stdout.decode().splitlines(): if line.rsplit(None, 1)[-1] == dev['device']: break else: raise unittest.SkipTest("unable to find ring device %s under " "rsync's exported devices for %s (%s)" % (dev['device'], rsync_export, cmd)) return ring
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger self._error_limiting = {} swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = swift_dir self.node_timeout = float(conf.get('node_timeout', 10)) self.recoverable_node_timeout = float( conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', DEFAULT_RECHECK_CONTAINER_EXISTENCE)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', DEFAULT_RECHECK_ACCOUNT_EXISTENCE)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') # ensure rings are loaded for all configured storage policies for policy in POLICIES: policy.load_ring(swift_dir) self.obj_controller_router = ObjectControllerRouter() self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) self.auto_create_account_prefix = ( conf.get('auto_create_account_prefix') or '.') self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip()] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip()] self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip()] self.cors_expose_headers = [ a.strip() for a in conf.get('cors_expose_headers', '').split(',') if a.strip()] self.strict_cors_mode = config_true_value( conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.concurrent_gets = \ config_true_value(conf.get('concurrent_gets')) self.concurrency_timeout = float(conf.get('concurrency_timeout', self.conn_timeout)) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: rnc_value = int(value[0]) self.request_node_count = lambda replicas: rnc_value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': rnc_value = int(value[0]) self.request_node_count = lambda replicas: rnc_value * replicas else: raise ValueError( 'Invalid request_node_count value: %r' % ''.join(value)) try: self._read_affinity = read_affinity = conf.get('read_affinity', '') self.read_affinity_sort_key = affinity_key_function(read_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid read_affinity value: %r (%s)" % (read_affinity, err.message)) try: write_affinity = conf.get('write_affinity', '') self.write_affinity_is_local_fn \ = affinity_locality_predicate(write_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid write_affinity value: %r (%s)" % (write_affinity, err.message)) value = conf.get('write_affinity_node_count', '2 * replicas').lower().split() if len(value) == 1: wanc_value = int(value[0]) self.write_affinity_node_count = lambda replicas: wanc_value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': wanc_value = int(value[0]) self.write_affinity_node_count = \ lambda replicas: wanc_value * replicas else: raise ValueError( 'Invalid write_affinity_node_count value: %r' % ''.join(value)) # swift_owner_headers are stripped by the account and container # controllers; we should extend header stripping to object controller # when a privileged object header is implemented. swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' 'x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() for name in swift_owner_headers.split(',') if name.strip()] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. socket._fileobject.default_bufsize = self.client_chunk_size self.expose_info = config_true_value( conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get('disallowed_sections', 'swift.valid_api_versions')) self.admin_key = conf.get('admin_key', None) register_swift_info( version=swift_version, strict_cors_mode=self.strict_cors_mode, policies=POLICIES.get_policy_info(), allow_account_management=self.allow_account_management, account_autocreate=self.account_autocreate, **constraints.EFFECTIVE_CONSTRAINTS)
def test_account_container_reload(self): for server in ('account', 'container'): ring = Ring('/etc/swift', ring_name=server) node = random.choice(ring.get_part_nodes(1)) self._check_reload(server, node['ip'], node['port'])
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'true')) self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip()] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip()] self.rate_limit_after_segment = \ int(conf.get('rate_limit_after_segment', 10)) self.rate_limit_segments_per_sec = \ int(conf.get('rate_limit_segments_per_sec', 1)) self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip()] self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.allow_static_large_object = config_true_value( conf.get('allow_static_large_object', 'true')) self.max_large_object_get_time = float( conf.get('max_large_object_get_time', '86400')) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.request_node_count = lambda r: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.request_node_count = lambda r: value * r.replica_count else: raise ValueError( 'Invalid request_node_count value: %r' % ''.join(value)) try: read_affinity = conf.get('read_affinity', '') self.read_affinity_sort_key = affinity_key_function(read_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid read_affinity value: %r (%s)" % (read_affinity, err.message)) try: write_affinity = conf.get('write_affinity', '') self.write_affinity_is_local_fn \ = affinity_locality_predicate(write_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid write_affinity value: %r (%s)" % (write_affinity, err.message)) value = conf.get('write_affinity_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.write_affinity_node_count = lambda r: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.write_affinity_node_count = lambda r: value * r.replica_count else: raise ValueError( 'Invalid write_affinity_node_count value: %r' % ''.join(value))
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): #logging.basicConfig(filename="/root/swift/swift/proxy/log", level=logging.info) #logging.info("proxy server test __init__") #logging.info("*"*20) if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'true')) self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip()] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip()] self.rate_limit_after_segment = \ int(conf.get('rate_limit_after_segment', 10)) self.rate_limit_segments_per_sec = \ int(conf.get('rate_limit_segments_per_sec', 1)) self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip()] self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.allow_static_large_object = config_true_value( conf.get('allow_static_large_object', 'true')) self.max_large_object_get_time = float( conf.get('max_large_object_get_time', '86400')) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.request_node_count = lambda r: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.request_node_count = lambda r: value * r.replica_count else: raise ValueError( 'Invalid request_node_count value: %r' % ''.join(value)) try: read_affinity = conf.get('read_affinity', '') self.read_affinity_sort_key = affinity_key_function(read_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid read_affinity value: %r (%s)" % (read_affinity, err.message)) try: write_affinity = conf.get('write_affinity', '') self.write_affinity_is_local_fn \ = affinity_locality_predicate(write_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid write_affinity value: %r (%s)" % (write_affinity, err.message)) value = conf.get('write_affinity_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.write_affinity_node_count = lambda r: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.write_affinity_node_count = lambda r: value * r.replica_count else: raise ValueError( 'Invalid write_affinity_node_count value: %r' % ''.join(value)) swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2') self.swift_owner_headers = [ name.strip() for name in swift_owner_headers.split(',') if name.strip()] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. socket._fileobject.default_bufsize = self.client_chunk_size self.expose_info = config_true_value( conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get('disallowed_sections')) self.admin_key = conf.get('admin_key', None) register_swift_info(version=swift_version)
def get_object_ring(self): """ The object :class:`swift.common.ring.Ring` for the cluster. """ if not self.object_ring: self.object_ring = Ring(self.swift_dir, ring_name='object') return self.object_ring
#!/usr/bin/python from sys import exit from smtplib import SMTP from socket import gethostname from swift.common.constraints import check_mount from swift.common.utils import whataremyips from swift.common.ring import Ring try: ring = Ring('/etc/swift/object.ring.gz') except IOError: exit() my_ips = whataremyips() mounted = 0 drivecount = 0 drivelabels = [] for dev in ring.devs: try: if dev['ip'] in my_ips and float(dev['weight']) > 0: drivecount += 1 if check_mount('/srv/node', dev['device']): mounted += 1 else: drivelabels.append(dev['device']) except TypeError: pass unmounted = drivecount - mounted
def get_account_ring(self): """The account :class:`swift.common.ring.Ring` for the cluster.""" if not self.account_ring: self.account_ring = Ring(self.swift_dir, ring_name='account') return self.account_ring
def load_ring(self, swift_dir): if self.object_ring: return self.object_ring = Ring(swift_dir, ring_name=self.ring_name)
def setUp(self): resetswift() self.ip_ports = [ (dev['ip'], dev['port']) for dev in Ring('/etc/swift', ring_name='account').devs if dev ]
def __init__(self, conf, container_ring=None, logger=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = logger or get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = config_true_value(conf.get('mount_check', 'true')) #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a container sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join(conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.http_proxies = [ a.strip() for a in conf.get('sync_proxy', '').split(',') if a.strip() ] #: ContainerSyncStore instance for iterating over synced containers self.sync_store = ContainerSyncStore(self.devices, self.logger, self.mount_check) #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers whose sync has been turned off, but #: are not yet cleared from the sync store. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Per container stats. These are collected per container. #: puts - the number of puts that were done for the container #: deletes - the number of deletes that were fot the container #: bytes - the total number of bytes transferred per the container self.container_stats = collections.defaultdict(int) self.container_stats.clear() #: Time of last stats report. self.reported = time() self.swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. self.container_ring = container_ring or Ring(self.swift_dir, ring_name='container') bind_ip = conf.get('bind_ip', '0.0.0.0') self._myips = whataremyips(bind_ip) self._myport = int(conf.get('bind_port', 6201)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.conn_timeout = float(conf.get('conn_timeout', 5)) request_tries = int(conf.get('request_tries') or 3) internal_client_conf_path = conf.get('internal_client_conf_path') if not internal_client_conf_path: self.logger.warning( _('Configuration option internal_client_conf_path not ' 'defined. Using default configuration, See ' 'internal-client.conf-sample for options')) internal_client_conf = ConfigString(ic_conf_body) else: internal_client_conf = internal_client_conf_path try: self.swift = InternalClient(internal_client_conf, 'Swift Container Sync', request_tries) except IOError as err: if err.errno != errno.ENOENT: raise raise SystemExit( _('Unable to load internal client from config: ' '%(conf)r (%(error)s)') % { 'conf': internal_client_conf_path, 'error': err })
def _test_ondisk_data_after_write_with_crypto(self, policy_name): policy = storage_policy.POLICIES.get_by_name(policy_name) self._create_container(self.proxy_app, policy_name=policy_name) self._put_object(self.crypto_app, self.plaintext) self._post_object(self.crypto_app) # Verify container listing etag is encrypted by direct GET to container # server. We can use any server for all nodes since they all share same # devices dir. cont_server = self._test_context['test_servers'][3] cont_ring = Ring(self._test_context['testdir'], ring_name='container') part, nodes = cont_ring.get_nodes('a', self.container_name) for node in nodes: req = Request.blank('/%s/%s/a/%s' % (node['device'], part, self.container_name), method='GET', query_string='format=json') resp = req.get_response(cont_server) listing = json.loads(resp.body) # sanity checks... self.assertEqual(1, len(listing)) self.assertEqual('o', listing[0]['name']) self.assertEqual('application/test', listing[0]['content_type']) # verify encrypted etag value parts = listing[0]['hash'].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), self.km.create_key('/a/%s' % self.container_name), listing_etag_iv)).decode('ascii') self.assertEqual(exp_enc_listing_etag, parts[0]) # Verify diskfile data and metadata is encrypted ring_object = self.proxy_app.get_object_ring(int(policy)) partition, nodes = ring_object.get_nodes('a', self.container_name, 'o') conf = {'devices': self._test_context["testdir"], 'mount_check': 'false'} df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy] ondisk_data = [] exp_enc_body = None for node_index, node in enumerate(nodes): df = df_mgr.get_diskfile(node['device'], partition, 'a', self.container_name, 'o', policy=policy) with df.open(): meta = df.get_metadata() contents = b''.join(df.reader()) metadata = dict((k.lower(), v) for k, v in meta.items()) # verify on disk data - body body_iv = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['iv'] body_key_meta = load_crypto_meta( metadata['x-object-sysmeta-crypto-body-meta'])['body_key'] obj_key = self.km.create_key('/a/%s/o' % self.container_name) body_key = Crypto().unwrap_key(obj_key, body_key_meta) exp_enc_body = encrypt(self.plaintext, body_key, body_iv) ondisk_data.append((node, contents)) # verify on disk user metadata enc_val, meta = metadata[ 'x-object-transient-sysmeta-crypto-meta-fruit'].split(';') meta = meta.strip()[len('swift_meta='):] metadata_iv = load_crypto_meta(meta)['iv'] exp_enc_meta = base64.b64encode(encrypt( b'Kiwi', obj_key, metadata_iv)).decode('ascii') self.assertEqual(exp_enc_meta, enc_val) self.assertNotIn('x-object-meta-fruit', metadata) self.assertIn( 'x-object-transient-sysmeta-crypto-meta', metadata) meta = load_crypto_meta( metadata['x-object-transient-sysmeta-crypto-meta']) self.assertIn('key_id', meta) self.assertIn('path', meta['key_id']) self.assertEqual( '/a/%s/%s' % (self.container_name, self.object_name), meta['key_id']['path']) self.assertIn('v', meta['key_id']) self.assertEqual('1', meta['key_id']['v']) self.assertIn('cipher', meta) self.assertEqual(Crypto.cipher, meta['cipher']) # verify etag actual_enc_etag, _junk, actual_etag_meta = metadata[ 'x-object-sysmeta-crypto-etag'].partition('; swift_meta=') etag_iv = load_crypto_meta(actual_etag_meta)['iv'] exp_enc_etag = base64.b64encode(encrypt( self.plaintext_etag.encode('ascii'), obj_key, etag_iv)).decode('ascii') self.assertEqual(exp_enc_etag, actual_enc_etag) # verify etag hmac exp_etag_mac = hmac.new( obj_key, self.plaintext_etag.encode('ascii'), digestmod=hashlib.sha256).digest() exp_etag_mac = base64.b64encode(exp_etag_mac).decode('ascii') self.assertEqual(exp_etag_mac, metadata['x-object-sysmeta-crypto-etag-mac']) # verify etag override for container updates override = 'x-object-sysmeta-container-update-override-etag' parts = metadata[override].rsplit(';', 1) crypto_meta_param = parts[1].strip() crypto_meta = crypto_meta_param[len('swift_meta='):] listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] cont_key = self.km.create_key('/a/%s' % self.container_name) exp_enc_listing_etag = base64.b64encode( encrypt(self.plaintext_etag.encode('ascii'), cont_key, listing_etag_iv)).decode('ascii') self.assertEqual(exp_enc_listing_etag, parts[0]) self._check_GET_and_HEAD(self.crypto_app) return exp_enc_body, ondisk_data
def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='object_endpoint') swift_dir = conf.get('swift_dir', '/etc/swift') self.object_ring = Ring(swift_dir, ring_name='object')
def get_container_ring(self): """Get the container ring. Load it, if it hasn't been yet.""" if not self.container_ring: self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', policy_name=''): """ Display information about an object read from the datafile. Optionally verify the datafile content matches the ETag metadata. :param datafile: path on disk to object file :param check_etag: boolean, will read datafile content and verify computed checksum matches value stored in metadata. :param swift_dir: the path on disk to rings :param policy_name: optionally the name to use when finding the ring """ if not os.path.exists(datafile): print "Data file doesn't exist" raise InfoSystemExit() if not datafile.startswith(('/', './')): datafile = './' + datafile policy_index = None ring = None datadir = DATADIR_BASE # try to extract policy index from datafile disk path try: policy_index = extract_policy_index(datafile) except ValueError: pass try: if policy_index: datadir += '-' + str(policy_index) ring = Ring(swift_dir, ring_name='object-' + str(policy_index)) elif policy_index == 0: ring = Ring(swift_dir, ring_name='object') except IOError: # no such ring pass if policy_name: policy = POLICIES.get_by_name(policy_name) if policy: policy_index_for_name = policy.idx if (policy_index is not None and policy_index_for_name is not None and policy_index != policy_index_for_name): print 'Attention: Ring does not match policy!' print 'Double check your policy name!' if not ring and policy_index_for_name: ring = POLICIES.get_object_ring(policy_index_for_name, swift_dir) datadir = get_data_dir(policy_index_for_name) with open(datafile, 'rb') as fp: try: metadata = read_metadata(fp) except EOFError: print "Invalid metadata" raise InfoSystemExit() etag = metadata.pop('ETag', '') length = metadata.pop('Content-Length', '') path = metadata.get('name', '') print_obj_metadata(metadata) # Optional integrity check; it's useful, but slow. file_len = None if check_etag: h = md5() file_len = 0 while True: data = fp.read(64 * 1024) if not data: break h.update(data) file_len += len(data) h = h.hexdigest() if etag: if h == etag: print 'ETag: %s (valid)' % etag else: print("ETag: %s doesn't match file hash of %s!" % (etag, h)) else: print 'ETag: Not found in metadata' else: print 'ETag: %s (not checked)' % etag file_len = os.fstat(fp.fileno()).st_size if length: if file_len == int(length): print 'Content-Length: %s (valid)' % length else: print("Content-Length: %s doesn't match file length of %s" % (length, file_len)) else: print 'Content-Length: Not found in metadata' account, container, obj = path.split('/', 3)[1:] if ring: print_ring_locations(ring, datadir, account, container, obj, policy_index=policy_index)
def get_account_ring(self): """Get the account ring. Load it if it hasn't been yet.""" if not self.account_ring: self.account_ring = Ring(self.swift_dir, ring_name='account') return self.account_ring
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs): """ Display placement information for an item based on ring lookup. If a ring is provided it always takes precedence, but warnings will be emitted if it doesn't match other optional arguments like the policy_name or ring_name. If no ring is provided the ring_name and/or policy_name will be used to lookup the ring. :param ring: a ring instance :param ring_name: server type, or storage policy ring name if object ring :param account: account name :param container: container name :param obj: object name :param partition: part number for non path lookups :param policy_name: name of storage policy to use to lookup the ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. """ policy_name = kwargs.get('policy_name', None) part = kwargs.get('partition', None) all_nodes = kwargs.get('all', False) swift_dir = kwargs.get('swift_dir', '/etc/swift') if ring and policy_name: policy = POLICIES.get_by_name(policy_name) if policy: if ring_name != policy.ring_name: print 'Attention! mismatch between ring and policy detected!' else: print 'Attention! Policy %s is not valid' % policy_name policy_index = None if ring is None and (obj or part): if not policy_name: print 'Need a ring or policy' raise InfoSystemExit() policy = POLICIES.get_by_name(policy_name) if not policy: print 'No policy named %r' % policy_name raise InfoSystemExit() policy_index = int(policy) ring = POLICIES.get_object_ring(policy_index, swift_dir) ring_name = (POLICIES.get_by_name(policy_name)).ring_name if account is None and (container is not None or obj is not None): print 'No account specified' raise InfoSystemExit() if container is None and obj is not None: print 'No container specified' raise InfoSystemExit() if account is None and part is None: print 'No target specified' raise InfoSystemExit() loc = '<type>' if part and ring_name: if '-' in ring_name and ring_name.startswith('object'): loc = 'objects-' + ring_name.split('-', 1)[1] else: loc = ring_name + 's' if account and container and obj: loc = 'objects' if '-' in ring_name and ring_name.startswith('object'): policy_index = int(ring_name.rsplit('-', 1)[1]) loc = 'objects-%d' % policy_index if account and container and not obj: loc = 'containers' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='container') else: if ring_name != 'container': print 'Attention! mismatch between ring and item detected!' if account and not container and not obj: loc = 'accounts' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='account') else: if ring_name != 'account': print 'Attention! mismatch between ring and item detected!' print '\nAccount \t%s' % account print 'Container\t%s' % container print 'Object \t%s\n\n' % obj print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def get_container_ring(self): """The container :class:`swift.common.ring.Ring` for the cluster.""" if not self.container_ring: self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring
def __init__(self, conf, logger=None, account_ring=None, container_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger self._error_limiting = {} swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = swift_dir self.node_timeout = float(conf.get('node_timeout', 10)) self.recoverable_node_timeout = float( conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = float(conf.get('client_timeout', 60)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', DEFAULT_RECHECK_CONTAINER_EXISTENCE)) self.recheck_updating_shard_ranges = \ int(conf.get('recheck_updating_shard_ranges', DEFAULT_RECHECK_UPDATING_SHARD_RANGES)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', DEFAULT_RECHECK_ACCOUNT_EXISTENCE)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') # ensure rings are loaded for all configured storage policies for policy in POLICIES: policy.load_ring(swift_dir) self.obj_controller_router = ObjectControllerRouter() mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) if conf.get('auto_create_account_prefix'): self.logger.warning('Option auto_create_account_prefix is ' 'deprecated. Configure ' 'auto_create_account_prefix under the ' 'swift-constraints section of ' 'swift.conf. This option will ' 'be ignored in a future release.') self.auto_create_account_prefix = \ conf['auto_create_account_prefix'] else: self.auto_create_account_prefix = \ constraints.AUTO_CREATE_ACCOUNT_PREFIX self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip() ] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip() ] self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip() ] self.cors_expose_headers = [ a.strip() for a in conf.get('cors_expose_headers', '').split(',') if a.strip() ] self.strict_cors_mode = config_true_value( conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: rnc_value = int(value[0]) self.request_node_count = lambda replicas: rnc_value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': rnc_value = int(value[0]) self.request_node_count = lambda replicas: rnc_value * replicas else: raise ValueError('Invalid request_node_count value: %r' % ''.join(value)) # swift_owner_headers are stripped by the account and container # controllers; we should extend header stripping to object controller # when a privileged object header is implemented. swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' 'x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() for name in swift_owner_headers.split(',') if name.strip() ] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. if sys.version_info < (3, ): socket._fileobject.default_bufsize = self.client_chunk_size # TODO: find a way to enable similar functionality in py3 self.expose_info = config_true_value(conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get( 'disallowed_sections', ', '.join([ 'swift.auto_create_account_prefix', 'swift.valid_api_versions', ]))) self.admin_key = conf.get('admin_key', None) self._override_options = self._load_per_policy_config(conf) self.sorts_by_timing = any(pc.sorting_method == 'timing' for pc in self._override_options.values()) register_swift_info( version=swift_version, strict_cors_mode=self.strict_cors_mode, policies=POLICIES.get_policy_info(), allow_account_management=self.allow_account_management, account_autocreate=self.account_autocreate, **constraints.EFFECTIVE_CONSTRAINTS) self.watchdog = Watchdog() self.watchdog.spawn()