def __init__(self, app, conf, *args, **kwargs): self.app = app self.conf = Config() # Set default values if they are not configured self.conf.allow_no_owner = config_true_value( conf.get('allow_no_owner', False)) self.conf.location = conf.get('location', 'US') self.conf.dns_compliant_bucket_names = config_true_value( conf.get('dns_compliant_bucket_names', True)) self.conf.max_bucket_listing = config_positive_int_value( conf.get('max_bucket_listing', 1000)) self.conf.max_parts_listing = config_positive_int_value( conf.get('max_parts_listing', 1000)) self.conf.max_multi_delete_objects = config_positive_int_value( conf.get('max_multi_delete_objects', 1000)) self.conf.s3_acl = config_true_value(conf.get('s3_acl', False)) self.conf.storage_domain = conf.get('storage_domain', '') self.conf.auth_pipeline_check = config_true_value( conf.get('auth_pipeline_check', True)) self.conf.max_upload_part_num = config_positive_int_value( conf.get('max_upload_part_num', 1000)) self.conf.check_bucket_owner = config_true_value( conf.get('check_bucket_owner', False)) self.conf.force_swift_request_proxy_log = config_true_value( conf.get('force_swift_request_proxy_log', False)) self.conf.allow_multipart_uploads = config_true_value( conf.get('allow_multipart_uploads', True)) self.conf.min_segment_size = config_positive_int_value( conf.get('min_segment_size', 5242880)) self.logger = get_logger(conf, log_route=conf.get('log_name', 's3api')) self.slo_enabled = self.conf.allow_multipart_uploads self.check_pipeline(self.conf)
def __init__(self, app, conf): self.app = app self.conf = conf self.logger = get_logger(conf, log_route='swift_zipkin') self.enabled = config_true_value(conf.get('zipkin_enable')) self.zipkin_v2_host = self.conf.get('zipkin_v2_host') or '127.0.0.1' self.zipkin_v2_port = config_positive_int_value( self.conf.get('zipkin_v2_port') or 9411) raw_sample_rate = self.conf.get('zipkin_sample_rate') if raw_sample_rate: self.zipkin_sample_rate = config_float_value(raw_sample_rate, minimum=0.0, maximum=1.0) else: self.zipkin_sample_rate = 1.0 self.zipkin_flush_threshold_size = config_positive_int_value( self.conf.get('zipkin_flush_threshold_size', 2**20)) self.zipkin_flush_threshold_sec = config_float_value( self.conf.get('zipkin_flush_threshold_sec', 2.0)) if not self.enabled: # It's not like we're going to get enabled between the first and # second invocations of this constructor... return # Use our class to store a count of instantiations; We'll get # one time before the forking off of workers, and again inside each # worker. We're interested in only doing our business post-fork, # during the SECOND instantiation. setattr(self.__class__, '_instantiation_count', 1 + getattr(self.__class__, '_instantiation_count', 0)) if self.__class__._instantiation_count < 2: self.logger.debug( 'ZipkinMiddleware() count=%d PID=%d; ' 'deferring work to 2nd instantiation.', self.__class__._instantiation_count, os.getpid()) return self.logger.debug( 'ZipkinMiddleware() count=%d PID=%d; ' 'tracing %.0f%% of reqs to Zipkin at ' '%s:%s', self.__class__._instantiation_count, os.getpid(), 100.0 * self.zipkin_sample_rate, self.zipkin_v2_host, self.zipkin_v2_port) patch_eventlet_and_swift( self.logger, self.zipkin_v2_host, self.zipkin_v2_port, self.zipkin_sample_rate, self.zipkin_flush_threshold_size, self.zipkin_flush_threshold_sec, )
def __init__(self, app, conf, *args, **kwargs): self.app = app self.conf = Config() # Set default values if they are not configured self.conf.allow_no_owner = config_true_value( conf.get('allow_no_owner', False)) self.conf.location = conf.get('location', 'US') self.conf.dns_compliant_bucket_names = config_true_value( conf.get('dns_compliant_bucket_names', True)) self.conf.max_bucket_listing = config_positive_int_value( conf.get('max_bucket_listing', 1000)) self.conf.max_parts_listing = config_positive_int_value( conf.get('max_parts_listing', 1000)) self.conf.max_multi_delete_objects = config_positive_int_value( conf.get('max_multi_delete_objects', 1000)) self.conf.s3_acl = config_true_value( conf.get('s3_acl', False)) self.conf.storage_domain = conf.get('storage_domain', '') self.conf.auth_pipeline_check = config_true_value( conf.get('auth_pipeline_check', True)) self.conf.max_upload_part_num = config_positive_int_value( conf.get('max_upload_part_num', 1000)) self.conf.check_bucket_owner = config_true_value( conf.get('check_bucket_owner', False)) self.conf.force_swift_request_proxy_log = config_true_value( conf.get('force_swift_request_proxy_log', False)) self.conf.allow_multipart_uploads = config_true_value( conf.get('allow_multipart_uploads', True)) self.conf.min_segment_size = config_positive_int_value( conf.get('min_segment_size', 5242880)) self.logger = get_logger( conf, log_route=conf.get('log_name', 's3api')) self.slo_enabled = self.conf.allow_multipart_uploads self.check_pipeline(self.conf)
def __init__(self, app, wsgi_conf, *args, **kwargs): self.app = app self.conf = Config() # Set default values if they are not configured self.conf.allow_no_owner = config_true_value( wsgi_conf.get('allow_no_owner', False)) self.conf.location = wsgi_conf.get('location', 'us-east-1') self.conf.dns_compliant_bucket_names = config_true_value( wsgi_conf.get('dns_compliant_bucket_names', True)) self.conf.max_bucket_listing = config_positive_int_value( wsgi_conf.get('max_bucket_listing', 1000)) self.conf.max_parts_listing = config_positive_int_value( wsgi_conf.get('max_parts_listing', 1000)) self.conf.max_multi_delete_objects = config_positive_int_value( wsgi_conf.get('max_multi_delete_objects', 1000)) self.conf.multi_delete_concurrency = config_positive_int_value( wsgi_conf.get('multi_delete_concurrency', 2)) self.conf.s3_acl = config_true_value(wsgi_conf.get('s3_acl', False)) self.conf.storage_domain = wsgi_conf.get('storage_domain', '') self.conf.auth_pipeline_check = config_true_value( wsgi_conf.get('auth_pipeline_check', True)) self.conf.max_upload_part_num = config_positive_int_value( wsgi_conf.get('max_upload_part_num', 1000)) self.conf.check_bucket_owner = config_true_value( wsgi_conf.get('check_bucket_owner', False)) self.conf.force_swift_request_proxy_log = config_true_value( wsgi_conf.get('force_swift_request_proxy_log', False)) self.conf.allow_multipart_uploads = config_true_value( wsgi_conf.get('allow_multipart_uploads', True)) self.conf.min_segment_size = config_positive_int_value( wsgi_conf.get('min_segment_size', 5242880)) self.conf.allowable_clock_skew = config_positive_int_value( wsgi_conf.get('allowable_clock_skew', 15 * 60)) self.conf.cors_preflight_allow_origin = list_from_csv( wsgi_conf.get('cors_preflight_allow_origin', '')) if '*' in self.conf.cors_preflight_allow_origin and \ len(self.conf.cors_preflight_allow_origin) > 1: raise ValueError('if cors_preflight_allow_origin should include ' 'all domains, * must be the only entry') self.conf.ratelimit_as_client_error = config_true_value( wsgi_conf.get('ratelimit_as_client_error', False)) self.logger = get_logger(wsgi_conf, log_route=wsgi_conf.get('log_name', 's3api')) self.check_pipeline(wsgi_conf)
def main(args=None): parser = _make_parser() args = parser.parse_args(args) if not args.subcommand: # On py2, subparsers are required; on py3 they are not; see # https://bugs.python.org/issue9253. py37 added a `required` kwarg # to let you control it, but prior to that, there was no choice in # the matter. So, check whether the destination was set and bomb # out if not. parser.print_help() print('\nA sub-command is required.') return 1 conf = {} rows_per_shard = DEFAULT_ROWS_PER_SHARD shrink_threshold = DEFAULT_SHRINK_THRESHOLD expansion_limit = DEFAULT_ROWS_PER_SHARD if args.conf_file: try: conf = readconf(args.conf_file, 'container-sharder') shard_container_threshold = config_positive_int_value(conf.get( 'shard_container_threshold', DEFAULT_SHARD_CONTAINER_THRESHOLD)) if shard_container_threshold: rows_per_shard = shard_container_threshold // 2 shrink_threshold = int( shard_container_threshold * config_percent_value( conf.get('shard_shrink_point', DEFAULT_SHARD_SHRINK_POINT))) expansion_limit = int( shard_container_threshold * config_percent_value( conf.get('shard_shrink_merge_point', DEFAULT_SHARD_MERGE_POINT))) except Exception as exc: print('Error opening config file %s: %s' % (args.conf_file, exc), file=sys.stderr) return 2 # seems having sub parsers mean sometimes an arg wont exist in the args # namespace. But we can check if it is with the 'in' statement. if "max_shrinking" in args and args.max_shrinking is None: args.max_shrinking = int(conf.get( "max_shrinking", DEFAULT_MAX_SHRINKING)) if "max_expanding" in args and args.max_expanding is None: args.max_expanding = int(conf.get( "max_expanding", DEFAULT_MAX_EXPANDING)) if "shrink_threshold" in args and args.shrink_threshold is None: args.shrink_threshold = shrink_threshold if "expansion_limit" in args and args.expansion_limit is None: args.expansion_limit = expansion_limit if "rows_per_shard" in args and args.rows_per_shard is None: args.rows_per_shard = rows_per_shard if args.func in (analyze_shard_ranges,): args.input = args.path_to_file return args.func(args) or 0 logger = get_logger({}, name='ContainerBroker', log_to_console=True) broker = ContainerBroker(os.path.realpath(args.path_to_file), logger=logger, skip_commits=not args.force_commits) try: broker.get_info() except Exception as exc: print('Error opening container DB %s: %s' % (args.path_to_file, exc), file=sys.stderr) return 2 print('Loaded db broker for %s' % broker.path, file=sys.stderr) return args.func(broker, args)
def __init__(self, idx, name='', aliases='', is_default=False, is_deprecated=False, object_ring=None, ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, ec_type=None, ec_ndata=None, ec_nparity=None, ec_duplication_factor=1): super(ECStoragePolicy, self).__init__(idx=idx, name=name, aliases=aliases, is_default=is_default, is_deprecated=is_deprecated, object_ring=object_ring) # Validate erasure_coding policy specific members # ec_type is one of the EC implementations supported by PyEClib if ec_type is None: raise PolicyError('Missing ec_type') if ec_type not in VALID_EC_TYPES: raise PolicyError('Wrong ec_type %s for policy %s, should be one' ' of "%s"' % (ec_type, self.name, ', '.join(VALID_EC_TYPES))) self._ec_type = ec_type # Define _ec_ndata as the number of EC data fragments # Accessible as the property "ec_ndata" try: value = int(ec_ndata) if value <= 0: raise ValueError self._ec_ndata = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata, index=self.idx) # Define _ec_nparity as the number of EC parity fragments # Accessible as the property "ec_nparity" try: value = int(ec_nparity) if value <= 0: raise ValueError self._ec_nparity = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_parity_fragments %r' % ec_nparity, index=self.idx) # Define _ec_segment_size as the encode segment unit size # Accessible as the property "ec_segment_size" try: value = int(ec_segment_size) if value <= 0: raise ValueError self._ec_segment_size = value except (TypeError, ValueError): raise PolicyError('Invalid ec_object_segment_size %r' % ec_segment_size, index=self.idx) if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5: logger = logging.getLogger("swift.common.storage_policy") if not logger.handlers: # If nothing else, log to stderr logger.addHandler(logging.StreamHandler(sys.__stderr__)) logger.warning( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. Any data in this policy should be migrated. ' 'See https://bugs.launchpad.net/swift/+bug/1639691 for ' 'more information.' % self.name) if not is_deprecated: raise PolicyError( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. This policy MUST be deprecated.' % self.name) # Initialize PyECLib EC backend try: self.pyeclib_driver = \ ECDriver(k=self._ec_ndata, m=self._ec_nparity, ec_type=self._ec_type) except ECDriverError as e: raise PolicyError("Error creating EC policy (%s)" % e, index=self.idx) # quorum size in the EC case depends on the choice of EC scheme. self._ec_quorum_size = \ self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed() self._fragment_size = None self._ec_duplication_factor = \ config_positive_int_value(ec_duplication_factor)
def __init__(self, idx, name='', aliases='', is_default=False, is_deprecated=False, object_ring=None, diskfile_module='egg:swift#erasure_coding.fs', ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, ec_type=None, ec_ndata=None, ec_nparity=None, ec_duplication_factor=1): super(ECStoragePolicy, self).__init__( idx=idx, name=name, aliases=aliases, is_default=is_default, is_deprecated=is_deprecated, object_ring=object_ring, diskfile_module=diskfile_module) # Validate erasure_coding policy specific members # ec_type is one of the EC implementations supported by PyEClib if ec_type is None: raise PolicyError('Missing ec_type') if ec_type not in VALID_EC_TYPES: raise PolicyError('Wrong ec_type %s for policy %s, should be one' ' of "%s"' % (ec_type, self.name, ', '.join(VALID_EC_TYPES))) self._ec_type = ec_type # Define _ec_ndata as the number of EC data fragments # Accessible as the property "ec_ndata" try: value = int(ec_ndata) if value <= 0: raise ValueError self._ec_ndata = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata, index=self.idx) # Define _ec_nparity as the number of EC parity fragments # Accessible as the property "ec_nparity" try: value = int(ec_nparity) if value <= 0: raise ValueError self._ec_nparity = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_parity_fragments %r' % ec_nparity, index=self.idx) # Define _ec_segment_size as the encode segment unit size # Accessible as the property "ec_segment_size" try: value = int(ec_segment_size) if value <= 0: raise ValueError self._ec_segment_size = value except (TypeError, ValueError): raise PolicyError('Invalid ec_object_segment_size %r' % ec_segment_size, index=self.idx) if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5: logger = logging.getLogger("swift.common.storage_policy") if not logger.handlers: # If nothing else, log to stderr logger.addHandler(logging.StreamHandler(sys.__stderr__)) logger.warning( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. Any data in this policy should be migrated. ' 'See https://bugs.launchpad.net/swift/+bug/1639691 for ' 'more information.' % self.name) if not is_deprecated: raise PolicyError( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. This policy MUST be deprecated.' % self.name) # Initialize PyECLib EC backend try: self.pyeclib_driver = \ ECDriver(k=self._ec_ndata, m=self._ec_nparity, ec_type=self._ec_type) except ECDriverError as e: raise PolicyError("Error creating EC policy (%s)" % e, index=self.idx) # quorum size in the EC case depends on the choice of EC scheme. self._ec_quorum_size = \ self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed() self._fragment_size = None self._ec_duplication_factor = \ config_positive_int_value(ec_duplication_factor)