def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) defaults = { 'methods': 'GET HEAD PUT POST DELETE', 'incoming_remove_headers': DEFAULT_INCOMING_REMOVE_HEADERS, 'incoming_allow_headers': DEFAULT_INCOMING_ALLOW_HEADERS, 'outgoing_remove_headers': DEFAULT_OUTGOING_REMOVE_HEADERS, 'outgoing_allow_headers': DEFAULT_OUTGOING_ALLOW_HEADERS, 'allowed_digests': DEFAULT_ALLOWED_DIGESTS, } info_conf = {k: conf.get(k, v).split() for k, v in defaults.items()} allowed_digests = set(digest.lower() for digest in info_conf['allowed_digests']) not_supported = allowed_digests - SUPPORTED_DIGESTS if not_supported: logger = get_logger(conf, log_route='tempurl') logger.warning('The following digest algorithms are configured but ' 'not supported: %s', ', '.join(not_supported)) allowed_digests -= not_supported if not allowed_digests: raise ValueError('No valid digest algorithms are configured ' 'for tempurls') info_conf['allowed_digests'] = sorted(allowed_digests) register_swift_info('tempurl', **info_conf) conf.update(info_conf) return lambda app: TempURL(app, conf)
def filter_factory(global_conf, **local_conf): """Provides a factory function for loading versioning middleware.""" conf = global_conf.copy() conf.update(local_conf) if config_true_value(conf.get('allow_versioned_writes')): register_swift_info('versioned_writes', allowed_flags=(CLIENT_VERSIONS_LOC, CLIENT_HISTORY_LOC)) allow_object_versioning = config_true_value( conf.get('allow_object_versioning')) allow_oio_versioning = config_true_value(conf.get('allow_oio_versioning')) if allow_object_versioning: register_swift_info('object_versioning') def versioning_filter(app): if allow_object_versioning: if 'symlink' not in get_swift_info(): raise ValueError('object versioning requires symlinks') if allow_oio_versioning: vfunc = OioObjectVersioningMiddleware.is_valid_version_id register_swift_info('object_versioning', is_valid_version_id=vfunc) return OioObjectVersioningMiddleware(app, conf) app = ObjectVersioningMiddleware(app, conf) return VersionedWritesMiddleware(app, conf) return versioning_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_containers_per_extraction = \ int(conf.get('max_containers_per_extraction', 10000)) max_failed_extractions = int(conf.get('max_failed_extractions', 1000)) max_deletes_per_request = int(conf.get('max_deletes_per_request', 10000)) max_failed_deletes = int(conf.get('max_failed_deletes', 1000)) yield_frequency = int(conf.get('yield_frequency', 10)) retry_count = int(conf.get('delete_container_retry_count', 0)) retry_interval = 1.5 register_swift_info( 'bulk_upload', max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions) register_swift_info( 'bulk_delete', max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes) def bulk_filter(app): return Bulk( app, conf, max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions, max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes, yield_frequency=yield_frequency, retry_count=retry_count, retry_interval=retry_interval) return bulk_filter
def filter_factory(global_conf, **local_conf): register_swift_info('container_quotas') def container_quota_filter(app): return ContainerQuotaMiddleware(app) return container_quota_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_containers_per_extraction = \ int(conf.get('max_containers_per_extraction', 10000)) max_failed_extractions = int(conf.get('max_failed_extractions', 1000)) max_deletes_per_request = int(conf.get('max_deletes_per_request', 10000)) max_failed_deletes = int(conf.get('max_failed_deletes', 1000)) yield_frequency = int(conf.get('yield_frequency', 60)) register_swift_info( 'bulk_upload', max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions) register_swift_info( 'bulk_delete', max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes) def bulk_filter(app): return Bulk( app, conf, max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions, max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes, yield_frequency=yield_frequency) return bulk_filter
def filter_factory(global_conf, **local_conf): """ paste.deploy app factory for creating WSGI proxy apps. """ conf = global_conf.copy() conf.update(local_conf) account_ratelimit = float(conf.get('account_ratelimit', 0)) max_sleep_time_seconds = \ float(conf.get('max_sleep_time_seconds', 60)) container_ratelimits, cont_limit_info = interpret_conf_limits( conf, 'container_ratelimit_', info=1) container_listing_ratelimits, cont_list_limit_info = \ interpret_conf_limits(conf, 'container_listing_ratelimit_', info=1) # not all limits are exposed (intentionally) register_swift_info('ratelimit', account_ratelimit=account_ratelimit, max_sleep_time_seconds=max_sleep_time_seconds, container_ratelimits=cont_limit_info, container_listing_ratelimits=cont_list_limit_info) def limit_filter(app): return RateLimitMiddleware(app, conf) return limit_filter
def filter_factory(global_conf, **local_conf): register_swift_info("container_quotas") def container_quota_filter(app): return ContainerQuotaMiddleware(app) return container_quota_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_manifest_segments = int( conf.get('max_manifest_segments', DEFAULT_MAX_MANIFEST_SEGMENTS)) max_manifest_size = int( conf.get('max_manifest_size', DEFAULT_MAX_MANIFEST_SIZE)) yield_frequency = int(conf.get('yield_frequency', DEFAULT_YIELD_FREQUENCY)) register_swift_info( 'slo', max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, yield_frequency=yield_frequency, # this used to be configurable; report it as 1 for # clients that might still care min_segment_size=1) def slo_filter(app): return OioStaticLargeObject( app, conf, max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, yield_frequency=yield_frequency) return slo_filter
def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" print "temp url filter_factory" conf = global_conf.copy() conf.update(local_conf) register_swift_info('tempurl') return lambda app: TempURL(app, conf)
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_manifest_segments = int(conf.get("max_manifest_segments", DEFAULT_MAX_MANIFEST_SEGMENTS)) max_manifest_size = int(conf.get("max_manifest_size", DEFAULT_MAX_MANIFEST_SIZE)) min_segment_size = int(conf.get("min_segment_size", DEFAULT_MIN_SEGMENT_SIZE)) register_swift_info( "slo", max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, min_segment_size=min_segment_size, ) def slo_filter(app): return StaticLargeObject( app, conf, max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, min_segment_size=min_segment_size, ) return slo_filter
def filter_factory(global_config, **local_config): conf = global_config.copy() conf.update(local_config) jit_conf = dict() jit_conf['totalseconds'] = int(conf.get( 'totalseconds', 60)) #allowed time diff in seconds between previous and next object jit_conf['chainsave'] = conf.get('chainsave', '/tmp/chain.p') #where to save the chain jit_conf['probthreshold'] = float(conf.get( 'probthreshold', '0.5')) #minimum probability to be prefetched jit_conf['nthreads'] = int(conf.get( 'nthreads', 5)) #number of threads in the download threadpool jit_conf['twolevels'] = bool(conf.get('twolevels', False)) #two levels true/false register_swift_info(name) def factory(app): return JITPrefetchMiddleware(app, conf, jit_conf) return factory
def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) defaults = { 'methods': 'GET HEAD PUT POST DELETE', 'incoming_remove_headers': DEFAULT_INCOMING_REMOVE_HEADERS, 'incoming_allow_headers': DEFAULT_INCOMING_ALLOW_HEADERS, 'outgoing_remove_headers': DEFAULT_OUTGOING_REMOVE_HEADERS, 'outgoing_allow_headers': DEFAULT_OUTGOING_ALLOW_HEADERS, 'allowed_digests': DEFAULT_ALLOWED_DIGESTS, } info_conf = {k: conf.get(k, v).split() for k, v in defaults.items()} allowed_digests = set(digest.lower() for digest in info_conf['allowed_digests']) not_supported = allowed_digests - SUPPORTED_DIGESTS if not_supported: logger = get_logger(conf, log_route='tempurl') logger.warning( 'The following digest algorithms are configured but ' 'not supported: %s', ', '.join(not_supported)) allowed_digests -= not_supported if not allowed_digests: raise ValueError('No valid digest algorithms are configured ' 'for tempurls') info_conf['allowed_digests'] = sorted(allowed_digests) register_swift_info('tempurl', **info_conf) conf.update(info_conf) return lambda app: TempURL(app, conf)
def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" register_swift_info('account_quotas') def account_quota_filter(app): return AccountQuotaMiddleware(app) return account_quota_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) module_name = conf.get('storlet_gateway_module', 'stub') gateway_class = load_gateway(module_name) conf['gateway_module'] = gateway_class configParser = ConfigParser.RawConfigParser() configParser.read( conf.get('storlet_gateway_conf', '/etc/swift/storlet_stub_gateway.conf')) gateway_conf = dict(configParser.items("DEFAULT")) # TODO(eranr): Add supported storlets languages and # supported storlet API version containers = get_container_names(conf) swift_info = { 'storlet_container': containers['storlet'], 'storlet_dependency': containers['dependency'], 'storlet_gateway_class': gateway_class.__name__ } register_swift_info('storlet_handler', False, **swift_info) def storlet_handler_filter(app): return StorletHandlerMiddleware(app, conf, gateway_conf) return storlet_handler_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) storlet_conf = dict() storlet_conf['storlet_timeout'] = conf.get('storlet_timeout',40) storlet_conf['storlet_container'] = conf.get('storlet_container','storlet') storlet_conf['storlet_dependency'] = conf.get('storlet_dependency', 'dependency') storlet_conf['execution_server'] = conf.get('execution_server', '') storlet_conf['storlet_execute_on_proxy_only'] = config_true_value(conf.get('storlet_execute_on_proxy_only', 'false')) storlet_conf['gateway_conf'] = {} module_name = conf.get('storlet_gateway_module','') mo = module_name[:module_name.rfind(':')] cl = module_name[module_name.rfind(':') + 1:] module = __import__(mo, fromlist=[cl]) the_class = getattr(module, cl) configParser = ConfigParser.RawConfigParser() configParser.read(conf.get('storlet_gateway_conf', '/etc/swift/storlet_stub_gateway.conf')) additional_items = configParser.items("DEFAULT") for key, val in additional_items: storlet_conf[key]= val swift_info = {} storlet_conf["gateway_module"] = the_class register_swift_info('storlet_handler', False, **swift_info) def storlet_handler_filter(app): return StorletHandlerMiddleware(app, conf, storlet_conf) return storlet_handler_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_containers_per_extraction = int(conf.get("max_containers_per_extraction", 10000)) max_failed_extractions = int(conf.get("max_failed_extractions", 1000)) max_deletes_per_request = int(conf.get("max_deletes_per_request", 10000)) max_failed_deletes = int(conf.get("max_failed_deletes", 1000)) yield_frequency = int(conf.get("yield_frequency", 10)) retry_count = int(conf.get("delete_container_retry_count", 0)) retry_interval = 1.5 register_swift_info( "bulk_upload", max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions, ) register_swift_info( "bulk_delete", max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes ) def bulk_filter(app): return Bulk( app, conf, max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions, max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes, yield_frequency=yield_frequency, retry_count=retry_count, retry_interval=retry_interval, ) return bulk_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('slo') def slo_filter(app): return StaticLargeObject(app, conf) return slo_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('hlm') def hlm_filter(app): return HlmMiddleware(app, conf) return hlm_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('bulk') def bulk_filter(app): return Bulk(app, conf) return bulk_filter
def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('formpost') return lambda app: FormPost(app, conf)
def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info("formpost") return lambda app: FormPost(app, conf)
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('crossdomain') def crossdomain_filter(app): return CrossDomainMiddleware(app, conf) return crossdomain_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('zipkin') def zipkin_filter(app): return ZipkinMiddleware(app, conf) return zipkin_filter
def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('tempauth', account_acls=True) def auth_filter(app): return TempAuth(app, conf) return auth_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('container_sync') def cache_filter(app): return ContainerSync(app, conf) return cache_filter
def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) methods = conf.get('methods', 'GET HEAD PUT').split() register_swift_info('tempurl', methods=methods) return lambda app: TempURL(app, conf, methods=methods)
def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('automime') def auth_filter(app): return AutoMimeMiddleware(app, conf) return auth_filter
def fixity_factory(conf, **kwargs): utils.register_swift_info("fixity") conf = conf.copy() conf.update(**kwargs) def fixity_filter(app, conf): return FixityCheckMiddleware(app, conf) return fixity_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('rotating_keymaster') def rotating_keymaster_filter(app): return RotatingKeyMaster(app, conf) return rotating_keymaster_filter
def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('mac', account_acls=True) def acc_filter(app): return mandatory_access_control(app, conf) return acc_filter
def filter_factory(global_conf, **local_conf): """Returns a Static Web WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('staticweb') def staticweb_filter(app): return StaticWeb(app, conf) return staticweb_filter
def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) methods = conf.get('methods', 'GET HEAD PUT POST DELETE').split() register_swift_info('tempurl', methods=methods) return lambda app: TempURL(app, conf, methods=methods)
def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('swiftpolicy') def auth_filter(app): return SwiftPolicy(app, conf) return auth_filter
def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('keystoneauth') def auth_filter(app): return KeystoneAuth(app, conf) return auth_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('pfs', bypass_mode=conf.get('bypass_mode', 'off')) def pfs_filter(app): return BimodalChecker(S3Compat(PfsMiddleware(app, conf), conf), conf) return pfs_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) if config_true_value(conf.get("allow_versioned_writes")): register_swift_info("versioned_writes", allowed_versions_mode=VERSIONING_MODES) def obj_versions_filter(app): return VersionedWritesMiddleware(app, conf) return obj_versions_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info("domain_remap", default_reseller_prefix=conf.get("default_reseller_prefix")) def domain_filter(app): return DomainRemapMiddleware(app, conf) return domain_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) if config_true_value(conf.get('allow_versioned_writes')): register_swift_info('versioned_writes') def obj_versions_filter(app): return VersionedWritesMiddleware(app, conf) return obj_versions_filter
def filter_factory(global_conf, **local_conf): """Provides a factory function for loading encryption middleware.""" conf = global_conf.copy() conf.update(local_conf) enabled = not config_true_value(conf.get('disable_encryption', 'false')) register_swift_info('encryption', admin=True, enabled=enabled) def encryption_filter(app): return Decrypter(Encrypter(app, conf), conf) return encryption_filter
def webhook_factory(global_conf, **local_conf): register_swift_info('webhook') conf = global_conf.copy() conf.update(local_conf) def webhook_filter(app): return WebhookMiddleware(app, conf) return webhook_filter
def filter_factory(global_conf, **local_conf): # pragma: no cover conf = global_conf.copy() conf.update(local_conf) register_swift_info('cname_lookup', lookup_depth=int(conf.get('lookup_depth', '1'))) def cname_filter(app): return CNAMELookupMiddleware(app, conf) return cname_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info( 'domain_remap', default_reseller_prefix=conf.get('default_reseller_prefix')) def domain_filter(app): return DomainRemapMiddleware(app, conf) return domain_filter
def filter_factory(global_conf, **local_conf): """ paste.deploy app factory for creating WSGI proxy apps. """ conf = global_conf.copy() conf.update(local_conf) register_swift_info('ratelimit') def limit_filter(app): return RateLimitMiddleware(app, conf) return limit_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) if config_true_value(conf.get('allow_versioned_writes')): register_swift_info('versioned_writes', allowed_flags=( vw.CLIENT_VERSIONS_LOC, vw.CLIENT_HISTORY_LOC)) def obj_versions_filter(app): return OioVersionedWritesMiddleware(app, conf) return obj_versions_filter
def filter_factory(global_conf, **local_conf): """Provides a factory function for loading encryption middleware.""" conf = global_conf.copy() conf.update(local_conf) enabled = not config_true_value(conf.get('disable_encryption', 'false')) register_swift_info('rotating_encryption', admin=True, enabled=enabled) def rotating_encryption_filter(app): return RotatingDecrypter(RotatingEncrypter(app, conf), conf) return rotating_encryption_filter
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) symloop_max = int(conf.get('symloop_max', DEFAULT_SYMLOOP_MAX)) if symloop_max < 1: symloop_max = int(DEFAULT_SYMLOOP_MAX) register_swift_info('symlink', symloop_max=symloop_max) def symlink_mw(app): return SymlinkMiddleware(app, conf, symloop_max) return symlink_mw
def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) exclude_inspectors = local_conf.get('exclude', '').lower().split() for inspector in exclude_inspectors: if inspector in inspector_handlers['object']: del inspector_handlers['object'][inspector] avaiable_inspectors = [i.title() for i in inspector_handlers['object']] utils.register_swift_info('inspector', inspectors=avaiable_inspectors) def informant_filter(app): return InspectorMiddleware(app, conf) return informant_filter