def __init__(self, idx, name='', is_default=False, is_deprecated=False,
object_ring=None):
try:
self.idx = int(idx)
except ValueError:
raise PolicyError('Invalid index', idx)
if self.idx < 0:
raise PolicyError('Invalid index', idx)
if not name:
raise PolicyError('Invalid name %r' % name, idx)
# this is defensively restrictive, but could be expanded in the future
if not all(c in VALID_CHARS for c in name):
raise PolicyError('Names are used as HTTP headers, and can not '
'reliably contain any characters not in %r. '
'Invalid name %r' % (VALID_CHARS, name))
if name.upper() == LEGACY_POLICY_NAME.upper() and self.idx != 0:
msg = 'The name %s is reserved for policy index 0. ' \
'Invalid name %r' % (LEGACY_POLICY_NAME, name)
raise PolicyError(msg, idx)
self.name = name
self.is_deprecated = config_true_value(is_deprecated)
self.is_default = config_true_value(is_default)
if self.is_deprecated and self.is_default:
raise PolicyError('Deprecated policy can not be default. '
'Invalid config', self.idx)
self.ring_name = _get_policy_string('object', self.idx)
self.object_ring = object_ring
def __init__(self, conf, logger=None):
super(ContainerController, self).__init__(conf)
self.logger = logger or get_logger(conf, log_route='container-server')
self.log_requests = config_true_value(conf.get('log_requests', 'true'))
self.root = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
#: ContainerSyncCluster instance for validating sync-to values.
self.realms_conf = ContainerSyncRealms(
os.path.join(
conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'),
self.logger)
#: The list of hosts we're allowed to send syncs to. This can be
#: overridden by data in self.realms_conf
self.allowed_sync_hosts = [
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()]
self.replicator_rpc = ContainerReplicatorRpc(
self.root, DATADIR, ContainerBroker, self.mount_check,
logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
if config_true_value(conf.get('allow_versions', 'f')):
self.save_headers.append('x-versions-location')
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
def __init__(self, conf, logger=None):
"""
Creates a new WSGI application for the Swift Object Server. An
example configuration is given at
<source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample.
"""
super(ObjectController, self).__init__(conf)
self.logger = logger or get_logger(conf, log_route='object-server')
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.client_timeout = int(conf.get('client_timeout', 60))
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
self.log_requests = config_true_value(conf.get('log_requests', 'true'))
self.max_upload_time = int(conf.get('max_upload_time', 86400))
self.slow = int(conf.get('slow', 0))
self.keep_cache_private = \
config_true_value(conf.get('keep_cache_private', 'false'))
default_allowed_headers = '''
content-disposition,
content-encoding,
x-delete-at,
x-object-manifest,
x-static-large-object,
'''
extra_allowed_headers = [
header.strip().lower() for header in conf.get(
'allowed_headers', default_allowed_headers).split(',')
if header.strip()
]
self.allowed_headers = set()
for header in extra_allowed_headers:
if header not in DATAFILE_SYSTEM_META:
self.allowed_headers.add(header)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
self.expiring_objects_account = self.auto_create_account_prefix + \
(conf.get('expiring_objects_account_name') or 'expiring_objects')
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
# Initialization was successful, so now apply the network chunk size
# parameter as the default read / write buffer size for the network
# sockets.
#
# NOTE WELL: This is a class setting, so until we get set this on a
# per-connection basis, this affects reading and writing on ALL
# sockets, those between the proxy servers and external clients, and
# those between the proxy servers and the other internal servers.
#
# ** Because the primary motivation for this is to optimize how data
# is written back to the proxy server, we could use the value from the
# disk_chunk_size parameter. However, it affects all created sockets
# using this class so we have chosen to tie it to the
# network_chunk_size parameter value instead.
socket._fileobject.default_bufsize = self.network_chunk_size
# Provide further setup specific to an object server implementation.
self.setup(conf)
def __init__(self, idx, name="", is_default=False, is_deprecated=False, object_ring=None):
# do not allow BaseStoragePolicy class to be instantiated directly
if type(self) == BaseStoragePolicy:
raise TypeError("Can't instantiate BaseStoragePolicy directly")
# policy parameter validation
try:
self.idx = int(idx)
except ValueError:
raise PolicyError("Invalid index", idx)
if self.idx < 0:
raise PolicyError("Invalid index", idx)
if not name:
raise PolicyError("Invalid name %r" % name, idx)
# this is defensively restrictive, but could be expanded in the future
if not all(c in VALID_CHARS for c in name):
raise PolicyError(
"Names are used as HTTP headers, and can not "
"reliably contain any characters not in %r. "
"Invalid name %r" % (VALID_CHARS, name)
)
if name.upper() == LEGACY_POLICY_NAME.upper() and self.idx != 0:
msg = "The name %s is reserved for policy index 0. " "Invalid name %r" % (LEGACY_POLICY_NAME, name)
raise PolicyError(msg, idx)
self.name = name
self.is_deprecated = config_true_value(is_deprecated)
self.is_default = config_true_value(is_default)
if self.policy_type not in BaseStoragePolicy.policy_type_to_policy_cls:
raise PolicyError("Invalid type", self.policy_type)
if self.is_deprecated and self.is_default:
raise PolicyError("Deprecated policy can not be default. " "Invalid config", self.idx)
self.ring_name = _get_policy_string("object", self.idx)
self.object_ring = object_ring
def filter_factory(global_conf, **local_conf):
"""
Returns the WSGI filter for use with paste.deploy.
Parameters in config:
# value to prepend to the account in order to compute the trash location
trash_prefix = ".trash-"
# how long, in seconds, trash objects should live before expiring. Set to 0
# to keep trash objects forever.
trash_lifetime = 7776000 # 90 days
# whether to block trash objects from being deleted
block_trash_deletes = no
# whether to enable undelete functionality by default. Administrators may
# explicitly enable or disable per account or container via the
# X-Undelete-Enabled header. Set this header to 'default' to resume default
# behavior.
enable_by_default = yes
"""
conf = global_conf.copy()
conf.update(local_conf)
trash_prefix = conf.get("trash_prefix", DEFAULT_TRASH_PREFIX)
trash_lifetime = int(conf.get("trash_lifetime", DEFAULT_TRASH_LIFETIME))
block_trash_deletes = utils.config_true_value(
conf.get('block_trash_deletes', 'no'))
enable_by_default = utils.config_true_value(
conf.get('enable_by_default', 'yes'))
def filt(app):
return UndeleteMiddleware(app, trash_prefix=trash_prefix,
trash_lifetime=trash_lifetime,
block_trash_deletes=block_trash_deletes,
enable_by_default=enable_by_default)
return filt
def __init__(self, idx, name='', is_default=False, is_deprecated=False,
object_ring=None, aliases=''):
# do not allow BaseStoragePolicy class to be instantiated directly
if type(self) == BaseStoragePolicy:
raise TypeError("Can't instantiate BaseStoragePolicy directly")
# policy parameter validation
try:
self.idx = int(idx)
except ValueError:
raise PolicyError('Invalid index', idx)
if self.idx < 0:
raise PolicyError('Invalid index', idx)
self.alias_list = []
self.add_name(name)
if aliases:
names_list = list_from_csv(aliases)
for alias in names_list:
if alias == name:
continue
self.add_name(alias)
self.is_deprecated = config_true_value(is_deprecated)
self.is_default = config_true_value(is_default)
if self.policy_type not in BaseStoragePolicy.policy_type_to_policy_cls:
raise PolicyError('Invalid type', self.policy_type)
if self.is_deprecated and self.is_default:
raise PolicyError('Deprecated policy can not be default. '
'Invalid config', self.idx)
self.ring_name = _get_policy_string('object', self.idx)
self.object_ring = object_ring
def __init__(self, app, conf, logger=None):
self.app = app
self.logger = logger or get_logger(conf, log_route='read_only')
self.read_only = config_true_value(conf.get('read_only'))
self.write_methods = {'COPY', 'POST', 'PUT'}
if not config_true_value(conf.get('allow_deletes')):
self.write_methods.add('DELETE')
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='container-updater')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.interval = int(conf.get('interval', 300))
self.account_ring = None
self.concurrency = int(conf.get('concurrency', 4))
self.slowdown = float(conf.get('slowdown', 0.01))
self.node_timeout = float(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.no_changes = 0
self.successes = 0
self.failures = 0
self.account_suppressions = {}
self.account_suppression_time = \
float(conf.get('account_suppression_time', 60))
self.new_account_suppressions = None
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "container.recon")
self.user_agent = 'container-updater %s' % os.getpid()
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = logger or get_logger(conf, log_route='replicator')
self.root = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.port = int(conf.get('bind_port', self.default_port))
concurrency = int(conf.get('concurrency', 8))
self.cpool = GreenPool(size=concurrency)
swift_dir = conf.get('swift_dir', '/etc/swift')
self.ring = ring.Ring(swift_dir, ring_name=self.server_type)
self._local_device_ids = set()
self.per_diff = int(conf.get('per_diff', 1000))
self.max_diffs = int(conf.get('max_diffs') or 100)
self.interval = int(conf.get('interval') or
conf.get('run_pause') or 30)
self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self._zero_stats()
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.recon_replicator = '%s.recon' % self.server_type
self.rcache = os.path.join(self.recon_cache_path,
self.recon_replicator)
self.extract_device_re = re.compile('%s%s([^%s]+)' % (
self.root, os.path.sep, os.path.sep))
def __init__(self, app, conf):
self.app = app
self.conf = conf
self.logger = get_logger(conf, log_route='kerbauth')
self.log_headers = config_true_value(conf.get('log_headers', 'f'))
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
if self.reseller_prefix and self.reseller_prefix[-1] != '_':
self.reseller_prefix += '_'
self.logger.set_statsd_prefix('kerbauth.%s' % (
self.reseller_prefix if self.reseller_prefix else 'NONE',))
self.auth_prefix = conf.get('auth_prefix', '/auth/')
if not self.auth_prefix or not self.auth_prefix.strip('/'):
self.logger.warning('Rewriting invalid auth prefix "%s" to '
'"/auth/" (Non-empty auth prefix path '
'is required)' % self.auth_prefix)
self.auth_prefix = '/auth/'
if self.auth_prefix[0] != '/':
self.auth_prefix = '/' + self.auth_prefix
if self.auth_prefix[-1] != '/':
self.auth_prefix += '/'
self.token_life = int(conf.get('token_life', 86400))
self.allow_overrides = config_true_value(
conf.get('allow_overrides', 't'))
self.storage_url_scheme = conf.get('storage_url_scheme', 'default')
self.ext_authentication_url = conf.get('ext_authentication_url')
if not self.ext_authentication_url:
raise RuntimeError("Missing filter parameter ext_authentication_"
"url in /etc/swift/proxy-server.conf")
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
ns = conf.get('sds_namespace')
acct = conf.get('sds_default_account')
proxy = conf.get('sds_proxy_url')
if ns is None:
raise ConfigurationException('No OIO-SDS namespace configured')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
if proxy is None:
raise ConfigurationException('No OIO-SDS proxy URL configured')
strip_v1 = config_true_value(local_config.pop('strip_v1', False))
account_first = config_true_value(local_config.pop('account_first', False))
skip_metadata = config_true_value(local_config.pop('skip_metadata', False))
def factory(app):
return HashedContainerMiddleware(app, ns, acct, proxy,
strip_v1=strip_v1,
account_first=account_first,
skip_metadata=skip_metadata,
**local_config)
return factory
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
acct = conf.get('sds_default_account')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
account_first = config_true_value(local_config.get('account_first'))
failsafe = config_true_value(local_config.get('failsafe'))
swift3_compat = config_true_value(local_config.get('swift3_compat'))
strip_v1 = config_true_value(local_config.get('strip_v1'))
# By default this is enabled, to be compatible with openio-sds < 4.2.
stop_at_first_match = config_true_value(
local_config.get('stop_at_first_match', True))
pattern_dict = {k: v for k, v in local_config.items()
if k.startswith("pattern")}
def factory(app):
patterns = [pattern_dict[k] for k in sorted(pattern_dict.keys())]
logger = get_logger(conf)
logger.info("Using patterns %s", patterns)
return RegexContainerMiddleware(
app, acct, patterns,
strip_v1=strip_v1, account_first=account_first,
swift3_compat=swift3_compat,
stop_at_first_match=stop_at_first_match,
failsafe=failsafe)
return factory
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='profile')
self.log_filename_prefix = conf.get('log_filename_prefix',
DEFAULT_PROFILE_PREFIX)
dirname = os.path.dirname(self.log_filename_prefix)
# Notes: this effort may fail due to permission denied.
# it is better to be created and authorized to current
# user in advance.
if not os.path.exists(dirname):
os.makedirs(dirname)
self.dump_interval = float(conf.get('dump_interval', 5.0))
self.dump_timestamp = config_true_value(conf.get(
'dump_timestamp', 'no'))
self.flush_at_shutdown = config_true_value(conf.get(
'flush_at_shutdown', 'no'))
self.path = conf.get('path', '__profile__').replace('/', '')
self.unwind = config_true_value(conf.get('unwind', 'no'))
self.profile_module = conf.get('profile_module',
'eventlet.green.profile')
self.profiler = get_profiler(self.profile_module)
self.profile_log = ProfileLog(self.log_filename_prefix,
self.dump_timestamp)
self.viewer = HTMLViewer(self.path, self.profile_module,
self.profile_log)
self.dump_pool = GreenPool(1000)
self.last_dump_at = None
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = logger or get_logger(conf, log_route='account-reaper')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.interval = int(conf.get('interval', 3600))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring = None
self.container_ring = None
self.object_ring = None
self.node_timeout = float(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.myips = whataremyips(conf.get('bind_ip', '0.0.0.0'))
self.bind_port = int(conf.get('bind_port', 6202))
self.concurrency = int(conf.get('concurrency', 25))
self.container_concurrency = self.object_concurrency = \
sqrt(self.concurrency)
self.container_pool = GreenPool(size=self.container_concurrency)
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
self.delay_reaping = int(conf.get('delay_reaping') or 0)
reap_warn_after = float(conf.get('reap_warn_after') or 86400 * 30)
self.reap_not_done_after = reap_warn_after + self.delay_reaping
self.start_time = time()
self.reset_stats()
def __init__(self, conf):
self.logger = get_logger(conf, log_route='container-server')
self.root = conf.get('devices', '/srv/node/')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
replication_server = conf.get('replication_server', None)
if replication_server is None:
allowed_methods = ['DELETE', 'PUT', 'HEAD', 'GET', 'REPLICATE',
'POST']
else:
replication_server = config_true_value(replication_server)
if replication_server:
allowed_methods = ['REPLICATE']
else:
allowed_methods = ['DELETE', 'PUT', 'HEAD', 'GET', 'POST']
self.replication_server = replication_server
self.allowed_methods = allowed_methods
self.allowed_sync_hosts = [
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()]
self.replicator_rpc = ReplicatorRpc(
self.root, DATADIR, ContainerBroker, self.mount_check,
logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
if config_true_value(conf.get('allow_versions', 'f')):
self.save_headers.append('x-versions-location')
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
def __init__(self, conf, logger=None):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = logger or get_logger(conf, log_route="object-reconstructor")
self.devices_dir = conf.get("devices", "/srv/node")
self.mount_check = config_true_value(conf.get("mount_check", "true"))
self.swift_dir = conf.get("swift_dir", "/etc/swift")
self.bind_ip = conf.get("bind_ip", "0.0.0.0")
self.servers_per_port = int(conf.get("servers_per_port", "0") or 0)
self.port = None if self.servers_per_port else int(conf.get("bind_port", 6000))
self.concurrency = int(conf.get("concurrency", 1))
self.stats_interval = int(conf.get("stats_interval", "300"))
self.ring_check_interval = int(conf.get("ring_check_interval", 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get("reclaim_age", 86400 * 7))
self.partition_times = []
self.interval = int(conf.get("interval") or conf.get("run_pause") or 30)
self.http_timeout = int(conf.get("http_timeout", 60))
self.lockup_timeout = int(conf.get("lockup_timeout", 1800))
self.recon_cache_path = conf.get("recon_cache_path", "/var/cache/swift")
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
# defaults subject to change after beta
self.conn_timeout = float(conf.get("conn_timeout", 0.5))
self.node_timeout = float(conf.get("node_timeout", 10))
self.network_chunk_size = int(conf.get("network_chunk_size", 65536))
self.disk_chunk_size = int(conf.get("disk_chunk_size", 65536))
self.headers = {"Content-Length": "0", "user-agent": "obj-reconstructor %s" % os.getpid()}
self.handoffs_first = config_true_value(conf.get("handoffs_first", False))
self._df_router = DiskFileRouter(conf, self.logger)
def __init__(self, conf, logger=None):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = logger or get_logger(conf, log_route='object-replicator')
self.devices_dir = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.bind_ip = conf.get('bind_ip', '0.0.0.0')
self.servers_per_port = int(conf.get('servers_per_port', '0') or 0)
self.port = None if self.servers_per_port else \
int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('concurrency', 1))
self.stats_interval = int(conf.get('stats_interval', '300'))
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.partition_times = []
self.interval = int(conf.get('interval') or
conf.get('run_pause') or 30)
self.rsync_timeout = int(conf.get('rsync_timeout', 900))
self.rsync_io_timeout = conf.get('rsync_io_timeout', '30')
self.rsync_bwlimit = conf.get('rsync_bwlimit', '0')
self.rsync_compress = config_true_value(
conf.get('rsync_compress', 'no'))
self.rsync_module = conf.get('rsync_module', '').rstrip('/')
if not self.rsync_module:
self.rsync_module = '{replication_ip}::object'
if config_true_value(conf.get('vm_test_mode', 'no')):
self.logger.warn('Option object-replicator/vm_test_mode is '
'deprecated and will be removed in a future '
'version. Update your configuration to use '
'option object-replicator/rsync_module.')
self.rsync_module += '{replication_port}'
self.http_timeout = int(conf.get('http_timeout', 60))
self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.node_timeout = float(conf.get('node_timeout', 10))
self.sync_method = getattr(self, conf.get('sync_method') or 'rsync')
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
self.default_headers = {
'Content-Length': '0',
'user-agent': 'object-replicator %s' % os.getpid()}
self.rsync_error_log_line_length = \
int(conf.get('rsync_error_log_line_length', 0))
self.handoffs_first = config_true_value(conf.get('handoffs_first',
False))
self.handoff_delete = config_auto_int_value(
conf.get('handoff_delete', 'auto'), 0)
if any((self.handoff_delete, self.handoffs_first)):
self.logger.warn('Handoff only mode is not intended for normal '
'operation, please disable handoffs_first and '
'handoff_delete before the next '
'normal rebalance')
self._diskfile_mgr = DiskFileManager(conf, self.logger)
def __init__(self, conf):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = get_logger(conf, log_route="object-replicator")
self.devices_dir = conf.get("devices", "/srv/node")
self.mount_check = config_true_value(conf.get("mount_check", "true"))
self.vm_test_mode = config_true_value(conf.get("vm_test_mode", "no"))
self.swift_dir = conf.get("swift_dir", "/etc/swift")
self.port = int(conf.get("bind_port", 6000))
self.concurrency = int(conf.get("concurrency", 1))
self.stats_interval = int(conf.get("stats_interval", "300"))
self.object_ring = Ring(self.swift_dir, ring_name="object")
self.ring_check_interval = int(conf.get("ring_check_interval", 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get("reclaim_age", 86400 * 7))
self.partition_times = []
self.run_pause = int(conf.get("run_pause", 30))
self.rsync_timeout = int(conf.get("rsync_timeout", 900))
self.rsync_io_timeout = conf.get("rsync_io_timeout", "30")
self.http_timeout = int(conf.get("http_timeout", 60))
self.lockup_timeout = int(conf.get("lockup_timeout", 1800))
self.recon_cache_path = conf.get("recon_cache_path", "/var/cache/swift")
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
def __init__(self, conf):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = get_logger(conf, log_route='object-mover')
self.devices_dir = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.bind_ip = conf.get('bind_ip', '0.0.0.0')
self.servers_per_port = int(conf.get('servers_per_port', '0') or 0)
self.port = None if self.servers_per_port else \
int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('concurrency', 1))
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.handoffs_first = config_true_value(conf.get('handoffs_first',
False))
self.data_moving_map_dump = (conf.get('data_moving_map_dump')
or DEFAULT_DUMP_FILE)
self._diskfile_mgr = DiskFileManager(conf, self.logger)
self.mover_tmp_dir = (conf.get('mover_tmp_dir') or 'data_mover')
self.retries = int(conf.get('retries', 3))
self.test = bool(conf.get('test', False))
self.retrie_list = []
def __init__(self, app, conf):
"""
This function is called when Swift Proxy inits.
"""
self.app = app
self.conf = conf
self.logger = get_logger(conf, log_route='customauth')
self.log_headers = config_true_value(conf.get('log_headers', 'f'))
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
if self.reseller_prefix and self.reseller_prefix[-1] != '_':
self.reseller_prefix += '_'
self.logger.set_statsd_prefix('customauth.%s' % (
self.reseller_prefix if self.reseller_prefix else 'NONE',))
self.auth_prefix = conf.get('auth_prefix', '/auth/')
#Organization
self.organization_id = conf.get('organization_id', '57b69c457792482c8d817c4945c6c8a8')
#Keystone
self.keystone_auth_endpoint = conf.get('keystone_auth_endpoint', 'http://cloud.lab.fiware.org:4730/v2.0/tokens')
self.keystone_tenant_endpoint = conf.get('keystone_tenant_endpoint', 'http://cloud.lab.fiware.org:4730/v2.0/tenants')
if not self.auth_prefix or not self.auth_prefix.strip('/'):
self.logger.warning('Rewriting invalid auth prefix "%s" to '
'"/auth/" (Non-empty auth prefix path '
'is required)' % self.auth_prefix)
self.auth_prefix = '/auth/'
if self.auth_prefix[0] != '/':
self.auth_prefix = '/' + self.auth_prefix
if self.auth_prefix[-1] != '/':
self.auth_prefix += '/'
self.token_life = int(conf.get('token_life', 86400))
self.allow_overrides = config_true_value(
conf.get('allow_overrides', 't'))
self.storage_url_scheme = conf.get('storage_url_scheme', 'default')
self.logger.info('CustomAuth v1.3 loaded successfully')
def __init__(self, conf, logger=None):
# location/directory of the metadata database (meta.db)
self.location = conf.get('location', '/srv/node/sdb1/metadata/')
# path the the actual file
self.db_file = os.path.join(self.location, 'meta.db')
self.logger = logger or get_logger(conf, log_route='metadata-server')
self.root = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.node_timeout = int(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('node_timeout', 3))
replication_server = conf.get('replication_server', None)
if replication_server is not None:
replication_server = config_true_value(replication_server)
self.replication_server = replication_server
self.allowed_sync_hosts = [
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()
]
self.replicator_rpc = ReplicatorRpc(
self.root,
DATADIR,
MetadataBroker,
self.mount_check,
logger=self.logger
)
if config_true_value(conf.get('allow_versions', 'f')):
self.save_headers.append('x-versions-location')
swift.common.db.DB_PREALLOCATION = config_true_value(
conf.get('db_preallocation', 'f'))
def __init__(self, conf, logger=None):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = logger or get_logger(
conf, log_route='object-reconstructor')
self.devices_dir = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('concurrency', 1))
self.stats_interval = int(conf.get('stats_interval', '300'))
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.partition_times = []
self.run_pause = int(conf.get('run_pause', 30))
self.http_timeout = int(conf.get('http_timeout', 60))
self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
# defaults subject to change after beta
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.node_timeout = float(conf.get('node_timeout', 10))
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.headers = {
'Content-Length': '0',
'user-agent': 'obj-reconstructor %s' % os.getpid()}
self.handoffs_first = config_true_value(conf.get('handoffs_first',
False))
self._df_router = DiskFileRouter(conf, self.logger)
def filter_factory(global_conf, **local_config):
conf = global_conf.copy()
conf.update(local_config)
global LOG
LOG = get_logger(conf)
acct = conf.get('sds_default_account')
if acct is None:
raise ConfigurationException('No OIO-SDS account configured')
account_first = config_true_value(local_config.get('account_first'))
swift3_compat = config_true_value(local_config.get('swift3_compat'))
strip_v1 = config_true_value(local_config.get('strip_v1'))
redis_keys_format = local_config.get('redis_keys_format',
REDIS_KEYS_FORMAT_V1)
redis_host = local_config.get('redis_host')
sentinel_hosts = local_config.get('sentinel_hosts')
sentinel_name = local_config.get('sentinel_name')
def factory(app):
return ContainerHierarchyMiddleware(
app, global_conf, acct,
strip_v1=strip_v1,
account_first=account_first,
swift3_compat=swift3_compat,
redis_keys_format=redis_keys_format,
redis_host=redis_host,
sentinel_hosts=sentinel_hosts,
sentinel_name=sentinel_name)
return factory
def __init__(self, conf):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = get_logger(conf, log_route='object-replicator')
self.devices_dir = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('concurrency', 1))
self.stats_interval = int(conf.get('stats_interval', '300'))
self.object_ring = Ring(self.swift_dir, ring_name='object')
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.partition_times = []
self.run_pause = int(conf.get('run_pause', 30))
self.rsync_timeout = int(conf.get('rsync_timeout', 900))
self.rsync_io_timeout = conf.get('rsync_io_timeout', '30')
self.rsync_bwlimit = conf.get('rsync_bwlimit', '0')
self.http_timeout = int(conf.get('http_timeout', 60))
self.lockup_timeout = int(conf.get('lockup_timeout', 1800))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
self.headers = {
'Content-Length': '0',
'user-agent': 'obj-replicator %s' % os.getpid()}
self.rsync_error_log_line_length = \
int(conf.get('rsync_error_log_line_length', 0))
def __init__(self, conf):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = get_logger(conf, log_route="object-replicator")
self.devices_dir = conf.get("devices", "/srv/node")
self.mount_check = config_true_value(conf.get("mount_check", "true"))
self.vm_test_mode = config_true_value(conf.get("vm_test_mode", "no"))
self.swift_dir = conf.get("swift_dir", "/etc/swift")
self.port = int(conf.get("bind_port", 6000))
self.concurrency = int(conf.get("concurrency", 1))
self.stats_interval = int(conf.get("stats_interval", "300"))
self.ring_check_interval = int(conf.get("ring_check_interval", 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get("reclaim_age", 86400 * 7))
self.partition_times = []
self.run_pause = int(conf.get("run_pause", 30))
self.rsync_timeout = int(conf.get("rsync_timeout", 900))
self.rsync_io_timeout = conf.get("rsync_io_timeout", "30")
self.rsync_bwlimit = conf.get("rsync_bwlimit", "0")
self.http_timeout = int(conf.get("http_timeout", 60))
self.lockup_timeout = int(conf.get("lockup_timeout", 1800))
self.recon_cache_path = conf.get("recon_cache_path", "/var/cache/swift")
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
self.conn_timeout = float(conf.get("conn_timeout", 0.5))
self.node_timeout = float(conf.get("node_timeout", 10))
self.sync_method = getattr(self, conf.get("sync_method") or "rsync")
self.network_chunk_size = int(conf.get("network_chunk_size", 65536))
self.headers = {"Content-Length": "0", "user-agent": "object-replicator %s" % os.getpid()}
self.rsync_error_log_line_length = int(conf.get("rsync_error_log_line_length", 0))
self.handoffs_first = config_true_value(conf.get("handoffs_first", False))
self.handoff_delete = config_auto_int_value(conf.get("handoff_delete", "auto"), 0)
self._diskfile_mgr = DiskFileManager(conf, self.logger)
def __init__(self, conf):
"""
Creates a new WSGI application for the Swift Object Server. An
example configuration is given at
<source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample.
"""
self.logger = get_logger(conf, log_route="object-server")
self.devices = conf.get("devices", "/srv/node/")
self.mount_check = config_true_value(conf.get("mount_check", "true"))
self.node_timeout = int(conf.get("node_timeout", 3))
self.conn_timeout = float(conf.get("conn_timeout", 0.5))
self.disk_chunk_size = int(conf.get("disk_chunk_size", 65536))
self.network_chunk_size = int(conf.get("network_chunk_size", 65536))
self.keep_cache_size = int(conf.get("keep_cache_size", 5242880))
self.keep_cache_private = config_true_value(conf.get("keep_cache_private", "false"))
self.log_requests = config_true_value(conf.get("log_requests", "true"))
self.max_upload_time = int(conf.get("max_upload_time", 86400))
self.slow = int(conf.get("slow", 0))
self.bytes_per_sync = int(conf.get("mb_per_sync", 512)) * 1024 * 1024
default_allowed_headers = """
content-disposition,
content-encoding,
x-delete-at,
x-object-manifest,
"""
self.allowed_headers = set(
i.strip().lower()
for i in conf.get("allowed_headers", default_allowed_headers).split(",")
if i.strip() and i.strip().lower() not in DISALLOWED_HEADERS
)
self.expiring_objects_account = (conf.get("auto_create_account_prefix") or ".") + "expiring_objects"
self.expiring_objects_container_divisor = int(conf.get("expiring_objects_container_divisor") or 86400)
def container_request(self, req, start_response, enabled):
# set version location header as sysmeta
if VERSIONS_LOC_CLIENT in req.headers:
val = req.headers.get(VERSIONS_LOC_CLIENT)
if val:
# differently from previous version, we are actually
# returning an error if user tries to set versions location
# while feature is explicitly disabled.
if not config_true_value(enabled) and req.method in ("PUT", "POST"):
raise HTTPPreconditionFailed(
request=req, content_type="text/plain", body="Versioned Writes is disabled"
)
location = check_container_format(req, val)
req.headers[VERSIONS_LOC_SYSMETA] = location
# reset original header to maintain sanity
# now only sysmeta is source of Versions Location
req.headers[VERSIONS_LOC_CLIENT] = ""
# if both headers are in the same request
# adding location takes precedence over removing
if "X-Remove-Versions-Location" in req.headers:
del req.headers["X-Remove-Versions-Location"]
else:
# empty value is the same as X-Remove-Versions-Location
req.headers["X-Remove-Versions-Location"] = "x"
# handle removing versions container
val = req.headers.get("X-Remove-Versions-Location")
if val:
req.headers.update({VERSIONS_LOC_SYSMETA: "", VERSIONS_LOC_CLIENT: ""})
del req.headers["X-Remove-Versions-Location"]
# handle versioning mode
if VERSIONS_MODE_CLIENT in req.headers:
val = req.headers.pop(VERSIONS_MODE_CLIENT)
if val:
if not config_true_value(enabled) and req.method in ("PUT", "POST"):
raise HTTPPreconditionFailed(
request=req, content_type="text/plain", body="Versioned Writes is disabled"
)
if val not in VERSIONING_MODES:
raise HTTPBadRequest(
request=req,
content_type="text/plain",
body="X-Versions-Mode must be one of %s" % ", ".join(VERSIONING_MODES),
)
req.headers[VERSIONS_MODE_SYSMETA] = val
else:
req.headers["X-Remove-Versions-Mode"] = "x"
if req.headers.pop("X-Remove-Versions-Mode", None):
req.headers.update({VERSIONS_MODE_SYSMETA: ""})
# send request and translate sysmeta headers from response
vw_ctx = VersionedWritesContext(self.app, self.logger)
return vw_ctx.handle_container_request(req.environ, start_response)
def __init__(self, conf, logger=None):
"""
Creates a new WSGI application for the Swift Object Server. An
example configuration is given at
<source-dir>/etc/object-server.conf-sample or
/etc/swift/object-server.conf-sample.
"""
self.logger = logger or get_logger(conf, log_route="object-server")
self.node_timeout = int(conf.get("node_timeout", 3))
self.conn_timeout = float(conf.get("conn_timeout", 0.5))
self.client_timeout = int(conf.get("client_timeout", 60))
self.disk_chunk_size = int(conf.get("disk_chunk_size", 65536))
self.network_chunk_size = int(conf.get("network_chunk_size", 65536))
self.log_requests = config_true_value(conf.get("log_requests", "true"))
self.max_upload_time = int(conf.get("max_upload_time", 86400))
self.slow = int(conf.get("slow", 0))
self.keep_cache_private = config_true_value(conf.get("keep_cache_private", "false"))
replication_server = conf.get("replication_server", None)
if replication_server is not None:
replication_server = config_true_value(replication_server)
self.replication_server = replication_server
default_allowed_headers = """
content-disposition,
content-encoding,
x-delete-at,
x-object-manifest,
x-static-large-object,
"""
extra_allowed_headers = [
header.strip().lower()
for header in conf.get("allowed_headers", default_allowed_headers).split(",")
if header.strip()
]
self.allowed_headers = set()
for header in extra_allowed_headers:
if header not in DATAFILE_SYSTEM_META:
self.allowed_headers.add(header)
self.expiring_objects_account = (conf.get("auto_create_account_prefix") or ".") + "expiring_objects"
self.expiring_objects_container_divisor = int(conf.get("expiring_objects_container_divisor") or 86400)
# Initialization was successful, so now apply the network chunk size
# parameter as the default read / write buffer size for the network
# sockets.
#
# NOTE WELL: This is a class setting, so until we get set this on a
# per-connection basis, this affects reading and writing on ALL
# sockets, those between the proxy servers and external clients, and
# those between the proxy servers and the other internal servers.
#
# ** Because the primary motivation for this is to optimize how data
# is written back to the proxy server, we could use the value from the
# disk_chunk_size parameter. However, it affects all created sockets
# using this class so we have chosen to tie it to the
# network_chunk_size parameter value instead.
socket._fileobject.default_bufsize = self.network_chunk_size
# Provide further setup sepecific to an object server implemenation.
self.setup(conf)
def __init__(self, conf, container_ring=None, object_ring=None):
#: The dict of configuration values from the [container-sync] section
#: of the container-server.conf.
self.conf = conf
#: Logger to use for container-sync log lines.
self.logger = get_logger(conf, log_route='container-sync')
#: Path to the local device mount points.
self.devices = conf.get('devices', '/srv/node')
#: Indicates whether mount points should be verified as actual mount
#: points (normally true, false for tests and SAIO).
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
#: Minimum time between full scans. This is to keep the daemon from
#: running wild on near empty systems.
self.interval = int(conf.get('interval', 300))
#: Maximum amount of time to spend syncing a container before moving on
#: to the next one. If a conatiner sync hasn't finished in this time,
#: it'll just be resumed next scan.
self.container_time = int(conf.get('container_time', 60))
#: ContainerSyncCluster instance for validating sync-to values.
self.realms_conf = ContainerSyncRealms(
os.path.join(
conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'),
self.logger)
#: The list of hosts we're allowed to send syncs to. This can be
#: overridden by data in self.realms_conf
self.allowed_sync_hosts = [
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()]
self.http_proxies = [
a.strip()
for a in conf.get('sync_proxy', '').split(',')
if a.strip()]
#: Number of containers with sync turned on that were successfully
#: synced.
self.container_syncs = 0
#: Number of successful DELETEs triggered.
self.container_deletes = 0
#: Number of successful PUTs triggered.
self.container_puts = 0
#: Number of containers that didn't have sync turned on.
self.container_skips = 0
#: Number of containers that had a failure of some type.
self.container_failures = 0
#: Time of last stats report.
self.reported = time()
swift_dir = conf.get('swift_dir', '/etc/swift')
#: swift.common.ring.Ring for locating containers.
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
#: swift.common.ring.Ring for locating objects.
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self._myips = whataremyips()
self._myport = int(conf.get('bind_port', 6001))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
def __init__(self, conf, logger=None):
super(AccountController, self).__init__(conf)
self.logger = logger or get_logger(conf, log_route="account-server")
self.log_requests = config_true_value(conf.get("log_requests", "true"))
self.root = conf.get("devices", "/srv/node")
self.mount_check = config_true_value(conf.get("mount_check", "true"))
self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger)
self.auto_create_account_prefix = conf.get("auto_create_account_prefix") or "."
swift.common.db.DB_PREALLOCATION = config_true_value(conf.get("db_preallocation", "f"))
def container_request(self, req, start_response, enabled):
sysmeta_version_hdr = get_sys_meta_prefix('container') + \
'versions-location'
# set version location header as sysmeta
if 'X-Versions-Location' in req.headers:
val = req.headers.get('X-Versions-Location')
if val:
# diferently from previous version, we are actually
# returning an error if user tries to set versions location
# while feature is explicitly disabled.
if not config_true_value(enabled) and \
req.method in ('PUT', 'POST'):
raise HTTPPreconditionFailed(
request=req,
content_type='text/plain',
body='Versioned Writes is disabled')
location = check_container_format(req, val)
req.headers[sysmeta_version_hdr] = location
# reset original header to maintain sanity
# now only sysmeta is source of Versions Location
req.headers['X-Versions-Location'] = ''
# if both headers are in the same request
# adding location takes precendence over removing
if 'X-Remove-Versions-Location' in req.headers:
del req.headers['X-Remove-Versions-Location']
else:
# empty value is the same as X-Remove-Versions-Location
req.headers['X-Remove-Versions-Location'] = 'x'
# handle removing versions container
val = req.headers.get('X-Remove-Versions-Location')
if val:
req.headers.update({sysmeta_version_hdr: ''})
req.headers.update({'X-Versions-Location': ''})
del req.headers['X-Remove-Versions-Location']
# send request and translate sysmeta headers from response
vw_ctx = VersionedWritesContext(self.app, self.logger)
return vw_ctx.handle_container_request(req.environ, start_response)
def __init__(self, config):
for key in 'auth_uri username password'.split():
if key not in config:
raise SkipTest(
"Missing required configuration parameter: %s" % key)
self.auth_url = config['auth_uri']
self.insecure = config_true_value(config.get('insecure', 'false'))
self.auth_version = str(config.get('auth_version', '1'))
self.domain = config.get('domain')
self.account = config.get('account')
self.username = config['username']
self.password = config['password']
self.storage_netloc = None
self.storage_path = None
self.conn_class = None
self.connection = None # until you call .http_connect()
def connect(self):
"""
Establishes a connection and starts an SSYNC request
with the object server.
"""
connection = response = None
node_addr = '%s:%s' % (self.node['replication_ip'],
self.node['replication_port'])
with exceptions.MessageTimeout(self.daemon.conn_timeout,
'connect send'):
connection = SsyncBufferedHTTPConnection(node_addr)
connection.putrequest(
'SSYNC',
'/%s/%s' % (self.node['device'], self.job['partition']))
connection.putheader('Transfer-Encoding', 'chunked')
connection.putheader('X-Backend-Storage-Policy-Index',
int(self.job['policy']))
# a sync job must use the node's backend_index for the frag_index
# of the rebuilt fragments instead of the frag_index from the job
# which will be rebuilding them
frag_index = self.node.get('backend_index')
if frag_index is not None:
connection.putheader('X-Backend-Ssync-Frag-Index', frag_index)
# Node-Index header is for backwards compat 2.4.0-2.20.0
connection.putheader('X-Backend-Ssync-Node-Index', frag_index)
connection.endheaders()
with exceptions.MessageTimeout(self.daemon.node_timeout,
'connect receive'):
response = connection.getresponse()
if response.status != http.HTTP_OK:
err_msg = response.read()[:1024]
raise exceptions.ReplicationException(
'Expected status %s; got %s (%s)' %
(http.HTTP_OK, response.status, err_msg))
if self.include_non_durable and not config_true_value(
response.getheader('x-backend-accept-no-commit', False)):
# fall back to legacy behaviour if receiver does not understand
# X-Backend-Commit
self.daemon.logger.warning(
'ssync receiver %s does not accept non-durable fragments' %
node_addr)
self.include_non_durable = False
return connection, response
def __init__(self, app, conf):
self.app = app
storage_domain = conf.get('storage_domain', 'example.com')
self.storage_domain = [
'.' + s for s in list_from_csv(storage_domain)
if not s.startswith('.')
]
self.storage_domain += [
s for s in list_from_csv(storage_domain) if s.startswith('.')
]
self.path_root = conf.get('path_root', 'v1').strip('/') + '/'
prefixes = conf.get('reseller_prefixes', 'AUTH')
self.reseller_prefixes = list_from_csv(prefixes)
self.reseller_prefixes_lower = [
x.lower() for x in self.reseller_prefixes
]
self.default_reseller_prefix = conf.get('default_reseller_prefix')
self.mangle_client_paths = config_true_value(
conf.get('mangle_client_paths'))
def __init__(self, *args, **kwargs):
swob.Response.__init__(self, *args, **kwargs)
if self.etag:
# add double quotes to the etag header
self.etag = self.etag
sw_sysmeta_headers = swob.HeaderKeyDict()
sw_headers = swob.HeaderKeyDict()
headers = HeaderKeyDict()
self.is_slo = False
for key, val in self.headers.iteritems():
_key = key.lower()
if _key.startswith(sysmeta_prefix('object')) or \
_key.startswith(sysmeta_prefix('container')):
sw_sysmeta_headers[key] = val
else:
sw_headers[key] = val
# Handle swift headers
for key, val in sw_headers.iteritems():
_key = key.lower()
if _key.startswith('x-object-meta-'):
# Note that AWS ignores user-defined headers with '=' in the
# header name. We translated underscores to '=5F' on the way
# in, though.
headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
elif _key in ('content-length', 'content-type', 'content-range',
'content-encoding', 'content-disposition',
'content-language', 'etag', 'last-modified',
'x-robots-tag', 'cache-control', 'expires'):
headers[key] = val
elif _key == 'x-static-large-object':
# for delete slo
self.is_slo = config_true_value(val)
self.headers = headers
# Used for pure swift header handling at the request layer
self.sw_headers = sw_headers
self.sysmeta_headers = sw_sysmeta_headers
def test_put_bucket_error_key2(self):
if config_true_value(tf.cluster_info['s3api'].get('s3_acl')):
if 's3_access_key2' not in tf.config or \
's3_secret_key2' not in tf.config:
raise tf.SkipTest(
'Cannot test for BucketAlreadyExists with second user; '
'need s3_access_key2 and s3_secret_key2 configured')
self.conn.create_bucket(Bucket='bucket')
# Other users of the same account get the same 409 error
conn2 = get_boto3_conn(tf.config['s3_access_key2'],
tf.config['s3_secret_key2'])
with self.assertRaises(botocore.exceptions.ClientError) as ctx:
conn2.create_bucket(Bucket='bucket')
self.assertEqual(
ctx.exception.response['ResponseMetadata']['HTTPStatusCode'],
409)
self.assertEqual(ctx.exception.response['Error']['Code'],
'BucketAlreadyExists')
def __init__(self, config):
for key in 'auth_host auth_port auth_ssl username password'.split():
if key not in config:
raise SkipTest("Missing required configuration parameter: %s" %
key)
self.auth_host = config['auth_host']
self.auth_port = int(config['auth_port'])
self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1')
self.insecure = config_true_value(config.get('insecure', 'false'))
self.auth_prefix = config.get('auth_prefix', '/')
self.auth_version = str(config.get('auth_version', '1'))
self.account = config.get('account')
self.username = config['username']
self.password = config['password']
self.storage_netloc = None
self.storage_path = None
self.conn_class = None
def __init__(self, app, wsgi_conf, *args, **kwargs):
self.app = app
self.conf = Config()
# Set default values if they are not configured
self.conf.allow_no_owner = config_true_value(
wsgi_conf.get('allow_no_owner', False))
self.conf.location = wsgi_conf.get('location', 'us-east-1')
self.conf.dns_compliant_bucket_names = config_true_value(
wsgi_conf.get('dns_compliant_bucket_names', True))
self.conf.max_bucket_listing = config_positive_int_value(
wsgi_conf.get('max_bucket_listing', 1000))
self.conf.max_parts_listing = config_positive_int_value(
wsgi_conf.get('max_parts_listing', 1000))
self.conf.max_multi_delete_objects = config_positive_int_value(
wsgi_conf.get('max_multi_delete_objects', 1000))
self.conf.multi_delete_concurrency = config_positive_int_value(
wsgi_conf.get('multi_delete_concurrency', 2))
self.conf.s3_acl = config_true_value(wsgi_conf.get('s3_acl', False))
self.conf.storage_domain = wsgi_conf.get('storage_domain', '')
self.conf.auth_pipeline_check = config_true_value(
wsgi_conf.get('auth_pipeline_check', True))
self.conf.max_upload_part_num = config_positive_int_value(
wsgi_conf.get('max_upload_part_num', 1000))
self.conf.check_bucket_owner = config_true_value(
wsgi_conf.get('check_bucket_owner', False))
self.conf.force_swift_request_proxy_log = config_true_value(
wsgi_conf.get('force_swift_request_proxy_log', False))
self.conf.allow_multipart_uploads = config_true_value(
wsgi_conf.get('allow_multipart_uploads', True))
self.conf.min_segment_size = config_positive_int_value(
wsgi_conf.get('min_segment_size', 5242880))
self.conf.allowable_clock_skew = config_positive_int_value(
wsgi_conf.get('allowable_clock_skew', 15 * 60))
self.conf.cors_preflight_allow_origin = list_from_csv(
wsgi_conf.get('cors_preflight_allow_origin', ''))
if '*' in self.conf.cors_preflight_allow_origin and \
len(self.conf.cors_preflight_allow_origin) > 1:
raise ValueError('if cors_preflight_allow_origin should include '
'all domains, * must be the only entry')
self.logger = get_logger(wsgi_conf,
log_route=wsgi_conf.get('log_name', 's3api'))
self.check_pipeline(wsgi_conf)
def get_bucket_db(conf):
"""
If `bucket_db_enabled` is set in `conf`, get the bucket database,
otherwise return `None`.
If `bucket_db_host` or `bucket_db_sentinel_hosts` are also set in `conf`,
return an instance of `RedisBucketDb`, otherwise return an instance of
`DummyBucketDb`.
"""
db_kwargs = {
k[10:]: v
for k, v in conf.items() if k.startswith('bucket_db_')
}
if config_true_value(db_kwargs.get('enabled', 'false')):
if 'host' in db_kwargs or 'sentinel_hosts' in db_kwargs:
return RedisBucketDb(**db_kwargs)
else:
return DummyBucketDb(**db_kwargs)
return None
def __init__(self, app, conf, logger=None):
self.app = app
self.conf = conf
self.logger = logger or get_logger(conf, log_route='container_sync')
self.realms_conf = ContainerSyncRealms(
os.path.join(conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'), self.logger)
self.allow_full_urls = config_true_value(
conf.get('allow_full_urls', 'true'))
# configure current realm/cluster for /info
self.realm = self.cluster = None
current = conf.get('current', None)
if current:
try:
self.realm, self.cluster = (
p.upper() for p in current.strip('/').split('/'))
except ValueError:
self.logger.error('Invalid current //REALM/CLUSTER (%s)',
current)
self.register_info()
def handle_container(self, env, start_response):
"""
Handles a possible static web request for a container.
:param env: The original WSGI environment dict.
:param start_response: The original WSGI start_response hook.
"""
container_info = self._get_container_info(env)
req = Request(env)
req.acl = container_info['read_acl']
# we checked earlier that swift.authorize is set in env
aresp = env['swift.authorize'](req)
if aresp:
resp = aresp(env, self._start_response)
return self._error_response(resp, env, start_response)
if not self._listings and not self._index:
if config_true_value(env.get('HTTP_X_WEB_MODE', 'f')):
return HTTPNotFound()(env, start_response)
return self.app(env, start_response)
if not env['PATH_INFO'].endswith('/'):
resp = HTTPMovedPermanently(
location=(env['PATH_INFO'] + '/'))
return resp(env, start_response)
if not self._index:
return self._listing(env, start_response)
tmp_env = dict(env)
tmp_env['HTTP_USER_AGENT'] = \
'%s StaticWeb' % env.get('HTTP_USER_AGENT')
tmp_env['swift.source'] = 'SW'
tmp_env['PATH_INFO'] += self._index
resp = self._app_call(tmp_env)
status_int = self._get_status_int()
if status_int == HTTP_NOT_FOUND:
return self._listing(env, start_response)
elif not is_success(self._get_status_int()) and \
not is_redirection(self._get_status_int()):
return self._error_response(resp, env, start_response)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
resp = None
is_enabled = config_true_value(allow_versioned_writes)
container_info = get_container_info(
req.environ, self.app)
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
versions_cont = container_info.get(
'sysmeta', {}).get('versions-location')
if not versions_cont:
versions_cont = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and versions_cont:
is_enabled = True
if is_enabled and versions_cont:
versions_cont = unquote(versions_cont).split('/')[0]
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method == 'PUT':
resp = vw_ctx.handle_obj_versions_put(
req, versions_cont, api_version, account_name,
object_name)
else: # handle DELETE
resp = vw_ctx.handle_obj_versions_delete(
req, versions_cont, api_version, account_name,
container_name, object_name)
if resp:
return resp
else:
return self.app
def _get_storlet_invocation_options(self, req):
options = dict()
filtered_key = ['X-Storlet-Range', 'X-Storlet-Generate-Log']
for key in req.headers:
prefix = 'X-Storlet-'
if key.startswith(prefix) and key not in filtered_key:
new_key = 'storlet_' + \
key[len(prefix):].lower().replace('-', '_')
options[new_key] = req.headers.get(key)
generate_log = req.headers.get('X-Storlet-Generate-Log')
options['generate_log'] = config_true_value(generate_log)
options['scope'] = self.scope
options['file_manager'] = \
SwiftFileManager(self.account, self.storlet_container,
self.storlet_dependency, self.log_container,
self.client_conf_file, self.logger)
return options
def run_server():
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
# Turn off logging requests by the underlying WSGI software.
wsgi.HttpProtocol.log_request = lambda *a: None
# Redirect logging other messages by the underlying WSGI software.
wsgi.HttpProtocol.log_message = \
lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a)
wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)
eventlet.hubs.use_hub(get_hub())
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
eventlet.debug.hub_exceptions(eventlet_debug)
app = loadapp('config:%s' % conf_file,
global_conf={'log_name': log_name})
pool = GreenPool(size=1024)
try:
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
except socket.error, err:
if err[0] != errno.EINVAL:
raise
def __init__(self, app, conf, *args, **kwargs):
self.app = app
self.devices = conf.get('devices', '/srv/node')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.logger = get_logger(conf, log_route='recon')
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.object_recon_cache = os.path.join(self.recon_cache_path,
'object.recon')
self.container_recon_cache = os.path.join(self.recon_cache_path,
'container.recon')
self.account_recon_cache = os.path.join(self.recon_cache_path,
'account.recon')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.rings = [self.account_ring_path, self.container_ring_path]
# include all object ring files (for all policies)
for f in os.listdir(swift_dir):
if f.startswith('object') and f.endswith('ring.gz'):
self.rings.append(os.path.join(swift_dir, f))
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
def __init__(self, app, conf, *args, **kwargs):
self.app = app
self.devices = conf.get('devices', '/srv/node/')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.logger = get_logger(conf, log_route='recon')
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.object_recon_cache = os.path.join(self.recon_cache_path,
'object.recon')
self.container_recon_cache = os.path.join(self.recon_cache_path,
'container.recon')
self.account_recon_cache = os.path.join(self.recon_cache_path,
'account.recon')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.rings = [
self.account_ring_path, self.container_ring_path,
self.object_ring_path
]
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
def make_object_response(self, req, metadata, stream=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.items():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
hash_ = metadata.get('hash')
if hash_ is not None:
hash_ = hash_.lower()
resp.headers['etag'] = hash_
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
resp.last_modified = int(metadata['mtime'])
if stream:
# Whether we are bothered with ranges or not, we wrap the
# stream in order to handle exceptions.
resp.app_iter = StreamRangeIterator(req, stream)
length_ = metadata.get('length')
if length_ is not None:
length_ = int(length_)
resp.content_length = length_
resp.content_encoding = metadata.get('encoding')
resp.accept_ranges = 'bytes'
return resp
def __init__(self, app, req, server_type, ring, partition, path,
backend_headers):
self.app = app
self.ring = ring
self.server_type = server_type
self.partition = partition
self.path = path
self.backend_headers = backend_headers
self.used_nodes = []
self.used_source_etag = ''
# stuff from request
self.req_method = req.method
self.req_path = req.path
self.req_query_string = req.query_string
self.newest = config_true_value(req.headers.get('x-newest', 'f'))
# populated when finding source
self.statuses = []
self.reasons = []
self.bodies = []
self.source_headers = []
def make_object_response(self, req, metadata, stream=None, ranges=None):
conditional_etag = None
if 'X-Backend-Etag-Is-At' in req.headers:
conditional_etag = metadata.get(
req.headers['X-Backend-Etag-Is-At'])
resp = Response(request=req,
conditional_response=True,
conditional_etag=conditional_etag)
if config_true_value(metadata['deleted']):
resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
else:
resp.headers['Content-Type'] = metadata.get(
'mime_type', 'application/octet-stream')
properties = metadata.get('properties')
if properties:
for k, v in properties.iteritems():
if is_sys_or_user_meta('object', k) or \
is_object_transient_sysmeta(k) or \
k.lower() in self.allowed_headers:
resp.headers[str(k)] = v
resp.headers['etag'] = metadata['hash'].lower()
resp.headers['x-object-sysmeta-version-id'] = metadata['version']
ts = Timestamp(metadata['ctime'])
resp.last_modified = math.ceil(float(ts))
if stream:
if ranges:
resp.app_iter = StreamRangeIterator(stream)
else:
resp.app_iter = stream
resp.content_length = int(metadata['length'])
try:
resp.content_encoding = metadata['encoding']
except KeyError:
pass
resp.accept_ranges = 'bytes'
return resp
def get_or_head_response(self, req, resp_headers, resp_iter):
segments = self._get_manifest_read(resp_iter)
slo_etag = None
content_length = None
response_headers = []
for header, value in resp_headers:
lheader = header.lower()
if lheader == SYSMETA_SLO_ETAG:
slo_etag = value
elif lheader == SYSMETA_SLO_SIZE:
content_length = value
elif lheader not in ('etag', 'content-length'):
response_headers.append((header, value))
if slo_etag is None or content_length is None:
etag = md5()
content_length = 0
for seg_dict in segments:
if seg_dict.get('range'):
etag.update('%s:%s;' %
(seg_dict['hash'], seg_dict['range']))
else:
etag.update(seg_dict['hash'])
if config_true_value(seg_dict.get('sub_slo')):
override_bytes_from_content_type(seg_dict,
logger=self.slo.logger)
content_length += self._segment_length(seg_dict)
slo_etag = etag.hexdigest()
response_headers.append(('Content-Length', str(content_length)))
response_headers.append(('Etag', '"%s"' % slo_etag))
if req.method == 'HEAD':
return self._manifest_head_response(req, response_headers)
else:
return self._manifest_get_response(req, content_length,
response_headers, segments)
def get_slo_segments(self, obj_name, req):
"""
Performs a swob.Request and returns the SLO manifest's segments.
:raises HTTPServerError: on unable to load obj_name or
on unable to load the SLO manifest data.
:raises HTTPBadRequest: on not an SLO manifest
:raises HTTPNotFound: on SLO manifest not found
:returns: SLO manifest's segments
"""
vrs, account, _junk = req.split_path(2, 3, True)
new_env = req.environ.copy()
new_env['REQUEST_METHOD'] = 'GET'
del(new_env['wsgi.input'])
new_env['QUERY_STRING'] = 'multipart-manifest=get'
new_env['CONTENT_LENGTH'] = 0
new_env['HTTP_USER_AGENT'] = \
'%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT')
new_env['swift.source'] = 'SLO'
new_env['PATH_INFO'] = (
'/%s/%s/%s' % (
vrs, account,
obj_name.lstrip('/'))).encode('utf-8')
resp = Request.blank('', new_env).get_response(self.app)
if resp.is_success:
if config_true_value(resp.headers.get('X-Static-Large-Object')):
try:
return json.loads(resp.body)
except ValueError:
raise HTTPServerError('Unable to load SLO manifest')
else:
raise HTTPBadRequest('Not an SLO manifest')
elif resp.status_int == HTTP_NOT_FOUND:
raise HTTPNotFound('SLO manifest not found')
elif resp.status_int == HTTP_UNAUTHORIZED:
raise HTTPUnauthorized('401 Unauthorized')
else:
raise HTTPServerError('Unable to load SLO manifest or segment.')
def check_delete_headers(request):
"""
Validate if 'x-delete' headers are have correct values
values should be positive integers and correspond to
a time in the future.
:param request: the swob request object
:returns: HTTPBadRequest in case of invalid values
or None if values are ok
"""
if 'x-delete-after' in request.headers:
try:
x_delete_after = int(request.headers['x-delete-after'])
except ValueError:
raise HTTPBadRequest(request=request,
content_type='text/plain',
body='Non-integer X-Delete-After')
actual_del_time = time.time() + x_delete_after
if actual_del_time < time.time():
raise HTTPBadRequest(request=request,
content_type='text/plain',
body='X-Delete-After in past')
request.headers['x-delete-at'] = utils.normalize_delete_at_timestamp(
actual_del_time)
if 'x-delete-at' in request.headers:
try:
x_delete_at = int(utils.normalize_delete_at_timestamp(
int(request.headers['x-delete-at'])))
except ValueError:
raise HTTPBadRequest(request=request, content_type='text/plain',
body='Non-integer X-Delete-At')
if x_delete_at < time.time() and not utils.config_true_value(
request.headers.get('x-backend-replication', 'f')):
raise HTTPBadRequest(request=request, content_type='text/plain',
body='X-Delete-At in past')
return request
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
storlet_conf = dict()
storlet_conf['storlet_timeout'] = conf.get('storlet_timeout', 40)
storlet_conf['storlet_container'] = conf.get('storlet_container',
'storlet')
storlet_conf['storlet_dependency'] = conf.get('storlet_dependency',
'dependency')
storlet_conf['execution_server'] = conf.get('execution_server', '')
storlet_conf['storlet_execute_on_proxy_only'] = config_true_value(
conf.get('storlet_execute_on_proxy_only', 'false'))
storlet_conf['gateway_conf'] = {}
module_name = conf.get('storlet_gateway_module', '')
mo = module_name[:module_name.rfind(':')]
cl = module_name[module_name.rfind(':') + 1:]
module = __import__(mo, fromlist=[cl])
the_class = getattr(module, cl)
configParser = ConfigParser.RawConfigParser()
configParser.read(
conf.get('storlet_gateway_conf',
'/etc/swift/storlet_stub_gateway.conf'))
additional_items = configParser.items("DEFAULT")
for key, val in additional_items:
storlet_conf[key] = val
swift_info = {}
storlet_conf["gateway_module"] = the_class
register_swift_info('storlet_handler', False, **swift_info)
def storlet_handler_filter(app):
return StorletHandlerMiddleware(app, conf, storlet_conf)
return storlet_handler_filter
def _segment_listing_iterator(self, req, version, account, segments,
byteranges):
for seg_dict in segments:
if config_true_value(seg_dict.get('sub_slo')):
override_bytes_from_content_type(seg_dict,
logger=self.slo.logger)
# We handle the range stuff here so that we can be smart about
# skipping unused submanifests. For example, if our first segment is a
# submanifest referencing 50 MiB total, but start_byte falls in
# the 51st MiB, then we can avoid fetching the first submanifest.
#
# If we were to make SegmentedIterable handle all the range
# calculations, we would be unable to make this optimization.
total_length = sum(self._segment_length(seg) for seg in segments)
if not byteranges:
byteranges = [(0, total_length - 1)]
# Cache segments from sub-SLOs in case more than one byterange
# includes data from a particular sub-SLO. We only cache a few sets
# of segments so that a malicious user cannot build a giant SLO tree
# and then GET it to run the proxy out of memory.
#
# LRUCache is a little awkward to use this way, but it beats doing
# things manually.
#
# 20 is sort of an arbitrary choice; it's twice our max recursion
# depth, so we know this won't expand memory requirements by too
# much.
cached_fetch_sub_slo_segments = \
LRUCache(maxsize=20)(self._fetch_sub_slo_segments)
for first_byte, last_byte in byteranges:
byterange_listing_iter = self._byterange_listing_iterator(
req, version, account, segments, first_byte, last_byte,
cached_fetch_sub_slo_segments)
for seg_info in byterange_listing_iter:
yield seg_info
def GET(self, req):
# import pydevd
# pydevd.settrace('172.29.132.122', port=5678)
"""Handle HTTP GET request."""
# import rpdb2
# rpdb2.start_embedded_debugger("12345")
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def get_bucket_db(conf):
"""
If `bucket_db_enabled` is set in `conf`, get the bucket database,
otherwise return `None`.
If `bucket_db_host` or `bucket_db_sentinel_hosts` are also set in `conf`,
return an instance of `RedisBucketDb`, otherwise return an instance of
`DummyBucketDb`.
"""
db_kwargs = {
k[10:]: v
for k, v in conf.items() if k.startswith('bucket_db_')
}
if config_true_value(db_kwargs.get('enabled', 'false')):
if 'host' in db_kwargs or 'sentinel_hosts' in db_kwargs:
if db_kwargs.get('sentinel_name') is None:
# TODO(adu): Delete when it will no longer be used
db_kwargs['sentinel_name'] = db_kwargs.pop('master_name', None)
return RedisBucketDb(**db_kwargs)
else:
return DummyBucketDb(**db_kwargs)
return None
def _initrp(conf_path, app_section, *args, **kwargs):
try:
conf = appconfig(conf_path, name=app_section)
except Exception as e:
raise ConfigFileError("Error trying to load config from %s: %s" %
(conf_path, e))
validate_configuration()
# pre-configure logger
log_name = conf.get('log_name', app_section)
if 'logger' in kwargs:
logger = kwargs.pop('logger')
else:
logger = get_logger(conf, log_name,
log_to_console=kwargs.pop('verbose', False),
log_route='wsgi')
# disable fallocate if desired
if config_true_value(conf.get('disable_fallocate', 'no')):
disable_fallocate()
return (conf, logger, log_name)
def __init__(self, request, conf, gateway_conf, app, logger):
"""
:param request: swob.Request instance
:param conf: middleware conf dict
:param gateway_conf: gatway conf dict
:param app: wsgi Application
:param logger: logger instance
"""
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH')
self.request = request
self.app = app
self.logger = logger
self.conf = conf
self.gateway_conf = gateway_conf
self.gateway_class = self.conf['gateway_module']
self.sreq_class = self.gateway_class.request_class
self.storlet_execute_on_proxy = \
config_true_value(conf.get('storlet_execute_on_proxy', 'false'))
containers = get_container_names(conf)
self.storlet_container = containers['storlet']
self.storlet_dependency = containers['dependency']
self.log_container = containers['log']
self.client_conf_file = '/etc/swift/storlet-proxy-server.conf'
def properties_from_headers(self, headers):
metadata = {
k: v
for k, v in headers.items()
if k.lower() in self.pass_through_headers
or is_sys_or_user_meta('container', k)
}
system = dict()
# This headers enable versioning.
# First the legacy one.
ver_loc = headers.get('X-Container-Sysmeta-Versions-Location')
if ver_loc is not None:
# When suspending versioning, header has empty string value
ver_val = "-1" if ver_loc else "1"
system['sys.m2.policy.version'] = ver_val
# Then the new one.
vers_enabled = headers.get(CLIENT_VERSIONS_ENABLED)
if vers_enabled is not None:
ver_val = "-1" if config_true_value(vers_enabled) else "1"
system['sys.m2.policy.version'] = ver_val
return metadata, system
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
container_list = broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'], reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
