class RequestCache(object):
def __init__(self, app, conf, *args, **kwargs):
self.app = app
_servers = conf.get('memcache_servers', '127.0.0.1:11211')
servers = [s.strip() for s in _servers.split(',') if s.strip()]
self.memcache = MemcacheRing(servers)
def __call__(self, env, start_response):
request = Request(env.copy())
if request.method != "GET":
return self.app(env, start_response)
(version, acc, con, obj) = request.split_path(1, 4, False)
if not obj:
return self.app(env, start_response)
memcache_key = 'RequestCache/%s/%s/%s' % (acc, con, obj)
cached_content = self.memcache.get(memcache_key)
if cached_content:
response = Response(request=request, body=cached_content)
response.headers['X-RequestCache'] = 'True'
return response(env, start_response)
sub_req = wsgi.make_subrequest(env)
sub_resp = sub_req.get_response(self.app)
self.memcache.set(memcache_key, sub_resp.body, time=86400.0)
return self.app(env, start_response)
def memcached_main(action, options, key=None, value=None):
memcache_ring = MemcacheRing(
options['servers'],
connect_timeout=options['connect_timeout'],
pool_timeout=options['pool_timeout'],
tries=options['tries'],
io_timeout=options['io_timeout'],
allow_pickle=(options['serialization_format'] == 0),
allow_unpickle=(options['serialization_format'] <= 1),
max_conns=options['max_conns'])
if action == 'set':
start_time = time.time()
memcache_ring = get_memcache_ring(options)
memcache_ring.set(key, value, time=600)
print('Duration: %s ms' % str((time.time() - start_time) * 1000))
elif action == 'get':
start_time = time.time()
memcache_ring = get_memcache_ring(options)
value, latency = memcache_get(memcache_ring, key)
print('Duration: %s ms' % str((time.time() - start_time) * 1000))
print('Value: %s' % value)
elif action == 'test_conns':
test_conns(options)
elif action == 'test_gets':
test_gets(memcache_ring, options)
def __init__(self, app, conf):
self.app = app
self.memcache_servers = conf.get('memcache_servers')
serialization_format = conf.get('memcache_serialization_support')
if not self.memcache_servers or serialization_format is None:
path = os.path.join(conf.get('swift_dir', '/etc/swift'),
'memcache.conf')
memcache_conf = ConfigParser()
if memcache_conf.read(path):
if not self.memcache_servers:
try:
self.memcache_servers = \
memcache_conf.get('memcache', 'memcache_servers')
except (NoSectionError, NoOptionError):
pass
if serialization_format is None:
try:
serialization_format = \
memcache_conf.get('memcache',
'memcache_serialization_support')
except (NoSectionError, NoOptionError):
pass
if not self.memcache_servers:
self.memcache_servers = '127.0.0.1:11211'
if serialization_format is None:
serialization_format = 2
self.memcache = MemcacheRing(
[s.strip() for s in self.memcache_servers.split(',') if s.strip()],
allow_pickle=(serialization_format == 0),
allow_unpickle=(serialization_format <= 1))
def get_cache(self, parsed_args):
if parsed_args.connection is None:
parsed_args.connection = self.app.client_manager.sds_conf.get(
'cache.connection', self.default_connection)
if parsed_args.connection == self.default_connection:
self.logger.warn(
'Using the default connection (%s) is probably '
'not what you want to do.', self.default_connection)
_, netloc, _ = parse_connection_string(parsed_args.connection)
return MemcacheRing(netloc.split(','))
def __init__(self, app, conf):
self.app = app
self.memcache_servers = conf.get('memcache_servers')
serialization_format = conf.get('memcache_serialization_support')
try:
# Originally, while we documented using memcache_max_connections
# we only accepted max_connections
max_conns = int(
conf.get('memcache_max_connections',
conf.get('max_connections', 0)))
except ValueError:
max_conns = 0
if (not self.memcache_servers or serialization_format is None
or max_conns <= 0):
path = os.path.join(conf.get('swift_dir', '/etc/swift'),
'memcache.conf')
memcache_conf = ConfigParser()
if memcache_conf.read(path):
if not self.memcache_servers:
try:
self.memcache_servers = \
memcache_conf.get('memcache', 'memcache_servers')
except (NoSectionError, NoOptionError):
pass
if serialization_format is None:
try:
serialization_format = \
memcache_conf.get('memcache',
'memcache_serialization_support')
except (NoSectionError, NoOptionError):
pass
if max_conns <= 0:
try:
new_max_conns = \
memcache_conf.get('memcache',
'memcache_max_connections')
max_conns = int(new_max_conns)
except (NoSectionError, NoOptionError, ValueError):
pass
if not self.memcache_servers:
self.memcache_servers = '127.0.0.1:11211'
if max_conns <= 0:
max_conns = 2
if serialization_format is None:
serialization_format = 2
else:
serialization_format = int(serialization_format)
self.memcache = MemcacheRing(
[s.strip() for s in self.memcache_servers.split(',') if s.strip()],
allow_pickle=(serialization_format == 0),
allow_unpickle=(serialization_format <= 1),
max_conns=max_conns)
def get_memcache_ring(options):
memcache_ring = MemcacheRing(
options['servers'],
connect_timeout=options['connect_timeout'],
pool_timeout=options['pool_timeout'],
tries=options['tries'],
io_timeout=options['io_timeout'],
allow_pickle=(options['serialization_format'] == 0),
allow_unpickle=(options['serialization_format'] <= 1),
max_conns=options['max_conns'])
return memcache_ring
def __init__(self, app, conf):
self.app = app
self.memcache_servers = conf.get('memcache_servers')
if not self.memcache_servers:
path = os.path.join(conf.get('swift_dir', '/etc/swift'),
'memcache.conf')
memcache_conf = ConfigParser()
if memcache_conf.read(path):
try:
self.memcache_servers = \
memcache_conf.get('memcache', 'memcache_servers')
except (NoSectionError, NoOptionError):
pass
if not self.memcache_servers:
self.memcache_servers = '127.0.0.1:11211'
self.memcache = MemcacheRing(
[s.strip() for s in self.memcache_servers.split(',') if s.strip()])
def __init__(self, conf, logger, log_route='log-processor'):
if isinstance(logger, tuple):
self.logger = get_logger(*logger, log_route=log_route)
else:
self.logger = logger
self.memcache = MemcacheRing([
s.strip() for s in conf.get('memcache_servers', '').split(',')
if s.strip()
])
self.conf = conf
self._internal_proxy = None
self.lookback_hours = int(conf.get('lookback_hours', '120'))
self.lookback_window = int(
conf.get('lookback_window', self.lookback_hours))
self.log_processor_account = conf['swift_account']
self.log_processor_container = conf.get('container_name',
'simple_billing_data')
self.processed_files_object_name = \
conf.get('processed_files_object_name',
'processed_files.pickle.gz')
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='memcache')
self.memcache_servers = conf.get('memcache_servers')
serialization_format = conf.get('memcache_serialization_support')
try:
# Originally, while we documented using memcache_max_connections
# we only accepted max_connections
max_conns = int(
conf.get('memcache_max_connections',
conf.get('max_connections', 0)))
except ValueError:
max_conns = 0
memcache_options = {}
if (not self.memcache_servers or serialization_format is None
or max_conns <= 0):
path = os.path.join(conf.get('swift_dir', '/etc/swift'),
'memcache.conf')
memcache_conf = ConfigParser()
if memcache_conf.read(path):
# if memcache.conf exists we'll start with those base options
try:
memcache_options = dict(memcache_conf.items('memcache'))
except NoSectionError:
pass
if not self.memcache_servers:
try:
self.memcache_servers = \
memcache_conf.get('memcache', 'memcache_servers')
except (NoSectionError, NoOptionError):
pass
if serialization_format is None:
try:
serialization_format = \
memcache_conf.get('memcache',
'memcache_serialization_support')
except (NoSectionError, NoOptionError):
pass
if max_conns <= 0:
try:
new_max_conns = \
memcache_conf.get('memcache',
'memcache_max_connections')
max_conns = int(new_max_conns)
except (NoSectionError, NoOptionError, ValueError):
pass
# while memcache.conf options are the base for the memcache
# middleware, if you set the same option also in the filter
# section of the proxy config it is more specific.
memcache_options.update(conf)
connect_timeout = float(
memcache_options.get('connect_timeout', CONN_TIMEOUT))
pool_timeout = float(memcache_options.get('pool_timeout',
POOL_TIMEOUT))
tries = int(memcache_options.get('tries', TRY_COUNT))
io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT))
if config_true_value(memcache_options.get('tls_enabled', 'false')):
tls_cafile = memcache_options.get('tls_cafile')
tls_certfile = memcache_options.get('tls_certfile')
tls_keyfile = memcache_options.get('tls_keyfile')
self.tls_context = ssl.create_default_context(cafile=tls_cafile)
if tls_certfile:
self.tls_context.load_cert_chain(tls_certfile, tls_keyfile)
else:
self.tls_context = None
error_suppression_interval = float(
memcache_options.get('error_suppression_interval',
ERROR_LIMIT_TIME))
error_suppression_limit = float(
memcache_options.get('error_suppression_limit', ERROR_LIMIT_COUNT))
if not self.memcache_servers:
self.memcache_servers = '127.0.0.1:11211'
if max_conns <= 0:
max_conns = 2
if serialization_format is None:
serialization_format = 2
else:
serialization_format = int(serialization_format)
self.memcache = MemcacheRing(
[s.strip() for s in self.memcache_servers.split(',') if s.strip()],
connect_timeout=connect_timeout,
pool_timeout=pool_timeout,
tries=tries,
io_timeout=io_timeout,
allow_pickle=(serialization_format == 0),
allow_unpickle=(serialization_format <= 1),
max_conns=max_conns,
tls_context=self.tls_context,
logger=self.logger,
error_limit_count=error_suppression_limit,
error_limit_time=error_suppression_interval,
error_limit_duration=error_suppression_interval)
def __init__(self, app, conf):
self.app = app
self.memcache = MemcacheRing([s.strip() for s in
conf.get('memcache_servers', '127.0.0.1:11211').split(',')
if s.strip()])
def __init__(self, app, conf):
self.app = app
self.memcache_servers = conf.get('memcache_servers')
serialization_format = conf.get('memcache_serialization_support')
try:
# Originally, while we documented using memcache_max_connections
# we only accepted max_connections
max_conns = int(
conf.get('memcache_max_connections',
conf.get('max_connections', 0)))
except ValueError:
max_conns = 0
memcache_options = {}
if (not self.memcache_servers or serialization_format is None
or max_conns <= 0):
path = os.path.join(conf.get('swift_dir', '/etc/swift'),
'memcache.conf')
memcache_conf = ConfigParser()
if memcache_conf.read(path):
# if memcache.conf exists we'll start with those base options
try:
memcache_options = dict(memcache_conf.items('memcache'))
except NoSectionError:
pass
if not self.memcache_servers:
try:
self.memcache_servers = \
memcache_conf.get('memcache', 'memcache_servers')
except (NoSectionError, NoOptionError):
pass
if serialization_format is None:
try:
serialization_format = \
memcache_conf.get('memcache',
'memcache_serialization_support')
except (NoSectionError, NoOptionError):
pass
if max_conns <= 0:
try:
new_max_conns = \
memcache_conf.get('memcache',
'memcache_max_connections')
max_conns = int(new_max_conns)
except (NoSectionError, NoOptionError, ValueError):
pass
# while memcache.conf options are the base for the memcache
# middleware, if you set the same option also in the filter
# section of the proxy config it is more specific.
memcache_options.update(conf)
connect_timeout = float(
memcache_options.get('connect_timeout', CONN_TIMEOUT))
pool_timeout = float(memcache_options.get('pool_timeout',
POOL_TIMEOUT))
tries = int(memcache_options.get('tries', TRY_COUNT))
io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT))
if not self.memcache_servers:
self.memcache_servers = '127.0.0.1:11211'
if max_conns <= 0:
max_conns = 2
if serialization_format is None:
serialization_format = 2
else:
serialization_format = int(serialization_format)
self.memcache = MemcacheRing(
[s.strip() for s in self.memcache_servers.split(',') if s.strip()],
connect_timeout=connect_timeout,
pool_timeout=pool_timeout,
tries=tries,
io_timeout=io_timeout,
allow_pickle=(serialization_format == 0),
allow_unpickle=(serialization_format <= 1),
max_conns=max_conns)
def __init__(self, app, conf, *args, **kwargs):
self.app = app
_servers = conf.get('memcache_servers', '127.0.0.1:11211')
servers = [s.strip() for s in _servers.split(',') if s.strip()]
self.memcache = MemcacheRing(servers)