def create_and_populate_client():
client = LocalClient(cache_options, cache_pfx)
# Monkey with the internals so we don't have to
# populate multiple times.
#print("Testing", type(bucket._dict))
# Only need to populate in the workers.
size_dists = [100] * 800 + [300] * 500 + [1024] * 300 + [2048] * 200 + [4096] * 150
with open('/dev/urandom', 'rb') as rnd:
data = [rnd.read(x) for x in size_dists]
data_iter = itertools.cycle(data)
keys_and_values = []
len_values = 0
j = 0
for j, datum in enumerate(data_iter):
if len(datum) > client.limit or len_values + len(datum) > client.limit:
break
len_values += len(datum)
# To ensure the pickle memo cache doesn't just write out "use object X",
# but distinct copies of the strings, we need to copy them
keys_and_values.append(
((j, j), (datum[:-1] + b'x', j))
)
# # We need to get the item so its frequency goes up enough to be written
# # (this is while we're doing an aging at write time, which may go away).
# # Using an assert statement causes us to write nothing if -O is used.
# if bucket[(j, j)] is datum:
# raise AssertionError()
mem_before = get_memory_usage()
client._bulk_update(keys_and_values, mem_usage_before=mem_before)
del keys_and_values
#print("Len", len(bucket), "size", bucket.size, "checkpoints", client.get_checkpoints())
return client
def _simulate_local(self, records, cache_local_mb, f):
from relstorage.cache.local_client import LocalClient
options = MockOptions()
options.cache_local_mb = cache_local_mb
options.cache_local_compression = 'none'
client = LocalClient(options)
now = time.time()
for record in records:
key = record.lba
if record.opcode == 'r':
data = client[key]
if data is None:
# Fill it in from the backend
client[key] = b'r' * record.size
else:
assert record.opcode == 'w'
client[key] = b'x' * record.size
done = time.time()
stats = client.stats()
self._report_one(stats, f, cache_local_mb, now, done)
return stats
def __init__(self,
adapter,
options,
prefix,
local_client=None,
_tracer=None):
self.adapter = adapter
self.options = options
self.prefix = prefix or ''
# checkpoints_key holds the current checkpoints.
self.checkpoints_key = '%s:checkpoints' % self.prefix
assert isinstance(self.checkpoints_key, str) # no unicode on Py2
# delta_after0 contains {oid: tid} after checkpoint 0
# and before or at self.current_tid.
self.delta_after0 = self._delta_map_type()
# delta_after1 contains {oid: tid} after checkpoint 1 and
# before or at checkpoint 0. The content of delta_after1 only
# changes when checkpoints move.
self.delta_after1 = self._delta_map_type()
# delta_size_limit places an approximate limit on the number of
# entries in the delta_after maps.
self.delta_size_limit = options.cache_delta_size_limit
self.clients_local_first = []
if local_client is None:
self.clients_local_first.append(LocalClient(options, self.prefix))
else:
self.clients_local_first.append(local_client)
if options.cache_servers:
module_name = options.cache_module_name
module = importlib.import_module(module_name)
servers = options.cache_servers
if isinstance(servers, string_types):
servers = servers.split()
self.clients_local_first.append(module.Client(servers))
# self.clients_local_first is in order from local to global caches,
# while self.clients_global_first is in order from global to local.
self.clients_global_first = list(reversed(self.clients_local_first))
if local_client is None:
self.restore()
if _tracer is None:
tracefile = persistence.trace_file(options, self.prefix)
if tracefile:
_tracer = ZEOTracer(tracefile)
_tracer.trace(0x00)
self._tracer = _tracer
if hasattr(self._tracer, 'trace_store_current'):
self._trace = self._tracer.trace
self._trace_store_current = self._tracer.trace_store_current
def __init__(self, adapter, options, prefix, _parent=None):
super(StorageCache, self).__init__()
self.adapter = adapter
self.options = options
self.keep_history = options.keep_history
self.prefix = prefix or ''
if _parent is None:
# I must be the master!
# This is shared between all instances of a cache in a tree,
# including the master, so that they can share information about
# polling.
self.polling_state = MVCCDatabaseCoordinator(self.options)
self.local_client = LocalClient(options, self.prefix)
shared_cache = MemcacheStateCache.from_options(
options, self.prefix)
if shared_cache is not None:
self.cache = MultiStateCache(self.local_client, shared_cache)
else:
self.cache = self.local_client
tracefile = persistence.trace_file(options, self.prefix)
if tracefile:
tracer = ZEOTracer(tracefile)
tracer.trace(0x00)
self.cache = TracingStateCache(self.cache, tracer)
else:
self.polling_state = _parent.polling_state # type: MVCCDatabaseCoordinator
self.local_client = _parent.local_client.new_instance()
self.cache = _parent.cache.new_instance()
# Once we have registered with the MVCCDatabaseCoordinator,
# we cannot make any changes to our own mvcc state without
# letting it know about them. In particular, that means we must
# not just assign to this object (except under careful circumstances
# where we're sure to be single threaded.)
# This object can be None
self.object_index = None
# It is also important not to register with the coordinator until
# we are fully initialized; we could be constructing a new_instance
# in a separate thread while polling is going on in other threads.
# We can get strange AttributeError if a partially constructed instance
# is exposed.
self.polling_state.register(self)
if _parent is None:
self.restore()
def makeOne(populate=True, data=None):
mem_before = get_memory_usage()
gc.collect()
gc.collect()
objects_before = len(gc.get_objects())
client = LocalClient(options, 'pfx')
client.b_mem_before = mem_before
client.b_objects_before = objects_before
if populate:
client._bulk_update([
(t[0], (t[1][0], t[1][1], False, 1))
for t in data or _make_data(random_data, KEY_GROUP_SIZE)
])
return client
def __init__(self,
adapter,
options,
prefix,
local_client=None,
_tracer=None):
self.adapter = adapter
self.options = options
self.prefix = prefix or ''
# delta_after0 contains {oid: tid} *after* checkpoint 0
# and before or at self.current_tid.
self.delta_after0 = self._delta_map_type()
# delta_after1 contains {oid: tid} *after* checkpoint 1 and
# *before* or at checkpoint 0. The content of delta_after1 only
# changes when checkpoints shift and we rebuild it.
self.delta_after1 = self._delta_map_type()
# delta_size_limit places an approximate limit on the number of
# entries in the delta_after maps.
self.delta_size_limit = options.cache_delta_size_limit
if local_client is None:
self.local_client = LocalClient(options, self.prefix)
else:
self.local_client = local_client
shared_cache = MemcacheStateCache.from_options(options, self.prefix)
if shared_cache is not None:
self.cache = MultiStateCache(self.local_client, shared_cache)
else:
self.cache = self.local_client
if local_client is None:
self.restore()
if _tracer is None:
tracefile = persistence.trace_file(options, self.prefix)
if tracefile:
_tracer = ZEOTracer(tracefile)
_tracer.trace(0x00)
self._tracer = _tracer
if hasattr(self._tracer, 'trace_store_current'):
self.cache = TracingStateCache(self.cache, _tracer)
def read_client():
begin = perf_counter()
c2 = LocalClient(cache_options, cache_pfx)
c2.restore()
end = perf_counter()
return end - begin
def makeOne(bucket_kind, populate=True):
options.cache_local_storage = bucket_kind
client = LocalClient(options, 'pfx')
if populate:
client._bucket0.bulk_update(ALL_DATA)
return client
def populate_empty():
c = LocalClient(options)
for k, v in ALL_DATA:
c.set(k, v)
