def __init__(self):
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
# TODO: An alternate idea would be a temporary sqlite database.
self._queue = AutoTemporaryFile()
# {oid: (startpos, endpos, prev_tid_int)}
self._queue_contents = OID_OBJECT_MAP_TYPE()
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
self.queue = AutoTemporaryFile()
self.queue_contents = {}
class _TemporaryStorage(object):
def __init__(self):
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
# TODO: An alternate idea would be a temporary sqlite database.
self._queue = AutoTemporaryFile()
# {oid: (startpos, endpos, prev_tid_int)}
self._queue_contents = OID_OBJECT_MAP_TYPE()
def reset(self):
self._queue_contents.clear()
self._queue.seek(0)
def store_temp(self, oid_int, state, prev_tid_int=0):
"""
Queue an object for caching.
Typically, we can't actually cache the object yet, because its
transaction ID is not yet chosen.
"""
assert isinstance(state, bytes)
queue = self._queue
queue.seek(0, 2) # seek to end
startpos = queue.tell()
queue.write(state)
endpos = queue.tell()
self._queue_contents[oid_int] = (startpos, endpos, prev_tid_int)
def __len__(self):
# How many distinct OIDs have been stored?
return len(self._queue_contents)
def __bool__(self):
return True
__nonzero__ = __bool__
@property
def stored_oids(self):
return self._queue_contents
def _read_temp_state(self, startpos, endpos):
self._queue.seek(startpos)
length = endpos - startpos
state = self._queue.read(length)
if len(state) != length:
raise AssertionError("Queued cache data is truncated")
return state
def read_temp(self, oid_int):
"""
Return the bytes for a previously stored temporary item.
"""
startpos, endpos, _ = self._queue_contents[oid_int]
return self._read_temp_state(startpos, endpos)
def __iter__(self):
return self.iter_for_oids(None)
def iter_for_oids(self, oids):
read_temp_state = self._read_temp_state
for startpos, endpos, oid_int, prev_tid_int in self.items(oids):
state = read_temp_state(startpos, endpos)
yield state, oid_int, prev_tid_int
def items(self, oids=None):
# Order the queue by file position, which should help
# if the file is large and needs to be read
# sequentially from disk.
items = [(startpos, endpos, oid_int, prev_tid_int)
for (oid_int,
(startpos, endpos,
prev_tid_int)) in iteroiditems(self._queue_contents)
if oids is None or oid_int in oids]
items.sort()
return items
def close(self):
self._queue.close()
self._queue = None
self._queue_contents = None
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
self.queue = AutoTemporaryFile()
self.queue_contents = {}
class StorageCache(object):
"""RelStorage integration with memcached or similar.
Holds a list of memcache clients in order from most local to
most global. The first is a LocalClient, which stores the cache
in the Python process, but shares the cache between threads.
"""
# send_limit: approximate limit on the bytes to buffer before
# sending to the cache.
send_limit = 1024 * 1024
# queue is an AutoTemporaryFile during transaction commit.
queue = None
# queue_contents is a map of {oid_int: (startpos, endpos)}
# during transaction commit.
queue_contents = None
# checkpoints, when set, is a tuple containing the integer
# transaction ID of the two current checkpoints. checkpoint0 is
# greater than or equal to checkpoint1.
checkpoints = None
# current_tid contains the last polled transaction ID. Invariant:
# when self.checkpoints is not None, self.delta_after0 has info
# from all transactions in the range:
# self.checkpoints[0] < tid <= self.current_tid
current_tid = 0
# commit_count contains the last polled value of the
# :commits cache key. The most global client currently
# responding stores the value.
commit_count = object()
def __init__(self, adapter, options, local_client=None):
self.adapter = adapter
self.options = options
if local_client is None:
local_client = LocalClient(options)
self.clients_local_first = [local_client]
if options.cache_servers:
module_name = options.cache_module_name
module = __import__(module_name, {}, {}, ['Client'])
servers = options.cache_servers
if isinstance(servers, basestring):
servers = servers.split()
self.clients_local_first.append(module.Client(servers))
# self.clients_local_first is in order from local to global caches,
# while self.clients_global_first is in order from global to local.
self.clients_global_first = list(self.clients_local_first)
self.clients_global_first.reverse()
# every cache key has a prefix
self.prefix = options.cache_prefix or ''
# commit_count_key contains a number that is incremented
# for every commit. See tpc_finish().
self.commit_count_key = '%s:commits' % self.prefix
# checkpoints_key holds the current checkpoints.
self.checkpoints_key = '%s:checkpoints' % self.prefix
# delta_after0 contains {oid: tid} after checkpoint 0
# and before or at self.current_tid.
self.delta_after0 = {}
# delta_after1 contains {oid: tid} after checkpoint 1 and
# before or at checkpoint 0. The content of delta_after1 only
# changes when checkpoints move.
self.delta_after1 = {}
# delta_size_limit places an approximate limit on the number of
# entries in the delta_after maps.
self.delta_size_limit = options.cache_delta_size_limit
def new_instance(self):
"""Return a copy of this instance sharing the same local client"""
if self.options.share_local_cache:
local_client = self.clients_local_first[0]
return StorageCache(self.adapter, self.options, local_client)
else:
return StorageCache(self.adapter, self.options)
def clear(self):
"""Remove all data from the cache. Called by speed tests."""
for client in self.clients_local_first:
client.flush_all()
self.checkpoints = None
self.delta_after0 = {}
self.delta_after1 = {}
self.current_tid = 0
self.commit_count = object()
def _check_tid_after_load(self,
oid_int,
actual_tid_int,
expect_tid_int=None):
"""Verify the tid of an object loaded from the database is sane."""
if actual_tid_int > self.current_tid:
# Strangely, the database just gave us data from a future
# transaction. We can't give the data to ZODB because that
# would be a consistency violation. However, the cause is hard
# to track down, so issue a ReadConflictError and hope that
# the application retries successfully.
raise ReadConflictError(
"Got data for OID 0x%(oid_int)x from "
"future transaction %(actual_tid_int)d (%(got_ts)s). "
"Current transaction is %(current_tid)d (%(current_ts)s)." % {
'oid_int': oid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'got_ts': str(TimeStamp(p64(actual_tid_int))),
'current_ts': str(TimeStamp(p64(self.current_tid))),
})
if expect_tid_int is not None and actual_tid_int != expect_tid_int:
# Uh-oh, the cache is inconsistent with the database.
# Possible causes:
#
# - The database MUST provide a snapshot view for each
# session; this error can occur if that requirement is
# violated. For example, MySQL's MyISAM engine is not
# sufficient for the object_state table because MyISAM
# can not provide a snapshot view. (InnoDB is
# sufficient.)
#
# - Something could be writing to the database out
# of order, such as a version of RelStorage that
# acquires a different commit lock.
#
# - A software bug. In the past, there was a subtle bug
# in after_poll() that caused it to ignore the
# transaction order, leading it to sometimes put the
# wrong tid in delta_after*.
cp0, cp1 = self.checkpoints
import os
import thread
raise AssertionError(
"Detected an inconsistency "
"between the RelStorage cache and the database "
"while loading an object using the delta_after0 dict. "
"Please verify the database is configured for "
"ACID compliance and that all clients are using "
"the same commit lock. "
"(oid_int=%(oid_int)r, expect_tid_int=%(expect_tid_int)r, "
"actual_tid_int=%(actual_tid_int)r, "
"current_tid=%(current_tid)r, cp0=%(cp0)r, cp1=%(cp1)r, "
"len(delta_after0)=%(lda0)r, len(delta_after1)=%(lda1)r, "
"pid=%(pid)r, thread_ident=%(thread_ident)r)" % {
'oid_int': oid_int,
'expect_tid_int': expect_tid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'cp0': cp0,
'cp1': cp1,
'lda0': len(self.delta_after0),
'lda1': len(self.delta_after1),
'pid': os.getpid(),
'thread_ident': thread.get_ident(),
})
def load(self, cursor, oid_int):
"""Load the given object from cache if possible.
Fall back to loading from the database.
"""
if not self.checkpoints:
# No poll has occurred yet. For safety, don't use the cache.
return self.adapter.mover.load_current(cursor, oid_int)
prefix = self.prefix
# Get the object from the transaction specified
# by the following values, in order:
#
# 1. delta_after0[oid_int]
# 2. checkpoints[0]
# 3. delta_after1[oid_int]
# 4. checkpoints[1]
# 5. The database.
#
# checkpoints[0] is the preferred location.
#
# If delta_after0 contains oid_int, we should not look at any
# other cache keys, since the tid_int specified in delta_after0
# replaces all older transaction IDs. Similarly, if
# delta_after1 contains oid_int, we should not look at
# checkpoints[1]. Also, when both checkpoints are set to the
# same transaction ID, we don't need to ask for the same key
# twice.
tid_int = self.delta_after0.get(oid_int)
if tid_int:
# This object changed after checkpoint0, so
# there is only one place to look for its state.
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
for client in self.clients_local_first:
cache_data = client.get(cachekey)
if cache_data and len(cache_data) >= 8:
# Cache hit.
assert cache_data[:8] == p64(tid_int)
return cache_data[8:], tid_int
# Cache miss.
state, actual_tid_int = self.adapter.mover.load_current(
cursor, oid_int)
self._check_tid_after_load(oid_int, actual_tid_int, tid_int)
cache_data = '%s%s' % (p64(tid_int), state or '')
for client in self.clients_local_first:
client.set(cachekey, cache_data)
return state, tid_int
# Make a list of cache keys to query. The list will have either
# 1 or 2 keys.
cp0, cp1 = self.checkpoints
cachekeys = []
cp0_key = '%s:state:%d:%d' % (prefix, cp0, oid_int)
cachekeys.append(cp0_key)
da1_key = None
cp1_key = None
tid_int = self.delta_after1.get(oid_int)
if tid_int:
da1_key = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
cachekeys.append(da1_key)
elif cp1 != cp0:
cp1_key = '%s:state:%d:%d' % (prefix, cp1, oid_int)
cachekeys.append(cp1_key)
for client in self.clients_local_first:
# Query the cache. Query multiple keys simultaneously to
# minimize latency.
response = client.get_multi(cachekeys)
if response:
cache_data = response.get(cp0_key)
if cache_data and len(cache_data) >= 8:
# Cache hit on the preferred cache key.
return cache_data[8:], u64(cache_data[:8])
if da1_key:
cache_data = response.get(da1_key)
elif cp1_key:
cache_data = response.get(cp1_key)
if cache_data and len(cache_data) >= 8:
# Cache hit, but copy the state to
# the currently preferred key.
for client_to_set in self.clients_local_first:
client_to_set.set(cp0_key, cache_data)
return cache_data[8:], u64(cache_data[:8])
# Cache miss.
state, tid_int = self.adapter.mover.load_current(cursor, oid_int)
if tid_int:
self._check_tid_after_load(oid_int, tid_int)
cache_data = '%s%s' % (p64(tid_int), state or '')
for client in self.clients_local_first:
client.set(cp0_key, cache_data)
return state, tid_int
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
self.queue = AutoTemporaryFile()
self.queue_contents = {}
def store_temp(self, oid_int, state):
"""Queue an object for caching.
Typically, we can't actually cache the object yet, because its
transaction ID is not yet chosen.
"""
assert isinstance(state, str)
queue = self.queue
queue.seek(0, 2) # seek to end
startpos = queue.tell()
queue.write(state)
endpos = queue.tell()
self.queue_contents[oid_int] = (startpos, endpos)
def send_queue(self, tid):
"""Now that this tid is known, send all queued objects to the cache"""
tid_int = u64(tid)
send_size = 0
to_send = {}
prefix = self.prefix
# Order the queue by file position, which should help if the
# file is large and needs to be read sequentially from disk.
items = [(startpos, endpos, oid_int)
for (oid_int, (startpos,
endpos)) in self.queue_contents.items()]
items.sort()
for startpos, endpos, oid_int in items:
self.queue.seek(startpos)
length = endpos - startpos
state = self.queue.read(length)
if len(state) != length:
raise AssertionError("Queued cache data is truncated")
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
item_size = length + len(cachekey)
if send_size and send_size + item_size >= self.send_limit:
for client in self.clients_local_first:
client.set_multi(to_send)
to_send.clear()
send_size = 0
to_send[cachekey] = '%s%s' % (tid, state)
send_size += item_size
if to_send:
for client in self.clients_local_first:
client.set_multi(to_send)
self.queue_contents.clear()
self.queue.seek(0)
def after_tpc_finish(self, tid):
"""Update the commit count in the cache.
This is called after the database commit lock is released,
but before releasing the storage lock that will allow other
threads to use this instance.
"""
tid_int = u64(tid)
# Why do we cache a commit count instead of the transaction ID?
# Here's why. This method gets called after the commit lock is
# released; other threads or processes could have committed
# more transactions in the time that has passed since releasing
# the lock, so a cached transaction ID would cause a race. It
# also wouldn't work to cache the transaction ID before
# releasing the commit lock, since that could cause some
# threads or processes watching the cache for changes to think
# they are up to date when they are not. The commit count
# solves these problems by ensuring that every commit is
# followed by a change to the cache that does not conflict with
# concurrent committers.
cachekey = self.commit_count_key
for client in self.clients_global_first:
if client.incr(cachekey) is None:
# Initialize commit_count.
# Use a random number for the base.
client.add(cachekey, random.randint(1, 1 << 31))
# A concurrent committer could have won the race to set the
# initial commit_count. Increment commit_count so that it
# doesn't matter who won.
if client.incr(cachekey) is not None:
break
# else the client is dead. Fall back to the next client.
if self.checkpoints:
for oid_int in self.queue_contents:
# Future cache lookups for oid_int should now use
# the tid just committed.
self.delta_after0[oid_int] = tid_int
self.send_queue(tid)
def clear_temp(self):
"""Discard all transaction-specific temporary data.
Called after transaction finish or abort.
"""
self.queue_contents = None
if self.queue is not None:
self.queue.close()
self.queue = None
def need_poll(self):
"""Return True if the commit count has changed"""
for client in self.clients_global_first:
new_commit_count = client.get(self.commit_count_key)
if new_commit_count is not None:
break
if new_commit_count != self.commit_count:
self.commit_count = new_commit_count
return True
return False
def after_poll(self, cursor, prev_tid_int, new_tid_int, changes):
"""Update checkpoint data after a database poll.
cursor is connected to a load connection.
changes lists all [(oid_int, tid_int)] changed after
prev_tid_int, up to and including new_tid_int, excluding the
changes last committed by the associated storage instance.
changes can be None to indicate too many objects changed
to list them all.
prev_tid_int can be None, in which case the changes
parameter will be ignored. new_tid_int can not be None.
"""
new_checkpoints = None
for client in self.clients_global_first:
s = client.get(self.checkpoints_key)
if s:
try:
c0, c1 = s.split()
c0 = int(c0)
c1 = int(c1)
except ValueError:
# Invalid checkpoint cache value; ignore it.
pass
else:
if c0 >= c1:
new_checkpoints = (c0, c1)
break
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
if not self.checkpoints:
# Initialize the checkpoints.
cache_data = '%d %d' % new_checkpoints
log.debug("Initializing checkpoints: %s", cache_data)
else:
# Suggest reinstatement of the former checkpoints, but
# use new checkpoints for this instance. Using new
# checkpoints ensures that we don't build up
# self.delta_after0 in case the cache is offline.
cache_data = '%d %d' % self.checkpoints
log.debug("Reinstating checkpoints: %s", cache_data)
for client in self.clients_global_first:
client.set(self.checkpoints_key, cache_data)
self.checkpoints = new_checkpoints
self.delta_after0 = {}
self.delta_after1 = {}
self.current_tid = new_tid_int
return
allow_shift = True
if new_checkpoints[0] > new_tid_int:
# checkpoint0 is in a future that this instance can't
# yet see. Ignore the checkpoint change for now.
new_checkpoints = self.checkpoints
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
allow_shift = False
if (new_checkpoints == self.checkpoints and changes is not None
and prev_tid_int and prev_tid_int <= self.current_tid
and new_tid_int >= self.current_tid):
# Keep the checkpoints and update self.delta_after0.
m = self.delta_after0
m_get = m.get
for oid_int, tid_int in changes:
my_tid_int = m_get(oid_int)
if my_tid_int is None or tid_int > my_tid_int:
m[oid_int] = tid_int
self.current_tid = new_tid_int
else:
cp0, cp1 = new_checkpoints
log.debug("Using new checkpoints: %d %d", cp0, cp1)
# Use the checkpoints specified by the cache.
# Rebuild delta_after0 and delta_after1.
new_delta_after0 = {}
new_delta_after1 = {}
if cp1 < new_tid_int:
# poller.list_changes provides an iterator of
# (oid, tid) where tid > after_tid and tid <= last_tid.
changes = self.adapter.poller.list_changes(
cursor, cp1, new_tid_int)
# Make a dictionary that contains, for each oid, the most
# recent tid listed in changes.
changes_dict = {}
if not isinstance(changes, list):
changes = list(changes)
changes.sort()
for oid_int, tid_int in changes:
changes_dict[oid_int] = tid_int
# Put the changes in new_delta_after*.
for oid_int, tid_int in changes_dict.iteritems():
if tid_int > cp0:
new_delta_after0[oid_int] = tid_int
elif tid_int > cp1:
new_delta_after1[oid_int] = tid_int
self.checkpoints = new_checkpoints
self.delta_after0 = new_delta_after0
self.delta_after1 = new_delta_after1
self.current_tid = new_tid_int
if allow_shift and len(self.delta_after0) >= self.delta_size_limit:
# delta_after0 has reached its limit. The way to
# shrink it is to shift the checkpoints. Suggest
# shifted checkpoints for future polls.
# If delta_after0 is far over the limit (caused by a large
# transaction), suggest starting new checkpoints instead of
# shifting.
oversize = (len(self.delta_after0) >= self.delta_size_limit * 2)
self._suggest_shifted_checkpoints(new_tid_int, oversize)
def _suggest_shifted_checkpoints(self, tid_int, oversize):
"""Suggest that future polls use a new pair of checkpoints.
This does nothing if another instance has already shifted
the checkpoints.
checkpoint0 shifts to checkpoint1 and the tid just committed
becomes checkpoint0.
"""
cp0, cp1 = self.checkpoints
assert tid_int > cp0
expect = '%d %d' % self.checkpoints
if oversize:
# start new checkpoints
change_to = '%d %d' % (tid_int, tid_int)
else:
# shift the existing checkpoints
change_to = '%d %d' % (tid_int, cp0)
for client in self.clients_global_first:
old_value = client.get(self.checkpoints_key)
if old_value:
break
if not old_value or old_value == expect:
# Shift the checkpoints.
# Although this is a race with other instances, the race
# should not matter.
log.debug("Shifting checkpoints to: %s. len(delta_after0) == %d.",
change_to, len(self.delta_after0))
for client in self.clients_global_first:
client.set(self.checkpoints_key, change_to)
# The poll code will later see the new checkpoints
# and update self.checkpoints and self.delta_after(0|1).
else:
log.debug(
"Checkpoints already shifted to %s. "
"len(delta_after0) == %d.", old_value, len(self.delta_after0))
class StorageCache(object):
"""RelStorage integration with memcached or similar.
Holds a list of memcache clients in order from most local to
most global. The first is a LocalClient, which stores the cache
in the Python process, but shares the cache between threads.
"""
# pylint:disable=too-many-instance-attributes,too-many-public-methods
# send_limit: approximate limit on the bytes to buffer before
# sending to the cache.
send_limit = 1024 * 1024
# queue is an AutoTemporaryFile during transaction commit.
queue = None
# queue_contents is a map of {oid_int: (startpos, endpos)}
# during transaction commit.
queue_contents = None
# checkpoints, when set, is a tuple containing the integer
# transaction ID of the two current checkpoints. checkpoint0 is
# greater than or equal to checkpoint1.
checkpoints = None
# current_tid contains the last polled transaction ID. Invariant:
# when self.checkpoints is not None, self.delta_after0 has info
# from all transactions in the range:
# self.checkpoints[0] < tid <= self.current_tid
current_tid = 0
_tracer = None
# An LLBTree uses much less memory than a dict, and is still plenty fast on CPython;
# it's just as big and slower on PyPy, though.
_delta_map_type = BTrees.family64.II.BTree if not PYPY else dict
def __init__(self, adapter, options, prefix, local_client=None,
_tracer=None):
self.adapter = adapter
self.options = options
self.prefix = prefix or ''
# checkpoints_key holds the current checkpoints.
self.checkpoints_key = '%s:checkpoints' % self.prefix
assert isinstance(self.checkpoints_key, str) # no unicode on Py2
# delta_after0 contains {oid: tid} after checkpoint 0
# and before or at self.current_tid.
self.delta_after0 = self._delta_map_type()
# delta_after1 contains {oid: tid} after checkpoint 1 and
# before or at checkpoint 0. The content of delta_after1 only
# changes when checkpoints move.
self.delta_after1 = self._delta_map_type()
# delta_size_limit places an approximate limit on the number of
# entries in the delta_after maps.
self.delta_size_limit = options.cache_delta_size_limit
self.clients_local_first = []
if local_client is None:
self.clients_local_first.append(LocalClient(options, self.prefix))
else:
self.clients_local_first.append(local_client)
if options.cache_servers:
module_name = options.cache_module_name
module = importlib.import_module(module_name)
servers = options.cache_servers
if isinstance(servers, string_types):
servers = servers.split()
self.clients_local_first.append(module.Client(servers))
# self.clients_local_first is in order from local to global caches,
# while self.clients_global_first is in order from global to local.
self.clients_global_first = list(reversed(self.clients_local_first))
if local_client is None:
self.restore()
if _tracer is None:
tracefile = persistence.trace_file(options, self.prefix)
if tracefile:
_tracer = ZEOTracer(tracefile)
_tracer.trace(0x00)
self._tracer = _tracer
if hasattr(self._tracer, 'trace_store_current'):
self._trace = self._tracer.trace
self._trace_store_current = self._tracer.trace_store_current
# XXX: Note that our __bool__ and __len__ are NOT consistent
def __bool__(self):
return True
__nonzero__ = __bool__
def __len__(self):
if self.clients_local_first is _UsedAfterRelease:
return 0
return len(self.local_client)
@property
def size(self):
if self.clients_local_first is _UsedAfterRelease:
return 0
return self.local_client.size
@property
def limit(self):
if self.clients_local_first is _UsedAfterRelease:
return 0
return self.local_client.limit
@property
def local_client(self):
"""
The (shared) local in-memory cache client.
"""
return self.clients_local_first[0]
def stats(self):
"""
Return stats. This is a debugging aid only. The format is undefined and intended
for human inspection only.
"""
try:
local_client = self.local_client
except TypeError:
return {'closed': True}
else:
return local_client.stats()
def new_instance(self):
"""Return a copy of this instance sharing the same local client"""
local_client = self.local_client if self.options.share_local_cache else None
cache = type(self)(self.adapter, self.options, self.prefix,
local_client,
_tracer=self._tracer or False)
return cache
def release(self):
"""
Release resources held by this instance.
This is usually memcache connections if they're in use.
"""
clients = ()
if self.clients_local_first is not _UsedAfterRelease:
clients = self.clients_local_first
for client in clients:
client.disconnect_all()
# Release our clients. If we had a non-shared local cache,
# this will also allow it to release any memory its holding.
# Set them to non-iterables to make it obvious if we are used
# after release.
self.clients_local_first = _UsedAfterRelease
self.clients_global_first = _UsedAfterRelease
def save(self):
"""
Store any persistent client data.
"""
if self.options.cache_local_dir and len(self): # pylint:disable=len-as-condition
# (our __bool__ is not consistent with our len)
return persistence.save_local_cache(self.options, self.prefix, self)
def write_to_stream(self, stream):
# We currently don't write anything to the stream, delegating instead
# just to the local client.
# We experimented with trying to save and load chcekpoints and
# the delta maps, but this turned out to be complex (because
# the `new_instance`s that have the actual data are released
# before we are, so their data gets lost, and we have to
# implement a parent/child relationship to fix that) and no
# more effective than relying on the default checkpoints we
# get from polling, if there have been no changes---at least
# in the case of zodbshootout benchmark (in fact, it was
# somewhat *slower*, for reasons that aren't fully clear).
# Note that if we did want to dump the delta maps, we would
# need to either wrap them in a dict or dump them pairwise; We
# can't dump a BTree larger than about 25000 without getting
# into recursion problems.
self.local_client.write_to_stream(stream)
def get_cache_modification_time_for_stream(self):
max_tid = 0
for key in self.local_client:
parts = key.split(':')
if len(parts) != 4:
continue
tid = int(parts[2])
max_tid = max(tid, max_tid)
if max_tid:
tid_str = p64(max_tid)
ts = TimeStamp(tid_str)
return ts.timeTime()
def restore(self):
options = self.options
if options.cache_local_dir:
persistence.load_local_cache(options, self.prefix, self)
def read_from_stream(self, stream):
return self.local_client.read_from_stream(stream)
def close(self):
"""
Release resources held by this instance, and
save any persistent data necessary.
"""
self.save()
self.release()
if self._tracer:
# Note we can't do this in release(). Release is called on
# all instances, while close() is only called on the main one.
self._tracer.close()
del self._trace
del self._trace_store_current
del self._tracer
def clear(self, load_persistent=True):
"""
Remove all data from the cache. Called by speed tests.
Starting from the introduction of persistent cache files,
this also results in the local client being repopulated with
the current set of persistent data. The *load_persistent* keyword can
be used to control this.
.. versionchanged:: 2.0b6
Added the ``load_persistent`` keyword. This argument is provisional.
"""
for client in self.clients_local_first:
client.flush_all()
self.checkpoints = None
self.delta_after0 = self._delta_map_type()
self.delta_after1 = self._delta_map_type()
self.current_tid = 0
if load_persistent:
self.restore()
@staticmethod
def _trace(*_args, **_kwargs): # pylint:disable=method-hidden
# Dummy method for when we don't do tracing
return
@staticmethod
def _trace_store_current(_tid_int, _items): # pylint:disable=method-hidden
# Dummy method for when we don't do tracing
return
def _check_tid_after_load(self, oid_int, actual_tid_int,
expect_tid_int=None):
"""Verify the tid of an object loaded from the database is sane."""
if actual_tid_int > self.current_tid:
# Strangely, the database just gave us data from a future
# transaction. We can't give the data to ZODB because that
# would be a consistency violation. However, the cause is hard
# to track down, so issue a ReadConflictError and hope that
# the application retries successfully.
msg = ("Got data for OID 0x%(oid_int)x from "
"future transaction %(actual_tid_int)d (%(got_ts)s). "
"Current transaction is %(current_tid)d (%(current_ts)s)."
% {
'oid_int': oid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'got_ts': str(TimeStamp(p64(actual_tid_int))),
'current_ts': str(TimeStamp(p64(self.current_tid))),
})
raise ReadConflictError(msg)
if expect_tid_int is not None and actual_tid_int != expect_tid_int:
# Uh-oh, the cache is inconsistent with the database.
# Possible causes:
#
# - The database MUST provide a snapshot view for each
# session; this error can occur if that requirement is
# violated. For example, MySQL's MyISAM engine is not
# sufficient for the object_state table because MyISAM
# can not provide a snapshot view. (InnoDB is
# sufficient.)
#
# - Something could be writing to the database out
# of order, such as a version of RelStorage that
# acquires a different commit lock.
#
# - A software bug. In the past, there was a subtle bug
# in after_poll() that caused it to ignore the
# transaction order, leading it to sometimes put the
# wrong tid in delta_after*.
cp0, cp1 = self.checkpoints
msg = ("Detected an inconsistency "
"between the RelStorage cache and the database "
"while loading an object using the delta_after0 dict. "
"Please verify the database is configured for "
"ACID compliance and that all clients are using "
"the same commit lock. "
"(oid_int=%(oid_int)r, expect_tid_int=%(expect_tid_int)r, "
"actual_tid_int=%(actual_tid_int)r, "
"current_tid=%(current_tid)r, cp0=%(cp0)r, cp1=%(cp1)r, "
"len(delta_after0)=%(lda0)r, len(delta_after1)=%(lda1)r, "
"pid=%(pid)r, thread_ident=%(thread_ident)r)"
% {
'oid_int': oid_int,
'expect_tid_int': expect_tid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'cp0': cp0,
'cp1': cp1,
'lda0': len(self.delta_after0),
'lda1': len(self.delta_after1),
'pid': os.getpid(),
'thread_ident': threading.current_thread(),
})
raise AssertionError(msg)
def load(self, cursor, oid_int):
"""Load the given object from cache if possible.
Fall back to loading from the database.
"""
# pylint:disable=too-many-statements,too-many-branches,too-many-locals
if not self.checkpoints:
# No poll has occurred yet. For safety, don't use the cache.
self._trace(0x20, oid_int)
return self.adapter.mover.load_current(cursor, oid_int)
prefix = self.prefix
# Get the object from the transaction specified
# by the following values, in order:
#
# 1. delta_after0[oid_int]
# 2. checkpoints[0]
# 3. delta_after1[oid_int]
# 4. checkpoints[1]
# 5. The database.
#
# checkpoints[0] is the preferred location.
#
# If delta_after0 contains oid_int, we should not look at any
# other cache keys, since the tid_int specified in delta_after0
# replaces all older transaction IDs. Similarly, if
# delta_after1 contains oid_int, we should not look at
# checkpoints[1]. Also, when both checkpoints are set to the
# same transaction ID, we don't need to ask for the same key
# twice.
tid_int = self.delta_after0.get(oid_int)
if tid_int:
# This object changed after checkpoint0, so
# there is only one place to look for its state.
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
for client in self.clients_local_first:
cache_data = client.get(cachekey)
if cache_data and len(cache_data) >= 8:
# Cache hit.
# Note that we trace all cache hits, not just the local cache hit.
# This makes the simulation less useful, but the stats might still have
# value to people trying different tuning options manually.
self._trace(0x22, oid_int, tid_int, dlen=len(cache_data) - 8)
assert cache_data[:8] == p64(tid_int)
return cache_data[8:], tid_int
# Cache miss.
self._trace(0x20, oid_int)
state, actual_tid_int = self.adapter.mover.load_current(
cursor, oid_int)
self._check_tid_after_load(oid_int, actual_tid_int, tid_int)
cache_data = p64(tid_int) + (state or b'')
for client in self.clients_local_first:
client.set(cachekey, cache_data)
return state, tid_int
# Make a list of cache keys to query. The list will have either
# 1 or 2 keys.
cp0, cp1 = self.checkpoints
cachekeys = []
cp0_key = '%s:state:%d:%d' % (prefix, cp0, oid_int)
cachekeys.append(cp0_key)
da1_key = None
cp1_key = None
tid_int = self.delta_after1.get(oid_int)
if tid_int:
da1_key = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
cachekeys.append(da1_key)
elif cp1 != cp0:
cp1_key = '%s:state:%d:%d' % (prefix, cp1, oid_int)
cachekeys.append(cp1_key)
for client in self.clients_local_first:
# Query the cache. Query multiple keys simultaneously to
# minimize latency.
response = client.get_multi(cachekeys)
if response:
cache_data = response.get(cp0_key)
if cache_data and len(cache_data) >= 8:
# Cache hit on the preferred cache key.
local_client = self.local_client
if client is not local_client:
# Copy to the local client.
local_client.set(cp0_key, cache_data)
self._trace(0x22, oid_int, u64(cache_data[:8]), dlen=len(cache_data) - 8)
return cache_data[8:], u64(cache_data[:8])
if da1_key:
cache_data = response.get(da1_key)
elif cp1_key:
cache_data = response.get(cp1_key)
if cache_data and len(cache_data) >= 8:
# Cache hit, but copy the state to
# the currently preferred key.
self._trace(0x22, oid_int, u64(cache_data[:8]), dlen=len(cache_data) - 8)
for client_to_set in self.clients_local_first:
client_to_set.set(cp0_key, cache_data)
return cache_data[8:], u64(cache_data[:8])
# Cache miss.
self._trace(0x20, oid_int)
state, tid_int = self.adapter.mover.load_current(cursor, oid_int)
if tid_int:
self._check_tid_after_load(oid_int, tid_int)
cache_data = p64(tid_int) + (state or b'')
# Record this as a store into the cache, ZEO does.
self._trace(0x52, oid_int, tid_int, dlen=len(state) if state else 0)
for client in self.clients_local_first:
client.set(cp0_key, cache_data)
return state, tid_int
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
self.queue = AutoTemporaryFile()
self.queue_contents = {}
def store_temp(self, oid_int, state):
"""Queue an object for caching.
Typically, we can't actually cache the object yet, because its
transaction ID is not yet chosen.
"""
assert isinstance(state, bytes)
queue = self.queue
queue.seek(0, 2) # seek to end
startpos = queue.tell()
queue.write(state)
endpos = queue.tell()
self.queue_contents[oid_int] = (startpos, endpos)
def _read_temp_state(self, startpos, endpos):
self.queue.seek(startpos)
length = endpos - startpos
state = self.queue.read(length)
if len(state) != length:
raise AssertionError("Queued cache data is truncated")
return state, length
def read_temp(self, oid_int):
"""
Return the bytes for a previously stored temporary item.
"""
startpos, endpos = self.queue_contents[oid_int]
return self._read_temp_state(startpos, endpos)[0]
def send_queue(self, tid):
"""Now that this tid is known, send all queued objects to the cache"""
tid_int = u64(tid)
send_size = 0
to_send = {}
prefix = self.prefix
# Order the queue by file position, which should help if the
# file is large and needs to be read sequentially from disk.
items = [
(startpos, endpos, oid_int)
for (oid_int, (startpos, endpos)) in iteritems(self.queue_contents)
]
items.sort()
# Trace these. This is the equivalent of ZEOs
# ClientStorage._update_cache.
self._trace_store_current(tid_int, items)
for startpos, endpos, oid_int in items:
state, length = self._read_temp_state(startpos, endpos)
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
item_size = length + len(cachekey)
if send_size and send_size + item_size >= self.send_limit:
for client in self.clients_local_first:
client.set_multi(to_send)
to_send.clear()
send_size = 0
to_send[cachekey] = tid + state
send_size += item_size
if to_send:
for client in self.clients_local_first:
client.set_multi(to_send)
self.queue_contents.clear()
self.queue.seek(0)
def after_tpc_finish(self, tid):
"""
Flush queued changes.
This is called after the database commit lock is released,
but before releasing the storage lock that will allow other
threads to use this instance.
"""
tid_int = u64(tid)
if self.checkpoints:
for oid_int in self.queue_contents:
# Future cache lookups for oid_int should now use
# the tid just committed.
self.delta_after0[oid_int] = tid_int
self.send_queue(tid)
def clear_temp(self):
"""Discard all transaction-specific temporary data.
Called after transaction finish or abort.
"""
self.queue_contents = None
if self.queue is not None:
self.queue.close()
self.queue = None
def after_poll(self, cursor, prev_tid_int, new_tid_int, changes):
"""Update checkpoint data after a database poll.
cursor is connected to a load connection.
changes lists all [(oid_int, tid_int)] changed after
prev_tid_int, up to and including new_tid_int, excluding the
changes last committed by the associated storage instance.
changes can be None to indicate too many objects changed
to list them all.
prev_tid_int can be None, in which case the changes
parameter will be ignored. new_tid_int can not be None.
"""
# pylint:disable=too-many-statements,too-many-branches,too-many-locals
new_checkpoints = None
for client in self.clients_global_first:
s = client.get(self.checkpoints_key)
if s:
try:
c0, c1 = s.split()
c0 = int(c0)
c1 = int(c1)
except ValueError:
# Invalid checkpoint cache value; ignore it.
pass
else:
if c0 >= c1:
new_checkpoints = (c0, c1)
break
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
if not self.checkpoints:
# Initialize the checkpoints.
cache_data = '%d %d' % new_checkpoints
log.debug("Initializing checkpoints: %s", cache_data)
else:
# Suggest reinstatement of the former checkpoints, but
# use new checkpoints for this instance. Using new
# checkpoints ensures that we don't build up
# self.delta_after0 in case the cache is offline.
cache_data = '%d %d' % self.checkpoints
log.debug("Reinstating checkpoints: %s", cache_data)
cache_data = cache_data.encode("ascii")
for client in self.clients_global_first:
client.set(self.checkpoints_key, cache_data)
self.checkpoints = new_checkpoints
self.delta_after0 = self._delta_map_type()
self.delta_after1 = self._delta_map_type()
self.current_tid = new_tid_int
return
allow_shift = True
if new_checkpoints[0] > new_tid_int:
# checkpoint0 is in a future that this instance can't
# yet see. Ignore the checkpoint change for now.
new_checkpoints = self.checkpoints
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
allow_shift = False
# We want to keep the current checkpoints for speed, but we
# have to replace them (to avoid consistency violations)
# if certain conditions happen (like emptying the ZODB cache).
if (new_checkpoints == self.checkpoints
and changes is not None
and prev_tid_int
and prev_tid_int <= self.current_tid
and new_tid_int >= self.current_tid):
# All the conditions for keeping the checkpoints were met,
# so just update self.delta_after0 and self.current_tid.
m = self.delta_after0
m_get = m.get
for oid_int, tid_int in changes:
my_tid_int = m_get(oid_int)
if my_tid_int is None or tid_int > my_tid_int:
m[oid_int] = tid_int
# 0x1E = invalidate (hit, saving non-current)
self._trace(0x1C, oid_int, tid_int)
self.current_tid = new_tid_int
else:
# We have to replace the checkpoints.
cp0, cp1 = new_checkpoints
log.debug("Using new checkpoints: %d %d", cp0, cp1)
# Use the checkpoints specified by the cache.
# Rebuild delta_after0 and delta_after1.
new_delta_after0 = self._delta_map_type()
new_delta_after1 = self._delta_map_type()
if cp1 < new_tid_int:
# poller.list_changes provides an iterator of
# (oid, tid) where tid > after_tid and tid <= last_tid.
change_list = self.adapter.poller.list_changes(
cursor, cp1, new_tid_int)
# Make a dictionary that contains, for each oid, the most
# recent tid listed in changes. This works because sorting the
# (oid, tid) pairs puts the newest tid at the back, and constructing
# the dictionary from that sorted list preserves order, keeping the
# last key that it saw.
try:
change_dict = self._delta_map_type(sorted(change_list))
except TypeError:
# pg8000 returns a list of lists, not a list of tuples. The
# BTree constructor is very particular about that. Normally one
# would use pg8000 on PyPy, where we don't use BTrees, so this shouldn't
# actually come up in practice.
change_dict = self._delta_map_type()
for oid_int, tid_int in sorted(change_list):
change_dict[oid_int] = tid_int
# Put the changes in new_delta_after*.
for oid_int, tid_int in change_dict.items():
# 0x1E = invalidate (hit, saving non-current)
self._trace(0x1C, oid_int, tid_int)
if tid_int > cp0:
new_delta_after0[oid_int] = tid_int
elif tid_int > cp1:
new_delta_after1[oid_int] = tid_int
self.checkpoints = new_checkpoints
self.delta_after0 = new_delta_after0
self.delta_after1 = new_delta_after1
self.current_tid = new_tid_int
if allow_shift and len(self.delta_after0) >= self.delta_size_limit:
# delta_after0 has reached its limit. The way to
# shrink it is to shift the checkpoints. Suggest
# shifted checkpoints for future polls.
# If delta_after0 is far over the limit (caused by a large
# transaction), suggest starting new checkpoints instead of
# shifting.
oversize = (len(self.delta_after0) >= self.delta_size_limit * 2)
self._suggest_shifted_checkpoints(new_tid_int, oversize)
def _suggest_shifted_checkpoints(self, tid_int, oversize):
"""Suggest that future polls use a new pair of checkpoints.
This does nothing if another instance has already shifted
the checkpoints.
checkpoint0 shifts to checkpoint1 and the tid just committed
becomes checkpoint0.
"""
cp0, _cp1 = self.checkpoints
assert tid_int > cp0
expect = '%d %d' % self.checkpoints
if oversize:
# start new checkpoints
change_to = '%d %d' % (tid_int, tid_int)
else:
# shift the existing checkpoints
change_to = '%d %d' % (tid_int, cp0)
expect = expect.encode('ascii')
change_to = change_to.encode('ascii')
for client in self.clients_global_first:
old_value = client.get(self.checkpoints_key)
if old_value:
break
if not old_value or old_value == expect:
# Shift the checkpoints.
# Although this is a race with other instances, the race
# should not matter.
log.debug("Shifting checkpoints to: %s. len(delta_after0) == %d.",
change_to, len(self.delta_after0))
for client in self.clients_global_first:
client.set(self.checkpoints_key, change_to)
# The poll code will later see the new checkpoints
# and update self.checkpoints and self.delta_after(0|1).
else:
log.debug("Checkpoints already shifted to %s. "
"len(delta_after0) == %d.", old_value, len(self.delta_after0))
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
self.queue = AutoTemporaryFile()
self.queue_contents = {}
class StorageCache(object):
"""RelStorage integration with memcached or similar.
Holds a list of memcache clients in order from most local to
most global. The first is a LocalClient, which stores the cache
in the Python process, but shares the cache between threads.
"""
# send_limit: approximate limit on the bytes to buffer before
# sending to the cache.
send_limit = 1024 * 1024
# queue is an AutoTemporaryFile during transaction commit.
queue = None
# queue_contents is a map of {oid_int: (startpos, endpos)}
# during transaction commit.
queue_contents = None
# checkpoints, when set, is a tuple containing the integer
# transaction ID of the two current checkpoints. checkpoint0 is
# greater than or equal to checkpoint1.
checkpoints = None
# current_tid contains the last polled transaction ID. Invariant:
# when self.checkpoints is not None, self.delta_after0 has info
# from all transactions in the range:
# self.checkpoints[0] < tid <= self.current_tid
current_tid = 0
# commit_count contains the last polled value of the
# :commits cache key. The most global client currently
# responding stores the value.
commit_count = object()
def __init__(self, adapter, options, prefix, local_client=None):
self.adapter = adapter
self.options = options
self.prefix = prefix or ''
if local_client is None:
local_client = LocalClient(options)
self.clients_local_first = [local_client]
if options.cache_servers:
module_name = options.cache_module_name
module = __import__(module_name, {}, {}, ['Client'])
servers = options.cache_servers
if isinstance(servers, basestring):
servers = servers.split()
self.clients_local_first.append(module.Client(servers))
# self.clients_local_first is in order from local to global caches,
# while self.clients_global_first is in order from global to local.
self.clients_global_first = list(self.clients_local_first)
self.clients_global_first.reverse()
# commit_count_key contains a number that is incremented
# for every commit. See tpc_finish().
self.commit_count_key = '%s:commits' % self.prefix
# checkpoints_key holds the current checkpoints.
self.checkpoints_key = '%s:checkpoints' % self.prefix
# delta_after0 contains {oid: tid} after checkpoint 0
# and before or at self.current_tid.
self.delta_after0 = {}
# delta_after1 contains {oid: tid} after checkpoint 1 and
# before or at checkpoint 0. The content of delta_after1 only
# changes when checkpoints move.
self.delta_after1 = {}
# delta_size_limit places an approximate limit on the number of
# entries in the delta_after maps.
self.delta_size_limit = options.cache_delta_size_limit
def new_instance(self):
"""Return a copy of this instance sharing the same local client"""
if self.options.share_local_cache:
local_client = self.clients_local_first[0]
return StorageCache(self.adapter, self.options, self.prefix,
local_client)
else:
return StorageCache(self.adapter, self.options, self.prefix)
def clear(self):
"""Remove all data from the cache. Called by speed tests."""
for client in self.clients_local_first:
client.flush_all()
self.checkpoints = None
self.delta_after0 = {}
self.delta_after1 = {}
self.current_tid = 0
self.commit_count = object()
def _check_tid_after_load(self, oid_int, actual_tid_int,
expect_tid_int=None):
"""Verify the tid of an object loaded from the database is sane."""
if actual_tid_int > self.current_tid:
# Strangely, the database just gave us data from a future
# transaction. We can't give the data to ZODB because that
# would be a consistency violation. However, the cause is hard
# to track down, so issue a ReadConflictError and hope that
# the application retries successfully.
raise ReadConflictError("Got data for OID 0x%(oid_int)x from "
"future transaction %(actual_tid_int)d (%(got_ts)s). "
"Current transaction is %(current_tid)d (%(current_ts)s)."
% {
'oid_int': oid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'got_ts': str(TimeStamp(p64(actual_tid_int))),
'current_ts': str(TimeStamp(p64(self.current_tid))),
})
if expect_tid_int is not None and actual_tid_int != expect_tid_int:
# Uh-oh, the cache is inconsistent with the database.
# Possible causes:
#
# - The database MUST provide a snapshot view for each
# session; this error can occur if that requirement is
# violated. For example, MySQL's MyISAM engine is not
# sufficient for the object_state table because MyISAM
# can not provide a snapshot view. (InnoDB is
# sufficient.)
#
# - Something could be writing to the database out
# of order, such as a version of RelStorage that
# acquires a different commit lock.
#
# - A software bug. In the past, there was a subtle bug
# in after_poll() that caused it to ignore the
# transaction order, leading it to sometimes put the
# wrong tid in delta_after*.
cp0, cp1 = self.checkpoints
import os
import thread
raise AssertionError("Detected an inconsistency "
"between the RelStorage cache and the database "
"while loading an object using the delta_after0 dict. "
"Please verify the database is configured for "
"ACID compliance and that all clients are using "
"the same commit lock. "
"(oid_int=%(oid_int)r, expect_tid_int=%(expect_tid_int)r, "
"actual_tid_int=%(actual_tid_int)r, "
"current_tid=%(current_tid)r, cp0=%(cp0)r, cp1=%(cp1)r, "
"len(delta_after0)=%(lda0)r, len(delta_after1)=%(lda1)r, "
"pid=%(pid)r, thread_ident=%(thread_ident)r)"
% {
'oid_int': oid_int,
'expect_tid_int': expect_tid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'cp0': cp0,
'cp1': cp1,
'lda0': len(self.delta_after0),
'lda1': len(self.delta_after1),
'pid': os.getpid(),
'thread_ident': thread.get_ident(),
})
def load(self, cursor, oid_int):
"""Load the given object from cache if possible.
Fall back to loading from the database.
"""
if not self.checkpoints:
# No poll has occurred yet. For safety, don't use the cache.
return self.adapter.mover.load_current(cursor, oid_int)
prefix = self.prefix
# Get the object from the transaction specified
# by the following values, in order:
#
# 1. delta_after0[oid_int]
# 2. checkpoints[0]
# 3. delta_after1[oid_int]
# 4. checkpoints[1]
# 5. The database.
#
# checkpoints[0] is the preferred location.
#
# If delta_after0 contains oid_int, we should not look at any
# other cache keys, since the tid_int specified in delta_after0
# replaces all older transaction IDs. Similarly, if
# delta_after1 contains oid_int, we should not look at
# checkpoints[1]. Also, when both checkpoints are set to the
# same transaction ID, we don't need to ask for the same key
# twice.
tid_int = self.delta_after0.get(oid_int)
if tid_int:
# This object changed after checkpoint0, so
# there is only one place to look for its state.
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
for client in self.clients_local_first:
cache_data = client.get(cachekey)
if cache_data and len(cache_data) >= 8:
# Cache hit.
assert cache_data[:8] == p64(tid_int)
return cache_data[8:], tid_int
# Cache miss.
state, actual_tid_int = self.adapter.mover.load_current(
cursor, oid_int)
self._check_tid_after_load(oid_int, actual_tid_int, tid_int)
cache_data = '%s%s' % (p64(tid_int), state or '')
for client in self.clients_local_first:
client.set(cachekey, cache_data)
return state, tid_int
# Make a list of cache keys to query. The list will have either
# 1 or 2 keys.
cp0, cp1 = self.checkpoints
cachekeys = []
cp0_key = '%s:state:%d:%d' % (prefix, cp0, oid_int)
cachekeys.append(cp0_key)
da1_key = None
cp1_key = None
tid_int = self.delta_after1.get(oid_int)
if tid_int:
da1_key = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
cachekeys.append(da1_key)
elif cp1 != cp0:
cp1_key = '%s:state:%d:%d' % (prefix, cp1, oid_int)
cachekeys.append(cp1_key)
for client in self.clients_local_first:
# Query the cache. Query multiple keys simultaneously to
# minimize latency.
response = client.get_multi(cachekeys)
if response:
cache_data = response.get(cp0_key)
if cache_data and len(cache_data) >= 8:
# Cache hit on the preferred cache key.
local_client = self.clients_local_first[0]
if client is not local_client:
# Copy to the local client.
local_client.set(cp0_key, cache_data)
return cache_data[8:], u64(cache_data[:8])
if da1_key:
cache_data = response.get(da1_key)
elif cp1_key:
cache_data = response.get(cp1_key)
if cache_data and len(cache_data) >= 8:
# Cache hit, but copy the state to
# the currently preferred key.
for client_to_set in self.clients_local_first:
client_to_set.set(cp0_key, cache_data)
return cache_data[8:], u64(cache_data[:8])
# Cache miss.
state, tid_int = self.adapter.mover.load_current(cursor, oid_int)
if tid_int:
self._check_tid_after_load(oid_int, tid_int)
cache_data = '%s%s' % (p64(tid_int), state or '')
for client in self.clients_local_first:
client.set(cp0_key, cache_data)
return state, tid_int
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
self.queue = AutoTemporaryFile()
self.queue_contents = {}
def store_temp(self, oid_int, state):
"""Queue an object for caching.
Typically, we can't actually cache the object yet, because its
transaction ID is not yet chosen.
"""
assert isinstance(state, str)
queue = self.queue
queue.seek(0, 2) # seek to end
startpos = queue.tell()
queue.write(state)
endpos = queue.tell()
self.queue_contents[oid_int] = (startpos, endpos)
def send_queue(self, tid):
"""Now that this tid is known, send all queued objects to the cache"""
tid_int = u64(tid)
send_size = 0
to_send = {}
prefix = self.prefix
# Order the queue by file position, which should help if the
# file is large and needs to be read sequentially from disk.
items = [
(startpos, endpos, oid_int)
for (oid_int, (startpos, endpos)) in self.queue_contents.items()
]
items.sort()
for startpos, endpos, oid_int in items:
self.queue.seek(startpos)
length = endpos - startpos
state = self.queue.read(length)
if len(state) != length:
raise AssertionError("Queued cache data is truncated")
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
item_size = length + len(cachekey)
if send_size and send_size + item_size >= self.send_limit:
for client in self.clients_local_first:
client.set_multi(to_send)
to_send.clear()
send_size = 0
to_send[cachekey] = '%s%s' % (tid, state)
send_size += item_size
if to_send:
for client in self.clients_local_first:
client.set_multi(to_send)
self.queue_contents.clear()
self.queue.seek(0)
def after_tpc_finish(self, tid):
"""Update the commit count in the cache.
This is called after the database commit lock is released,
but before releasing the storage lock that will allow other
threads to use this instance.
"""
tid_int = u64(tid)
# Why do we cache a commit count instead of the transaction ID?
# Here's why. This method gets called after the commit lock is
# released; other threads or processes could have committed
# more transactions in the time that has passed since releasing
# the lock, so a cached transaction ID would cause a race. It
# also wouldn't work to cache the transaction ID before
# releasing the commit lock, since that could cause some
# threads or processes watching the cache for changes to think
# they are up to date when they are not. The commit count
# solves these problems by ensuring that every commit is
# followed by a change to the cache that does not conflict with
# concurrent committers.
cachekey = self.commit_count_key
for client in self.clients_global_first:
if client.incr(cachekey) is None:
# Initialize commit_count.
# Use a random number for the base.
client.add(cachekey, random.randint(1, 1<<31))
# A concurrent committer could have won the race to set the
# initial commit_count. Increment commit_count so that it
# doesn't matter who won.
if client.incr(cachekey) is not None:
break
# else the client is dead. Fall back to the next client.
if self.checkpoints:
for oid_int in self.queue_contents:
# Future cache lookups for oid_int should now use
# the tid just committed.
self.delta_after0[oid_int] = tid_int
self.send_queue(tid)
def clear_temp(self):
"""Discard all transaction-specific temporary data.
Called after transaction finish or abort.
"""
self.queue_contents = None
if self.queue is not None:
self.queue.close()
self.queue = None
def need_poll(self):
"""Return True if the commit count has changed"""
for client in self.clients_global_first:
new_commit_count = client.get(self.commit_count_key)
if new_commit_count is not None:
break
if new_commit_count != self.commit_count:
self.commit_count = new_commit_count
return True
return False
def after_poll(self, cursor, prev_tid_int, new_tid_int, changes):
"""Update checkpoint data after a database poll.
cursor is connected to a load connection.
changes lists all [(oid_int, tid_int)] changed after
prev_tid_int, up to and including new_tid_int, excluding the
changes last committed by the associated storage instance.
changes can be None to indicate too many objects changed
to list them all.
prev_tid_int can be None, in which case the changes
parameter will be ignored. new_tid_int can not be None.
"""
new_checkpoints = None
for client in self.clients_global_first:
s = client.get(self.checkpoints_key)
if s:
try:
c0, c1 = s.split()
c0 = int(c0)
c1 = int(c1)
except ValueError:
# Invalid checkpoint cache value; ignore it.
pass
else:
if c0 >= c1:
new_checkpoints = (c0, c1)
break
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
if not self.checkpoints:
# Initialize the checkpoints.
cache_data = '%d %d' % new_checkpoints
log.debug("Initializing checkpoints: %s", cache_data)
else:
# Suggest reinstatement of the former checkpoints, but
# use new checkpoints for this instance. Using new
# checkpoints ensures that we don't build up
# self.delta_after0 in case the cache is offline.
cache_data = '%d %d' % self.checkpoints
log.debug("Reinstating checkpoints: %s", cache_data)
for client in self.clients_global_first:
client.set(self.checkpoints_key, cache_data)
self.checkpoints = new_checkpoints
self.delta_after0 = {}
self.delta_after1 = {}
self.current_tid = new_tid_int
return
allow_shift = True
if new_checkpoints[0] > new_tid_int:
# checkpoint0 is in a future that this instance can't
# yet see. Ignore the checkpoint change for now.
new_checkpoints = self.checkpoints
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
allow_shift = False
# We want to keep the current checkpoints for speed, but we
# have to replace them (to avoid consistency violations)
# if certain conditions happen (like emptying the ZODB cache).
if (new_checkpoints == self.checkpoints
and changes is not None
and prev_tid_int
and prev_tid_int <= self.current_tid
and new_tid_int >= self.current_tid
):
# All the conditions for keeping the checkpoints were met,
# so just update self.delta_after0 and self.current_tid.
m = self.delta_after0
m_get = m.get
for oid_int, tid_int in changes:
my_tid_int = m_get(oid_int)
if my_tid_int is None or tid_int > my_tid_int:
m[oid_int] = tid_int
self.current_tid = new_tid_int
else:
# We have to replace the checkpoints.
cp0, cp1 = new_checkpoints
log.debug("Using new checkpoints: %d %d", cp0, cp1)
# Use the checkpoints specified by the cache.
# Rebuild delta_after0 and delta_after1.
new_delta_after0 = {}
new_delta_after1 = {}
if cp1 < new_tid_int:
# poller.list_changes provides an iterator of
# (oid, tid) where tid > after_tid and tid <= last_tid.
change_list = self.adapter.poller.list_changes(
cursor, cp1, new_tid_int)
# Make a dictionary that contains, for each oid, the most
# recent tid listed in changes.
change_dict = {}
if not isinstance(change_list, list):
change_list = list(change_list)
change_list.sort()
for oid_int, tid_int in change_list:
change_dict[oid_int] = tid_int
# Put the changes in new_delta_after*.
for oid_int, tid_int in change_dict.iteritems():
if tid_int > cp0:
new_delta_after0[oid_int] = tid_int
elif tid_int > cp1:
new_delta_after1[oid_int] = tid_int
self.checkpoints = new_checkpoints
self.delta_after0 = new_delta_after0
self.delta_after1 = new_delta_after1
self.current_tid = new_tid_int
if allow_shift and len(self.delta_after0) >= self.delta_size_limit:
# delta_after0 has reached its limit. The way to
# shrink it is to shift the checkpoints. Suggest
# shifted checkpoints for future polls.
# If delta_after0 is far over the limit (caused by a large
# transaction), suggest starting new checkpoints instead of
# shifting.
oversize = (len(self.delta_after0) >= self.delta_size_limit * 2)
self._suggest_shifted_checkpoints(new_tid_int, oversize)
def _suggest_shifted_checkpoints(self, tid_int, oversize):
"""Suggest that future polls use a new pair of checkpoints.
This does nothing if another instance has already shifted
the checkpoints.
checkpoint0 shifts to checkpoint1 and the tid just committed
becomes checkpoint0.
"""
cp0, cp1 = self.checkpoints
assert tid_int > cp0
expect = '%d %d' % self.checkpoints
if oversize:
# start new checkpoints
change_to = '%d %d' % (tid_int, tid_int)
else:
# shift the existing checkpoints
change_to = '%d %d' % (tid_int, cp0)
for client in self.clients_global_first:
old_value = client.get(self.checkpoints_key)
if old_value:
break
if not old_value or old_value == expect:
# Shift the checkpoints.
# Although this is a race with other instances, the race
# should not matter.
log.debug("Shifting checkpoints to: %s. len(delta_after0) == %d.",
change_to, len(self.delta_after0))
for client in self.clients_global_first:
client.set(self.checkpoints_key, change_to)
# The poll code will later see the new checkpoints
# and update self.checkpoints and self.delta_after(0|1).
else:
log.debug("Checkpoints already shifted to %s. "
"len(delta_after0) == %d.", old_value, len(self.delta_after0))
class StorageCache(object):
"""RelStorage integration with memcached or similar.
Holds a list of memcache clients in order from most local to
most global. The first is a LocalClient, which stores the cache
in the Python process, but shares the cache between threads.
"""
# pylint:disable=too-many-instance-attributes,too-many-public-methods
# send_limit: approximate limit on the bytes to buffer before
# sending to the cache.
send_limit = 1024 * 1024
# queue is an AutoTemporaryFile during transaction commit.
queue = None
# queue_contents is a map of {oid_int: (startpos, endpos)}
# during transaction commit.
queue_contents = None
# checkpoints, when set, is a tuple containing the integer
# transaction ID of the two current checkpoints. checkpoint0 is
# greater than or equal to checkpoint1.
checkpoints = None
# current_tid contains the last polled transaction ID. Invariant:
# when self.checkpoints is not None, self.delta_after0 has info
# from all transactions in the range:
# self.checkpoints[0] < tid <= self.current_tid
current_tid = 0
_tracer = None
# An LLBTree uses much less memory than a dict, and is still plenty fast on CPython;
# it's just as big and slower on PyPy, though.
_delta_map_type = BTrees.family64.II.BTree if not PYPY else dict
def __init__(self,
adapter,
options,
prefix,
local_client=None,
_tracer=None):
self.adapter = adapter
self.options = options
self.prefix = prefix or ''
# checkpoints_key holds the current checkpoints.
self.checkpoints_key = '%s:checkpoints' % self.prefix
assert isinstance(self.checkpoints_key, str) # no unicode on Py2
# delta_after0 contains {oid: tid} after checkpoint 0
# and before or at self.current_tid.
self.delta_after0 = self._delta_map_type()
# delta_after1 contains {oid: tid} after checkpoint 1 and
# before or at checkpoint 0. The content of delta_after1 only
# changes when checkpoints move.
self.delta_after1 = self._delta_map_type()
# delta_size_limit places an approximate limit on the number of
# entries in the delta_after maps.
self.delta_size_limit = options.cache_delta_size_limit
self.clients_local_first = []
if local_client is None:
self.clients_local_first.append(LocalClient(options, self.prefix))
else:
self.clients_local_first.append(local_client)
if options.cache_servers:
module_name = options.cache_module_name
module = importlib.import_module(module_name)
servers = options.cache_servers
if isinstance(servers, string_types):
servers = servers.split()
self.clients_local_first.append(module.Client(servers))
# self.clients_local_first is in order from local to global caches,
# while self.clients_global_first is in order from global to local.
self.clients_global_first = list(reversed(self.clients_local_first))
if local_client is None:
self.restore()
if _tracer is None:
tracefile = persistence.trace_file(options, self.prefix)
if tracefile:
_tracer = ZEOTracer(tracefile)
_tracer.trace(0x00)
self._tracer = _tracer
if hasattr(self._tracer, 'trace_store_current'):
self._trace = self._tracer.trace
self._trace_store_current = self._tracer.trace_store_current
# XXX: Note that our __bool__ and __len__ are NOT consistent
def __bool__(self):
return True
__nonzero__ = __bool__
def __len__(self):
if self.clients_local_first is _UsedAfterRelease:
return 0
return len(self.local_client)
@property
def size(self):
if self.clients_local_first is _UsedAfterRelease:
return 0
return self.local_client.size
@property
def limit(self):
if self.clients_local_first is _UsedAfterRelease:
return 0
return self.local_client.limit
@property
def local_client(self):
"""
The (shared) local in-memory cache client.
"""
return self.clients_local_first[0]
def stats(self):
"""
Return stats. This is a debugging aid only. The format is undefined and intended
for human inspection only.
"""
try:
local_client = self.local_client
except TypeError:
return {'closed': True}
else:
return local_client.stats()
def new_instance(self):
"""Return a copy of this instance sharing the same local client"""
local_client = self.local_client if self.options.share_local_cache else None
cache = type(self)(self.adapter,
self.options,
self.prefix,
local_client,
_tracer=self._tracer or False)
return cache
def release(self):
"""
Release resources held by this instance.
This is usually memcache connections if they're in use.
"""
clients = ()
if self.clients_local_first is not _UsedAfterRelease:
clients = self.clients_local_first
for client in clients:
client.disconnect_all()
# Release our clients. If we had a non-shared local cache,
# this will also allow it to release any memory its holding.
# Set them to non-iterables to make it obvious if we are used
# after release.
self.clients_local_first = _UsedAfterRelease
self.clients_global_first = _UsedAfterRelease
def save(self):
"""
Store any persistent client data.
"""
if self.options.cache_local_dir and len(self): # pylint:disable=len-as-condition
# (our __bool__ is not consistent with our len)
return persistence.save_local_cache(self.options, self.prefix,
self)
def write_to_stream(self, stream):
# We currently don't write anything to the stream, delegating instead
# just to the local client.
# We experimented with trying to save and load chcekpoints and
# the delta maps, but this turned out to be complex (because
# the `new_instance`s that have the actual data are released
# before we are, so their data gets lost, and we have to
# implement a parent/child relationship to fix that) and no
# more effective than relying on the default checkpoints we
# get from polling, if there have been no changes---at least
# in the case of zodbshootout benchmark (in fact, it was
# somewhat *slower*, for reasons that aren't fully clear).
# Note that if we did want to dump the delta maps, we would
# need to either wrap them in a dict or dump them pairwise; We
# can't dump a BTree larger than about 25000 without getting
# into recursion problems.
self.local_client.write_to_stream(stream)
def get_cache_modification_time_for_stream(self):
max_tid = 0
for key in self.local_client:
parts = key.split(':')
if len(parts) != 4:
continue
tid = int(parts[2])
max_tid = max(tid, max_tid)
if max_tid:
tid_str = p64(max_tid)
ts = TimeStamp(tid_str)
return ts.timeTime()
def restore(self):
options = self.options
if options.cache_local_dir:
persistence.load_local_cache(options, self.prefix, self)
def read_from_stream(self, stream):
return self.local_client.read_from_stream(stream)
def close(self):
"""
Release resources held by this instance, and
save any persistent data necessary.
"""
self.save()
self.release()
if self._tracer:
# Note we can't do this in release(). Release is called on
# all instances, while close() is only called on the main one.
self._tracer.close()
del self._trace
del self._trace_store_current
del self._tracer
def clear(self, load_persistent=True):
"""
Remove all data from the cache. Called by speed tests.
Starting from the introduction of persistent cache files,
this also results in the local client being repopulated with
the current set of persistent data. The *load_persistent* keyword can
be used to control this.
.. versionchanged:: 2.0b6
Added the ``load_persistent`` keyword. This argument is provisional.
"""
for client in self.clients_local_first:
client.flush_all()
self.checkpoints = None
self.delta_after0 = self._delta_map_type()
self.delta_after1 = self._delta_map_type()
self.current_tid = 0
if load_persistent:
self.restore()
@staticmethod
def _trace(*_args, **_kwargs): # pylint:disable=method-hidden
# Dummy method for when we don't do tracing
return
@staticmethod
def _trace_store_current(_tid_int, _items): # pylint:disable=method-hidden
# Dummy method for when we don't do tracing
return
def _check_tid_after_load(self,
oid_int,
actual_tid_int,
expect_tid_int=None):
"""Verify the tid of an object loaded from the database is sane."""
if actual_tid_int > self.current_tid:
# Strangely, the database just gave us data from a future
# transaction. We can't give the data to ZODB because that
# would be a consistency violation. However, the cause is hard
# to track down, so issue a ReadConflictError and hope that
# the application retries successfully.
msg = ("Got data for OID 0x%(oid_int)x from "
"future transaction %(actual_tid_int)d (%(got_ts)s). "
"Current transaction is %(current_tid)d (%(current_ts)s)." %
{
'oid_int': oid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'got_ts': str(TimeStamp(p64(actual_tid_int))),
'current_ts': str(TimeStamp(p64(self.current_tid))),
})
raise ReadConflictError(msg)
if expect_tid_int is not None and actual_tid_int != expect_tid_int:
# Uh-oh, the cache is inconsistent with the database.
# Possible causes:
#
# - The database MUST provide a snapshot view for each
# session; this error can occur if that requirement is
# violated. For example, MySQL's MyISAM engine is not
# sufficient for the object_state table because MyISAM
# can not provide a snapshot view. (InnoDB is
# sufficient.)
#
# - Something could be writing to the database out
# of order, such as a version of RelStorage that
# acquires a different commit lock.
#
# - A software bug. In the past, there was a subtle bug
# in after_poll() that caused it to ignore the
# transaction order, leading it to sometimes put the
# wrong tid in delta_after*.
cp0, cp1 = self.checkpoints
msg = ("Detected an inconsistency "
"between the RelStorage cache and the database "
"while loading an object using the delta_after0 dict. "
"Please verify the database is configured for "
"ACID compliance and that all clients are using "
"the same commit lock. "
"(oid_int=%(oid_int)r, expect_tid_int=%(expect_tid_int)r, "
"actual_tid_int=%(actual_tid_int)r, "
"current_tid=%(current_tid)r, cp0=%(cp0)r, cp1=%(cp1)r, "
"len(delta_after0)=%(lda0)r, len(delta_after1)=%(lda1)r, "
"pid=%(pid)r, thread_ident=%(thread_ident)r)" % {
'oid_int': oid_int,
'expect_tid_int': expect_tid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'cp0': cp0,
'cp1': cp1,
'lda0': len(self.delta_after0),
'lda1': len(self.delta_after1),
'pid': os.getpid(),
'thread_ident': threading.current_thread(),
})
raise AssertionError(msg)
def load(self, cursor, oid_int):
"""Load the given object from cache if possible.
Fall back to loading from the database.
"""
# pylint:disable=too-many-statements,too-many-branches,too-many-locals
if not self.checkpoints:
# No poll has occurred yet. For safety, don't use the cache.
self._trace(0x20, oid_int)
return self.adapter.mover.load_current(cursor, oid_int)
prefix = self.prefix
# Get the object from the transaction specified
# by the following values, in order:
#
# 1. delta_after0[oid_int]
# 2. checkpoints[0]
# 3. delta_after1[oid_int]
# 4. checkpoints[1]
# 5. The database.
#
# checkpoints[0] is the preferred location.
#
# If delta_after0 contains oid_int, we should not look at any
# other cache keys, since the tid_int specified in delta_after0
# replaces all older transaction IDs. Similarly, if
# delta_after1 contains oid_int, we should not look at
# checkpoints[1]. Also, when both checkpoints are set to the
# same transaction ID, we don't need to ask for the same key
# twice.
tid_int = self.delta_after0.get(oid_int)
if tid_int:
# This object changed after checkpoint0, so
# there is only one place to look for its state.
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
for client in self.clients_local_first:
cache_data = client.get(cachekey)
if cache_data and len(cache_data) >= 8:
# Cache hit.
# Note that we trace all cache hits, not just the local cache hit.
# This makes the simulation less useful, but the stats might still have
# value to people trying different tuning options manually.
self._trace(0x22,
oid_int,
tid_int,
dlen=len(cache_data) - 8)
assert cache_data[:8] == p64(tid_int)
return cache_data[8:], tid_int
# Cache miss.
self._trace(0x20, oid_int)
state, actual_tid_int = self.adapter.mover.load_current(
cursor, oid_int)
self._check_tid_after_load(oid_int, actual_tid_int, tid_int)
cache_data = p64(tid_int) + (state or b'')
for client in self.clients_local_first:
client.set(cachekey, cache_data)
return state, tid_int
# Make a list of cache keys to query. The list will have either
# 1 or 2 keys.
cp0, cp1 = self.checkpoints
cachekeys = []
cp0_key = '%s:state:%d:%d' % (prefix, cp0, oid_int)
cachekeys.append(cp0_key)
da1_key = None
cp1_key = None
tid_int = self.delta_after1.get(oid_int)
if tid_int:
da1_key = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
cachekeys.append(da1_key)
elif cp1 != cp0:
cp1_key = '%s:state:%d:%d' % (prefix, cp1, oid_int)
cachekeys.append(cp1_key)
for client in self.clients_local_first:
# Query the cache. Query multiple keys simultaneously to
# minimize latency.
response = client.get_multi(cachekeys)
if response:
cache_data = response.get(cp0_key)
if cache_data and len(cache_data) >= 8:
# Cache hit on the preferred cache key.
local_client = self.local_client
if client is not local_client:
# Copy to the local client.
local_client.set(cp0_key, cache_data)
self._trace(0x22,
oid_int,
u64(cache_data[:8]),
dlen=len(cache_data) - 8)
return cache_data[8:], u64(cache_data[:8])
if da1_key:
cache_data = response.get(da1_key)
elif cp1_key:
cache_data = response.get(cp1_key)
if cache_data and len(cache_data) >= 8:
# Cache hit, but copy the state to
# the currently preferred key.
self._trace(0x22,
oid_int,
u64(cache_data[:8]),
dlen=len(cache_data) - 8)
for client_to_set in self.clients_local_first:
client_to_set.set(cp0_key, cache_data)
return cache_data[8:], u64(cache_data[:8])
# Cache miss.
self._trace(0x20, oid_int)
state, tid_int = self.adapter.mover.load_current(cursor, oid_int)
if tid_int:
self._check_tid_after_load(oid_int, tid_int)
cache_data = p64(tid_int) + (state or b'')
# Record this as a store into the cache, ZEO does.
self._trace(0x52,
oid_int,
tid_int,
dlen=len(state) if state else 0)
for client in self.clients_local_first:
client.set(cp0_key, cache_data)
return state, tid_int
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
self.queue = AutoTemporaryFile()
self.queue_contents = {}
def store_temp(self, oid_int, state):
"""Queue an object for caching.
Typically, we can't actually cache the object yet, because its
transaction ID is not yet chosen.
"""
assert isinstance(state, bytes)
queue = self.queue
queue.seek(0, 2) # seek to end
startpos = queue.tell()
queue.write(state)
endpos = queue.tell()
self.queue_contents[oid_int] = (startpos, endpos)
def _read_temp_state(self, startpos, endpos):
self.queue.seek(startpos)
length = endpos - startpos
state = self.queue.read(length)
if len(state) != length:
raise AssertionError("Queued cache data is truncated")
return state, length
def read_temp(self, oid_int):
"""
Return the bytes for a previously stored temporary item.
"""
startpos, endpos = self.queue_contents[oid_int]
return self._read_temp_state(startpos, endpos)[0]
def send_queue(self, tid):
"""Now that this tid is known, send all queued objects to the cache"""
tid_int = u64(tid)
send_size = 0
to_send = {}
prefix = self.prefix
# Order the queue by file position, which should help if the
# file is large and needs to be read sequentially from disk.
items = [(startpos, endpos, oid_int)
for (oid_int, (startpos,
endpos)) in iteritems(self.queue_contents)]
items.sort()
# Trace these. This is the equivalent of ZEOs
# ClientStorage._update_cache.
self._trace_store_current(tid_int, items)
for startpos, endpos, oid_int in items:
state, length = self._read_temp_state(startpos, endpos)
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
item_size = length + len(cachekey)
if send_size and send_size + item_size >= self.send_limit:
for client in self.clients_local_first:
client.set_multi(to_send)
to_send.clear()
send_size = 0
to_send[cachekey] = tid + state
send_size += item_size
if to_send:
for client in self.clients_local_first:
client.set_multi(to_send)
self.queue_contents.clear()
self.queue.seek(0)
def after_tpc_finish(self, tid):
"""
Flush queued changes.
This is called after the database commit lock is released,
but before releasing the storage lock that will allow other
threads to use this instance.
"""
tid_int = u64(tid)
if self.checkpoints:
for oid_int in self.queue_contents:
# Future cache lookups for oid_int should now use
# the tid just committed.
self.delta_after0[oid_int] = tid_int
self.send_queue(tid)
def clear_temp(self):
"""Discard all transaction-specific temporary data.
Called after transaction finish or abort.
"""
self.queue_contents = None
if self.queue is not None:
self.queue.close()
self.queue = None
def after_poll(self, cursor, prev_tid_int, new_tid_int, changes):
"""Update checkpoint data after a database poll.
cursor is connected to a load connection.
changes lists all [(oid_int, tid_int)] changed after
prev_tid_int, up to and including new_tid_int, excluding the
changes last committed by the associated storage instance.
changes can be None to indicate too many objects changed
to list them all.
prev_tid_int can be None, in which case the changes
parameter will be ignored. new_tid_int can not be None.
"""
# pylint:disable=too-many-statements,too-many-branches,too-many-locals
new_checkpoints = None
for client in self.clients_global_first:
s = client.get(self.checkpoints_key)
if s:
try:
c0, c1 = s.split()
c0 = int(c0)
c1 = int(c1)
except ValueError:
# Invalid checkpoint cache value; ignore it.
pass
else:
if c0 >= c1:
new_checkpoints = (c0, c1)
break
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
if not self.checkpoints:
# Initialize the checkpoints.
cache_data = '%d %d' % new_checkpoints
log.debug("Initializing checkpoints: %s", cache_data)
else:
# Suggest reinstatement of the former checkpoints, but
# use new checkpoints for this instance. Using new
# checkpoints ensures that we don't build up
# self.delta_after0 in case the cache is offline.
cache_data = '%d %d' % self.checkpoints
log.debug("Reinstating checkpoints: %s", cache_data)
cache_data = cache_data.encode("ascii")
for client in self.clients_global_first:
client.set(self.checkpoints_key, cache_data)
self.checkpoints = new_checkpoints
self.delta_after0 = self._delta_map_type()
self.delta_after1 = self._delta_map_type()
self.current_tid = new_tid_int
return
allow_shift = True
if new_checkpoints[0] > new_tid_int:
# checkpoint0 is in a future that this instance can't
# yet see. Ignore the checkpoint change for now.
new_checkpoints = self.checkpoints
if not new_checkpoints:
new_checkpoints = (new_tid_int, new_tid_int)
allow_shift = False
# We want to keep the current checkpoints for speed, but we
# have to replace them (to avoid consistency violations)
# if certain conditions happen (like emptying the ZODB cache).
if (new_checkpoints == self.checkpoints and changes is not None
and prev_tid_int and prev_tid_int <= self.current_tid
and new_tid_int >= self.current_tid):
# All the conditions for keeping the checkpoints were met,
# so just update self.delta_after0 and self.current_tid.
m = self.delta_after0
m_get = m.get
for oid_int, tid_int in changes:
my_tid_int = m_get(oid_int)
if my_tid_int is None or tid_int > my_tid_int:
m[oid_int] = tid_int
# 0x1E = invalidate (hit, saving non-current)
self._trace(0x1C, oid_int, tid_int)
self.current_tid = new_tid_int
else:
# We have to replace the checkpoints.
cp0, cp1 = new_checkpoints
log.debug("Using new checkpoints: %d %d", cp0, cp1)
# Use the checkpoints specified by the cache.
# Rebuild delta_after0 and delta_after1.
new_delta_after0 = self._delta_map_type()
new_delta_after1 = self._delta_map_type()
if cp1 < new_tid_int:
# poller.list_changes provides an iterator of
# (oid, tid) where tid > after_tid and tid <= last_tid.
change_list = self.adapter.poller.list_changes(
cursor, cp1, new_tid_int)
# Make a dictionary that contains, for each oid, the most
# recent tid listed in changes. This works because sorting the
# (oid, tid) pairs puts the newest tid at the back, and constructing
# the dictionary from that sorted list preserves order, keeping the
# last key that it saw.
try:
change_dict = self._delta_map_type(sorted(change_list))
except TypeError:
# pg8000 returns a list of lists, not a list of tuples. The
# BTree constructor is very particular about that. Normally one
# would use pg8000 on PyPy, where we don't use BTrees, so this shouldn't
# actually come up in practice.
change_dict = self._delta_map_type()
for oid_int, tid_int in sorted(change_list):
change_dict[oid_int] = tid_int
# Put the changes in new_delta_after*.
for oid_int, tid_int in change_dict.items():
# 0x1E = invalidate (hit, saving non-current)
self._trace(0x1C, oid_int, tid_int)
if tid_int > cp0:
new_delta_after0[oid_int] = tid_int
elif tid_int > cp1:
new_delta_after1[oid_int] = tid_int
self.checkpoints = new_checkpoints
self.delta_after0 = new_delta_after0
self.delta_after1 = new_delta_after1
self.current_tid = new_tid_int
if allow_shift and len(self.delta_after0) >= self.delta_size_limit:
# delta_after0 has reached its limit. The way to
# shrink it is to shift the checkpoints. Suggest
# shifted checkpoints for future polls.
# If delta_after0 is far over the limit (caused by a large
# transaction), suggest starting new checkpoints instead of
# shifting.
oversize = (len(self.delta_after0) >= self.delta_size_limit * 2)
self._suggest_shifted_checkpoints(new_tid_int, oversize)
def _suggest_shifted_checkpoints(self, tid_int, oversize):
"""Suggest that future polls use a new pair of checkpoints.
This does nothing if another instance has already shifted
the checkpoints.
checkpoint0 shifts to checkpoint1 and the tid just committed
becomes checkpoint0.
"""
cp0, _cp1 = self.checkpoints
assert tid_int > cp0
expect = '%d %d' % self.checkpoints
if oversize:
# start new checkpoints
change_to = '%d %d' % (tid_int, tid_int)
else:
# shift the existing checkpoints
change_to = '%d %d' % (tid_int, cp0)
expect = expect.encode('ascii')
change_to = change_to.encode('ascii')
for client in self.clients_global_first:
old_value = client.get(self.checkpoints_key)
if old_value:
break
if not old_value or old_value == expect:
# Shift the checkpoints.
# Although this is a race with other instances, the race
# should not matter.
log.debug("Shifting checkpoints to: %s. len(delta_after0) == %d.",
change_to, len(self.delta_after0))
for client in self.clients_global_first:
client.set(self.checkpoints_key, change_to)
# The poll code will later see the new checkpoints
# and update self.checkpoints and self.delta_after(0|1).
else:
log.debug(
"Checkpoints already shifted to %s. "
"len(delta_after0) == %d.", old_value, len(self.delta_after0))
def tpc_begin(self):
"""Prepare temp space for objects to cache."""
# start with a fresh in-memory buffer instead of reusing one that might
# already be spooled to disk.
self.queue = AutoTemporaryFile()
self.queue_contents = {}
