def set_multi(self, d):
if not self.limit:
# don't bother
return
compress = self._compress
items = [] # [(key, value)]
for key, value in iteritems(d):
# This used to allow non-byte values, but that's confusing
# on Py3 and wasn't used outside of tests, so we enforce it.
assert isinstance(key, str), (type(key), key)
assert isinstance(value, bytes)
cvalue = compress(value) if compress else value # pylint:disable=not-callable
if len(cvalue) >= self._value_limit:
# This value is too big, so don't cache it.
continue
items.append((key, cvalue))
bucket0 = self.__bucket
set_key = bucket0.__setitem__
with self._lock:
for key, cvalue in items:
set_key(key, cvalue)
def run_script_stmt(self, cursor, generic_stmt, generic_params=()):
"""Execute a statement from a script with the given parameters.
params should be either an empty tuple (no parameters) or
a map.
"""
if generic_params:
# Oracle raises ORA-01036 if the parameter map contains extra keys,
# so filter out any unused parameters.
tracker = TrackingMap(self.script_vars)
stmt = generic_stmt % tracker
used = tracker.used
params = {}
for k, v in iteritems(generic_params):
if k in used:
params[k] = v
else:
stmt = generic_stmt % self.script_vars
params = ()
try:
cursor.execute(stmt, params)
except:
log.warning("script statement failed: %r; parameters: %r",
stmt, params)
raise
def send_queue(self, tid):
"""Now that this tid is known, send all queued objects to the cache"""
tid_int = u64(tid)
send_size = 0
to_send = {}
prefix = self.prefix
# Order the queue by file position, which should help if the
# file is large and needs to be read sequentially from disk.
items = [
(startpos, endpos, oid_int)
for (oid_int, (startpos, endpos)) in iteritems(self.queue_contents)
]
items.sort()
# Trace these. This is the equivalent of ZEOs
# ClientStorage._update_cache.
self._trace_store_current(tid_int, items)
for startpos, endpos, oid_int in items:
state, length = self._read_temp_state(startpos, endpos)
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
item_size = length + len(cachekey)
if send_size and send_size + item_size >= self.send_limit:
for client in self.clients_local_first:
client.set_multi(to_send)
to_send.clear()
send_size = 0
to_send[cachekey] = tid + state
send_size += item_size
if to_send:
for client in self.clients_local_first:
client.set_multi(to_send)
self.queue_contents.clear()
self.queue.seek(0)
def _do_inserts(self):
count = 0
# In the case that we have only a single table we're inserting into,
# and thus common key values, we don't need to sort or iterate.
# If we have multiple tables, we never want to sort by the rows too,
# that doesn't make any sense.
if len(self.inserts) == 1:
items = [self.inserts.popitem()]
else:
items = sorted(iteritems(self.inserts), key=lambda i: i[0])
for (command, header, row_schema, suffix), rows in items:
# Batched inserts
rows = list(rows.values())
count += len(rows)
value_template = "(%s)" % row_schema
values_template = [value_template] * len(rows)
params = list(itertools.chain.from_iterable(rows))
stmt = "%s INTO %s VALUES\n%s\n%s" % (
command, header, ',\n'.join(values_template), suffix)
# e.g.,
# INSERT INTO table(c1, c2)
# VALUES (%s, %s), (%s, %s), (%s, %s)
# <suffix>
__traceback_info__ = stmt
self.cursor.execute(stmt, params)
return count
def set_multi(self, d):
if not self.limit:
# don't bother
return
compress = self._compress
items = [] # [(key, value)]
for key, value in iteritems(d):
# This used to allow non-byte values, but that's confusing
# on Py3 and wasn't used outside of tests, so we enforce it.
assert isinstance(key, str), (type(key), key)
assert isinstance(value, bytes)
cvalue = compress(value) if compress else value # pylint:disable=not-callable
if len(cvalue) >= self._value_limit:
# This value is too big, so don't cache it.
continue
items.append((key, cvalue))
bucket0 = self.__bucket
set_key = bucket0.__setitem__
with self._lock:
for key, cvalue in items:
set_key(key, cvalue)
def run_script_stmt(self, cursor, generic_stmt, generic_params=()):
"""Execute a statement from a script with the given parameters.
params should be either an empty tuple (no parameters) or
a map.
"""
if generic_params:
# Oracle raises ORA-01036 if the parameter map contains extra keys,
# so filter out any unused parameters.
tracker = TrackingMap(self.script_vars)
stmt = generic_stmt % tracker
used = tracker.used
params = {}
for k, v in iteritems(generic_params):
if k in used:
params[k] = v
else:
stmt = generic_stmt % self.script_vars
params = ()
try:
cursor.execute(stmt, params)
except:
log.warning("script statement failed: %r; parameters: %r",
stmt, params)
raise
def send_queue(self, tid):
"""Now that this tid is known, send all queued objects to the cache"""
tid_int = u64(tid)
send_size = 0
to_send = {}
prefix = self.prefix
# Order the queue by file position, which should help if the
# file is large and needs to be read sequentially from disk.
items = [(startpos, endpos, oid_int)
for (oid_int, (startpos,
endpos)) in iteritems(self.queue_contents)]
items.sort()
# Trace these. This is the equivalent of ZEOs
# ClientStorage._update_cache.
self._trace_store_current(tid_int, items)
for startpos, endpos, oid_int in items:
state, length = self._read_temp_state(startpos, endpos)
cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int)
item_size = length + len(cachekey)
if send_size and send_size + item_size >= self.send_limit:
for client in self.clients_local_first:
client.set_multi(to_send)
to_send.clear()
send_size = 0
to_send[cachekey] = tid + state
send_size += item_size
if to_send:
for client in self.clients_local_first:
client.set_multi(to_send)
self.queue_contents.clear()
self.queue.seek(0)
def _mark_pass(self, this_pass):
"""Mark OIDs as reachable. Produce an OID set for the next pass.
Return (found, next_pass), where `found` is the number of
new OIDs marked and `next_pass` is the collection of OIDs to
follow in the next pass.
"""
# pylint:disable=too-many-locals
# next_pass: {oid_hi: IISet32X}
next_pass = collections.defaultdict(IISet32X)
found = 0
refs = self._refs
reachable = self._reachable
lo = self.lo
for oid_hi, oids_lo in iteritems(this_pass):
from_reachable_set = reachable[oid_hi]
for oid_lo in oids_lo:
if oid_lo in from_reachable_set:
# This OID is already known to be reachable.
continue
found += 1
from_reachable_set.add(oid_lo)
if oid_hi not in refs:
# This OID doesn't reference anything.
continue
# Add the children of this OID to next_pass.
for to_oid_hi, s in iteritems(refs[oid_hi]):
min_key = oid_lo << 32
max_key = min_key | 0xffffffff
keys = s.keys(min=min_key, max=max_key)
if not keys:
# No references found here.
continue
to_reachable_set = reachable[to_oid_hi]
next_pass_add = next_pass[to_oid_hi].add
for key in keys:
child_oid_lo = int(key & lo)
if child_oid_lo not in to_reachable_set:
next_pass_add(child_oid_lo)
return found, next_pass
def _mark_pass(self, this_pass):
"""Mark OIDs as reachable. Produce an OID set for the next pass.
Return (found, next_pass), where `found` is the number of
new OIDs marked and `next_pass` is the collection of OIDs to
follow in the next pass.
"""
# pylint:disable=too-many-locals
# next_pass: {oid_hi: IISet32X}
next_pass = collections.defaultdict(IISet32X)
found = 0
refs = self._refs
reachable = self._reachable
lo = self.lo
for oid_hi, oids_lo in iteritems(this_pass):
from_reachable_set = reachable[oid_hi]
for oid_lo in oids_lo:
if oid_lo in from_reachable_set:
# This OID is already known to be reachable.
continue
found += 1
from_reachable_set.add(oid_lo)
if oid_hi not in refs:
# This OID doesn't reference anything.
continue
# Add the children of this OID to next_pass.
for to_oid_hi, s in iteritems(refs[oid_hi]):
min_key = oid_lo << 32
max_key = min_key | 0xffffffff
keys = s.keys(min=min_key, max=max_key)
if not keys:
# No references found here.
continue
to_reachable_set = reachable[to_oid_hi]
next_pass_add = next_pass[to_oid_hi].add
for key in keys:
child_oid_lo = int(key & lo)
if child_oid_lo not in to_reachable_set:
next_pass_add(child_oid_lo)
return found, next_pass
def abort(self):
if self._txn_blobs:
for _oid, filename in iteritems(self._txn_blobs):
if os.path.exists(filename):
ZODB.blob.remove_committed(filename)
if self.shared_blob_dir:
dirname = os.path.dirname(filename)
if not _has_files(dirname):
ZODB.blob.remove_committed_dir(dirname)
def abort(self):
if self._txn_blobs:
for _oid, filename in iteritems(self._txn_blobs):
if os.path.exists(filename):
ZODB.blob.remove_committed(filename)
if self.shared_blob_dir:
dirname = os.path.dirname(filename)
if not _has_files(dirname):
ZODB.blob.remove_committed_dir(dirname)
def abort(self):
try:
if not self._txn_blobs:
return
for _oid, filename in iteritems(self._txn_blobs):
if os.path.exists(filename):
ZODB.blob.remove_committed(filename)
self._abort_filename(filename)
finally:
self.clear_temp()
def __str__(self):
parts = [self.__class__.__name__]
if self.keep_history:
parts.append('history preserving')
else:
parts.append('history free')
p = self._params.copy()
if 'passwd' in p:
del p['passwd']
p = sorted(iteritems(p))
parts.extend('%s=%r' % item for item in p)
return ", ".join(parts)
def __str__(self):
parts = [self.__class__.__name__]
if self.keep_history:
parts.append('history preserving')
else:
parts.append('history free')
p = self._params.copy()
if 'passwd' in p:
del p['passwd']
p = sorted(iteritems(p))
parts.extend('%s=%r' % item for item in p)
return ", ".join(parts)
def __str__(self):
parts = [self.__class__.__name__]
if self.keep_history:
parts.append("history preserving")
else:
parts.append("history free")
p = self._params.copy()
if "passwd" in p:
del p["passwd"]
p = sorted(iteritems(p))
parts.extend("%s=%r" % item for item in p)
return ", ".join(parts)
def get_multi(self, keys):
res = {}
decompress = self._decompress
get = self.__bucket.get_and_bubble_all
with self._lock:
res = get(keys)
# Finally, while not holding the lock, decompress if needed
res = {k: decompress(v) for k, v in iteritems(res)}
return res
def _do_inserts(self):
items = sorted(iteritems(self.inserts))
for (command, header, row_schema), rows in items:
# Batched inserts
parts = []
params = []
s = "(%s)" % row_schema
for row in rows.values():
parts.append(s)
params.extend(row)
parts = ",\n".join(parts)
stmt = "%s INTO %s VALUES\n%s" % (command, header, parts)
self.cursor.execute(stmt, tuple(params))
def get_multi(self, keys):
res = {}
decompress = self._decompress
get = self.__bucket.get_and_bubble_all
with self._lock:
res = get(keys)
# Finally, while not holding the lock, decompress if needed
res = {k: decompress(v)
for k, v in iteritems(res)}
return res
def _do_inserts(self):
items = sorted(iteritems(self.inserts))
for (command, header, row_schema), rows in items:
# Batched inserts
parts = []
params = []
s = "(%s)" % row_schema
for row in rows.values():
parts.append(s)
params.extend(row)
parts = ',\n'.join(parts)
stmt = "%s INTO %s VALUES\n%s" % (command, header, parts)
self.cursor.execute(stmt, tuple(params))
def _do_deletes(self):
for (table, columns), rows in sorted(iteritems(self.deletes)):
rows = list(sorted(rows))
if len(columns) == 1:
value_str = ",".join(v for (v,) in rows)
stmt = "DELETE FROM %s WHERE %s IN (%s)" % (table, columns[0], value_str)
else:
lines = []
for row in rows:
line = []
for i, column in enumerate(columns):
line.append("%s = %s" % (column, row[i]))
lines.append(" AND ".join(line))
stmt = "DELETE FROM %s WHERE %s" % (table, " OR ".join(lines))
self.cursor.execute(stmt)
def _do_deletes(self):
for (table, columns), rows in sorted(iteritems(self.deletes)):
rows = list(sorted(rows))
if len(columns) == 1:
value_str = ','.join(v for (v, ) in rows)
stmt = "DELETE FROM %s WHERE %s IN (%s)" % (table, columns[0],
value_str)
else:
lines = []
for row in rows:
line = []
for i, column in enumerate(columns):
line.append("%s = %s" % (column, row[i]))
lines.append(" AND ".join(line))
stmt = "DELETE FROM %s WHERE %s" % (table, " OR ".join(lines))
self.cursor.execute(stmt)
def _do_inserts(self):
items = sorted(iteritems(self.inserts))
for (command, header, row_schema, suffix), rows in items:
# Batched inserts
parts = []
params = []
s = "(%s)" % row_schema
for row in itervalues(rows):
parts.append(s)
params.extend(row)
stmt = "%s INTO %s VALUES\n%s\n%s" % (
command, header, ',\n'.join(parts), suffix)
# e.g.,
# INSERT INTO table(c1, c2)
# VALUES (%s, %s), (%s, %s), (%s, %s)
# <suffix>
self.cursor.execute(stmt, tuple(params))
def _do_inserts(self):
items = sorted(iteritems(self.inserts))
for (command, header, row_schema, suffix), rows in items:
# Batched inserts
parts = []
params = []
s = "(%s)" % row_schema
for row in itervalues(rows):
parts.append(s)
params.extend(row)
stmt = "%s INTO %s VALUES\n%s\n%s" % (
command, header, ',\n'.join(parts), suffix)
# e.g.,
# INSERT INTO table(c1, c2)
# VALUES (%s, %s), (%s, %s), (%s, %s)
# <suffix>
self.cursor.execute(stmt, tuple(params))
def _do_deletes(self):
for (table, columns), rows in sorted(iteritems(self.deletes)):
# XXX: Stop doing string conversion manually. Let the
# cursor do it. It may have a non-text protocol for integer
# objects; it may also have a different representation in text.
if len(columns) == 1:
value_str = ','.join(str(v) for (v,) in rows)
stmt = "DELETE FROM %s WHERE %s IN (%s)" % (
table, columns[0], value_str)
else:
lines = []
for row in rows:
line = []
for i, column in enumerate(columns):
line.append("%s = %s" % (column, row[i]))
lines.append(" AND ".join(line))
stmt = "DELETE FROM %s WHERE %s" % (
table, " OR ".join(lines))
self.cursor.execute(stmt)
def _do_deletes(self):
for (table, columns), rows in sorted(iteritems(self.deletes)):
# XXX: Stop doing string conversion manually. Let the
# cursor do it. It may have a non-text protocol for integer
# objects; it may also have a different representation in text.
if len(columns) == 1:
value_str = ','.join(str(v) for (v,) in rows)
stmt = "DELETE FROM %s WHERE %s IN (%s)" % (
table, columns[0], value_str)
else:
lines = []
for row in rows:
line = []
for i, column in enumerate(columns):
line.append("%s = %s" % (column, row[i]))
lines.append(" AND ".join(line))
stmt = "DELETE FROM %s WHERE %s" % (
table, " OR ".join(lines))
self.cursor.execute(stmt)
def run_script_stmt(self, cursor, generic_stmt, generic_params=()):
"""Execute a statement from a script with the given parameters.
params should be either an empty tuple (no parameters) or
a map.
"""
generic_stmt = generic_stmt.format(**self.format_vars)
# We can't quote "transaction", but we have to for sqlite.
generic_stmt = generic_stmt.replace(' "transaction"', ' transaction')
if generic_params and isinstance(generic_params, tuple):
generic_stmt = _format_to_named(generic_stmt) # Unnamed params become numbered.
if generic_params and isinstance(generic_params, dict):
# Oracle raises ORA-01036 if the parameter map contains extra keys,
# so filter out any unused parameters.
tracker = TrackingMap(self.script_vars)
stmt = generic_stmt % tracker
used = tracker.used
params = {}
for k, v in iteritems(generic_params):
if k in used:
params[k] = v
else:
stmt = generic_stmt % self.script_vars
params = ()
if generic_params and isinstance(generic_params, tuple):
params = generic_params
__traceback_info__ = stmt
try:
cursor.execute(stmt, params)
except:
log.warning("script statement failed: %r; parameters: %r",
stmt, params)
raise
def reachable(self):
"""Iterate over all the reachable OIDs."""
for oid_hi, oids_lo in iteritems(self._reachable):
for oid_lo in oids_lo:
# Decode the OID.
yield oid_hi | oid_lo
def reachable(self):
"""Iterate over all the reachable OIDs."""
for oid_hi, oids_lo in iteritems(self._reachable):
for oid_lo in oids_lo:
# Decode the OID.
yield oid_hi | oid_lo
def _set_multi(self, keys_and_values):
formatted = {
'%s:state:%d:%d' % (self.prefix, tid, oid): (p64(actual_tid) + (state or b''))
for (oid, tid), (state, actual_tid) in iteritems(keys_and_values)
}
self.client.set_multi(formatted)
def compare_exact(self, storage1, storage2):
"""Confirm that storage1 and storage2 contain equivalent data"""
eq = self.assertEqual
missing = object()
iter1 = storage1.iterator()
iter2 = storage2.iterator()
for txn1, txn2 in zip(iter1, iter2):
eq(txn1.tid, txn2.tid)
eq(txn1.status, txn2.status)
eq(txn1.user, txn2.user)
eq(txn1.description, txn2.description)
# b/w compat on the 'extension' attribute
e1 = getattr(txn1, 'extension', missing)
if e1 is missing:
# old attribute name
e1 = txn1._extension
e2 = getattr(txn2, 'extension', missing)
if e2 is missing:
# old attribute name
e2 = txn2._extension
eq(e1, e2)
# compare the objects in the transaction, but disregard
# the order of the objects and any duplicated records
# since those are not important.
recs1 = dict([(r.oid, r) for r in txn1])
recs2 = dict([(r.oid, r) for r in txn2])
eq(len(recs1), len(recs2))
recs1 = sorted(iteritems(recs1))
recs2 = sorted(iteritems(recs2))
recs2.sort()
for (_oid1, rec1), (_oid2, rec2) in zip(recs1, recs2):
eq(rec1.oid, rec2.oid)
eq(rec1.tid, rec2.tid)
eq(rec1.data, rec2.data)
if is_blob_record(rec1.data):
try:
fn1 = storage1.loadBlob(rec1.oid, rec1.tid)
except ZODB.POSException.POSKeyError:
self.assertRaises(ZODB.POSException.POSKeyError,
storage2.loadBlob, rec1.oid,
rec1.tid)
else:
fn2 = storage2.loadBlob(rec1.oid, rec1.tid)
self.assertNotEqual(fn1, fn2)
with open(fn1, 'rb') as f1, open(fn2, 'rb') as f2:
eq(f1.read(), f2.read())
# Make sure ther are no more records left in txn1 and txn2, meaning
# they were the same length
try:
next(iter1)
except (IndexError, StopIteration):
pass
else:
self.fail("storage1 has more records")
try:
next(iter2)
except (IndexError, StopIteration):
pass
else:
self.fail("storage2 has more records")
iter1.close()
iter2.close()
def _do_deletes(self):
return self._do_batch('DELETE', sorted(iteritems(self.deletes)))
def compare_exact(self, storage1, storage2):
"""Confirm that storage1 and storage2 contain equivalent data"""
eq = self.assertEqual
missing = object()
iter1 = storage1.iterator()
iter2 = storage2.iterator()
for txn1, txn2 in zip(iter1, iter2):
eq(txn1.tid, txn2.tid)
eq(txn1.status, txn2.status)
eq(txn1.user, txn2.user)
eq(txn1.description, txn2.description)
# b/w compat on the 'extension' attribute
e1 = getattr(txn1, 'extension', missing)
if e1 is missing:
# old attribute name
e1 = txn1._extension
e2 = getattr(txn2, 'extension', missing)
if e2 is missing:
# old attribute name
e2 = txn2._extension
eq(e1, e2)
# compare the objects in the transaction, but disregard
# the order of the objects and any duplicated records
# since those are not important.
recs1 = {r.oid: r for r in txn1}
recs2 = {r.oid: r for r in txn1}
eq(len(recs1), len(recs2))
recs1 = sorted(iteritems(recs1))
recs2 = sorted(iteritems(recs2))
recs2.sort()
for (_oid1, rec1), (_oid2, rec2) in zip(recs1, recs2):
eq(rec1.oid, rec2.oid)
eq(rec1.tid, rec2.tid)
eq(rec1.data, rec2.data)
if is_blob_record(rec1.data):
try:
fn1 = storage1.loadBlob(rec1.oid, rec1.tid)
except ZODB.POSException.POSKeyError:
self.assertRaises(
ZODB.POSException.POSKeyError,
storage2.loadBlob, rec1.oid, rec1.tid)
else:
fn2 = storage2.loadBlob(rec1.oid, rec1.tid)
self.assertNotEqual(fn1, fn2)
with open(fn1, 'rb') as f1, open(fn2, 'rb') as f2:
eq(f1.read(), f2.read())
# Make sure ther are no more records left in txn1 and txn2, meaning
# they were the same length
try:
next(iter1)
except (IndexError, StopIteration):
pass
else:
self.fail("storage1 has more records")
try:
next(iter2)
except (IndexError, StopIteration):
pass
else:
self.fail("storage2 has more records")
iter1.close()
iter2.close()
