def set_multi(self, d): if not self.limit: # don't bother return compress = self._compress items = [] # [(key, value)] for key, value in iteritems(d): # This used to allow non-byte values, but that's confusing # on Py3 and wasn't used outside of tests, so we enforce it. assert isinstance(key, str), (type(key), key) assert isinstance(value, bytes) cvalue = compress(value) if compress else value # pylint:disable=not-callable if len(cvalue) >= self._value_limit: # This value is too big, so don't cache it. continue items.append((key, cvalue)) bucket0 = self.__bucket set_key = bucket0.__setitem__ with self._lock: for key, cvalue in items: set_key(key, cvalue)
def run_script_stmt(self, cursor, generic_stmt, generic_params=()): """Execute a statement from a script with the given parameters. params should be either an empty tuple (no parameters) or a map. """ if generic_params: # Oracle raises ORA-01036 if the parameter map contains extra keys, # so filter out any unused parameters. tracker = TrackingMap(self.script_vars) stmt = generic_stmt % tracker used = tracker.used params = {} for k, v in iteritems(generic_params): if k in used: params[k] = v else: stmt = generic_stmt % self.script_vars params = () try: cursor.execute(stmt, params) except: log.warning("script statement failed: %r; parameters: %r", stmt, params) raise
def send_queue(self, tid): """Now that this tid is known, send all queued objects to the cache""" tid_int = u64(tid) send_size = 0 to_send = {} prefix = self.prefix # Order the queue by file position, which should help if the # file is large and needs to be read sequentially from disk. items = [ (startpos, endpos, oid_int) for (oid_int, (startpos, endpos)) in iteritems(self.queue_contents) ] items.sort() # Trace these. This is the equivalent of ZEOs # ClientStorage._update_cache. self._trace_store_current(tid_int, items) for startpos, endpos, oid_int in items: state, length = self._read_temp_state(startpos, endpos) cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int) item_size = length + len(cachekey) if send_size and send_size + item_size >= self.send_limit: for client in self.clients_local_first: client.set_multi(to_send) to_send.clear() send_size = 0 to_send[cachekey] = tid + state send_size += item_size if to_send: for client in self.clients_local_first: client.set_multi(to_send) self.queue_contents.clear() self.queue.seek(0)
def _do_inserts(self): count = 0 # In the case that we have only a single table we're inserting into, # and thus common key values, we don't need to sort or iterate. # If we have multiple tables, we never want to sort by the rows too, # that doesn't make any sense. if len(self.inserts) == 1: items = [self.inserts.popitem()] else: items = sorted(iteritems(self.inserts), key=lambda i: i[0]) for (command, header, row_schema, suffix), rows in items: # Batched inserts rows = list(rows.values()) count += len(rows) value_template = "(%s)" % row_schema values_template = [value_template] * len(rows) params = list(itertools.chain.from_iterable(rows)) stmt = "%s INTO %s VALUES\n%s\n%s" % ( command, header, ',\n'.join(values_template), suffix) # e.g., # INSERT INTO table(c1, c2) # VALUES (%s, %s), (%s, %s), (%s, %s) # <suffix> __traceback_info__ = stmt self.cursor.execute(stmt, params) return count
def send_queue(self, tid): """Now that this tid is known, send all queued objects to the cache""" tid_int = u64(tid) send_size = 0 to_send = {} prefix = self.prefix # Order the queue by file position, which should help if the # file is large and needs to be read sequentially from disk. items = [(startpos, endpos, oid_int) for (oid_int, (startpos, endpos)) in iteritems(self.queue_contents)] items.sort() # Trace these. This is the equivalent of ZEOs # ClientStorage._update_cache. self._trace_store_current(tid_int, items) for startpos, endpos, oid_int in items: state, length = self._read_temp_state(startpos, endpos) cachekey = '%s:state:%d:%d' % (prefix, tid_int, oid_int) item_size = length + len(cachekey) if send_size and send_size + item_size >= self.send_limit: for client in self.clients_local_first: client.set_multi(to_send) to_send.clear() send_size = 0 to_send[cachekey] = tid + state send_size += item_size if to_send: for client in self.clients_local_first: client.set_multi(to_send) self.queue_contents.clear() self.queue.seek(0)
def _mark_pass(self, this_pass): """Mark OIDs as reachable. Produce an OID set for the next pass. Return (found, next_pass), where `found` is the number of new OIDs marked and `next_pass` is the collection of OIDs to follow in the next pass. """ # pylint:disable=too-many-locals # next_pass: {oid_hi: IISet32X} next_pass = collections.defaultdict(IISet32X) found = 0 refs = self._refs reachable = self._reachable lo = self.lo for oid_hi, oids_lo in iteritems(this_pass): from_reachable_set = reachable[oid_hi] for oid_lo in oids_lo: if oid_lo in from_reachable_set: # This OID is already known to be reachable. continue found += 1 from_reachable_set.add(oid_lo) if oid_hi not in refs: # This OID doesn't reference anything. continue # Add the children of this OID to next_pass. for to_oid_hi, s in iteritems(refs[oid_hi]): min_key = oid_lo << 32 max_key = min_key | 0xffffffff keys = s.keys(min=min_key, max=max_key) if not keys: # No references found here. continue to_reachable_set = reachable[to_oid_hi] next_pass_add = next_pass[to_oid_hi].add for key in keys: child_oid_lo = int(key & lo) if child_oid_lo not in to_reachable_set: next_pass_add(child_oid_lo) return found, next_pass
def abort(self): if self._txn_blobs: for _oid, filename in iteritems(self._txn_blobs): if os.path.exists(filename): ZODB.blob.remove_committed(filename) if self.shared_blob_dir: dirname = os.path.dirname(filename) if not _has_files(dirname): ZODB.blob.remove_committed_dir(dirname)
def abort(self): try: if not self._txn_blobs: return for _oid, filename in iteritems(self._txn_blobs): if os.path.exists(filename): ZODB.blob.remove_committed(filename) self._abort_filename(filename) finally: self.clear_temp()
def __str__(self): parts = [self.__class__.__name__] if self.keep_history: parts.append('history preserving') else: parts.append('history free') p = self._params.copy() if 'passwd' in p: del p['passwd'] p = sorted(iteritems(p)) parts.extend('%s=%r' % item for item in p) return ", ".join(parts)
def __str__(self): parts = [self.__class__.__name__] if self.keep_history: parts.append("history preserving") else: parts.append("history free") p = self._params.copy() if "passwd" in p: del p["passwd"] p = sorted(iteritems(p)) parts.extend("%s=%r" % item for item in p) return ", ".join(parts)
def get_multi(self, keys): res = {} decompress = self._decompress get = self.__bucket.get_and_bubble_all with self._lock: res = get(keys) # Finally, while not holding the lock, decompress if needed res = {k: decompress(v) for k, v in iteritems(res)} return res
def _do_inserts(self): items = sorted(iteritems(self.inserts)) for (command, header, row_schema), rows in items: # Batched inserts parts = [] params = [] s = "(%s)" % row_schema for row in rows.values(): parts.append(s) params.extend(row) parts = ",\n".join(parts) stmt = "%s INTO %s VALUES\n%s" % (command, header, parts) self.cursor.execute(stmt, tuple(params))
def _do_inserts(self): items = sorted(iteritems(self.inserts)) for (command, header, row_schema), rows in items: # Batched inserts parts = [] params = [] s = "(%s)" % row_schema for row in rows.values(): parts.append(s) params.extend(row) parts = ',\n'.join(parts) stmt = "%s INTO %s VALUES\n%s" % (command, header, parts) self.cursor.execute(stmt, tuple(params))
def _do_deletes(self): for (table, columns), rows in sorted(iteritems(self.deletes)): rows = list(sorted(rows)) if len(columns) == 1: value_str = ",".join(v for (v,) in rows) stmt = "DELETE FROM %s WHERE %s IN (%s)" % (table, columns[0], value_str) else: lines = [] for row in rows: line = [] for i, column in enumerate(columns): line.append("%s = %s" % (column, row[i])) lines.append(" AND ".join(line)) stmt = "DELETE FROM %s WHERE %s" % (table, " OR ".join(lines)) self.cursor.execute(stmt)
def _do_deletes(self): for (table, columns), rows in sorted(iteritems(self.deletes)): rows = list(sorted(rows)) if len(columns) == 1: value_str = ','.join(v for (v, ) in rows) stmt = "DELETE FROM %s WHERE %s IN (%s)" % (table, columns[0], value_str) else: lines = [] for row in rows: line = [] for i, column in enumerate(columns): line.append("%s = %s" % (column, row[i])) lines.append(" AND ".join(line)) stmt = "DELETE FROM %s WHERE %s" % (table, " OR ".join(lines)) self.cursor.execute(stmt)
def _do_inserts(self): items = sorted(iteritems(self.inserts)) for (command, header, row_schema, suffix), rows in items: # Batched inserts parts = [] params = [] s = "(%s)" % row_schema for row in itervalues(rows): parts.append(s) params.extend(row) stmt = "%s INTO %s VALUES\n%s\n%s" % ( command, header, ',\n'.join(parts), suffix) # e.g., # INSERT INTO table(c1, c2) # VALUES (%s, %s), (%s, %s), (%s, %s) # <suffix> self.cursor.execute(stmt, tuple(params))
def _do_deletes(self): for (table, columns), rows in sorted(iteritems(self.deletes)): # XXX: Stop doing string conversion manually. Let the # cursor do it. It may have a non-text protocol for integer # objects; it may also have a different representation in text. if len(columns) == 1: value_str = ','.join(str(v) for (v,) in rows) stmt = "DELETE FROM %s WHERE %s IN (%s)" % ( table, columns[0], value_str) else: lines = [] for row in rows: line = [] for i, column in enumerate(columns): line.append("%s = %s" % (column, row[i])) lines.append(" AND ".join(line)) stmt = "DELETE FROM %s WHERE %s" % ( table, " OR ".join(lines)) self.cursor.execute(stmt)
def run_script_stmt(self, cursor, generic_stmt, generic_params=()): """Execute a statement from a script with the given parameters. params should be either an empty tuple (no parameters) or a map. """ generic_stmt = generic_stmt.format(**self.format_vars) # We can't quote "transaction", but we have to for sqlite. generic_stmt = generic_stmt.replace(' "transaction"', ' transaction') if generic_params and isinstance(generic_params, tuple): generic_stmt = _format_to_named(generic_stmt) # Unnamed params become numbered. if generic_params and isinstance(generic_params, dict): # Oracle raises ORA-01036 if the parameter map contains extra keys, # so filter out any unused parameters. tracker = TrackingMap(self.script_vars) stmt = generic_stmt % tracker used = tracker.used params = {} for k, v in iteritems(generic_params): if k in used: params[k] = v else: stmt = generic_stmt % self.script_vars params = () if generic_params and isinstance(generic_params, tuple): params = generic_params __traceback_info__ = stmt try: cursor.execute(stmt, params) except: log.warning("script statement failed: %r; parameters: %r", stmt, params) raise
def reachable(self): """Iterate over all the reachable OIDs.""" for oid_hi, oids_lo in iteritems(self._reachable): for oid_lo in oids_lo: # Decode the OID. yield oid_hi | oid_lo
def _set_multi(self, keys_and_values): formatted = { '%s:state:%d:%d' % (self.prefix, tid, oid): (p64(actual_tid) + (state or b'')) for (oid, tid), (state, actual_tid) in iteritems(keys_and_values) } self.client.set_multi(formatted)
def compare_exact(self, storage1, storage2): """Confirm that storage1 and storage2 contain equivalent data""" eq = self.assertEqual missing = object() iter1 = storage1.iterator() iter2 = storage2.iterator() for txn1, txn2 in zip(iter1, iter2): eq(txn1.tid, txn2.tid) eq(txn1.status, txn2.status) eq(txn1.user, txn2.user) eq(txn1.description, txn2.description) # b/w compat on the 'extension' attribute e1 = getattr(txn1, 'extension', missing) if e1 is missing: # old attribute name e1 = txn1._extension e2 = getattr(txn2, 'extension', missing) if e2 is missing: # old attribute name e2 = txn2._extension eq(e1, e2) # compare the objects in the transaction, but disregard # the order of the objects and any duplicated records # since those are not important. recs1 = dict([(r.oid, r) for r in txn1]) recs2 = dict([(r.oid, r) for r in txn2]) eq(len(recs1), len(recs2)) recs1 = sorted(iteritems(recs1)) recs2 = sorted(iteritems(recs2)) recs2.sort() for (_oid1, rec1), (_oid2, rec2) in zip(recs1, recs2): eq(rec1.oid, rec2.oid) eq(rec1.tid, rec2.tid) eq(rec1.data, rec2.data) if is_blob_record(rec1.data): try: fn1 = storage1.loadBlob(rec1.oid, rec1.tid) except ZODB.POSException.POSKeyError: self.assertRaises(ZODB.POSException.POSKeyError, storage2.loadBlob, rec1.oid, rec1.tid) else: fn2 = storage2.loadBlob(rec1.oid, rec1.tid) self.assertNotEqual(fn1, fn2) with open(fn1, 'rb') as f1, open(fn2, 'rb') as f2: eq(f1.read(), f2.read()) # Make sure ther are no more records left in txn1 and txn2, meaning # they were the same length try: next(iter1) except (IndexError, StopIteration): pass else: self.fail("storage1 has more records") try: next(iter2) except (IndexError, StopIteration): pass else: self.fail("storage2 has more records") iter1.close() iter2.close()
def _do_deletes(self): return self._do_batch('DELETE', sorted(iteritems(self.deletes)))
def compare_exact(self, storage1, storage2): """Confirm that storage1 and storage2 contain equivalent data""" eq = self.assertEqual missing = object() iter1 = storage1.iterator() iter2 = storage2.iterator() for txn1, txn2 in zip(iter1, iter2): eq(txn1.tid, txn2.tid) eq(txn1.status, txn2.status) eq(txn1.user, txn2.user) eq(txn1.description, txn2.description) # b/w compat on the 'extension' attribute e1 = getattr(txn1, 'extension', missing) if e1 is missing: # old attribute name e1 = txn1._extension e2 = getattr(txn2, 'extension', missing) if e2 is missing: # old attribute name e2 = txn2._extension eq(e1, e2) # compare the objects in the transaction, but disregard # the order of the objects and any duplicated records # since those are not important. recs1 = {r.oid: r for r in txn1} recs2 = {r.oid: r for r in txn1} eq(len(recs1), len(recs2)) recs1 = sorted(iteritems(recs1)) recs2 = sorted(iteritems(recs2)) recs2.sort() for (_oid1, rec1), (_oid2, rec2) in zip(recs1, recs2): eq(rec1.oid, rec2.oid) eq(rec1.tid, rec2.tid) eq(rec1.data, rec2.data) if is_blob_record(rec1.data): try: fn1 = storage1.loadBlob(rec1.oid, rec1.tid) except ZODB.POSException.POSKeyError: self.assertRaises( ZODB.POSException.POSKeyError, storage2.loadBlob, rec1.oid, rec1.tid) else: fn2 = storage2.loadBlob(rec1.oid, rec1.tid) self.assertNotEqual(fn1, fn2) with open(fn1, 'rb') as f1, open(fn2, 'rb') as f2: eq(f1.read(), f2.read()) # Make sure ther are no more records left in txn1 and txn2, meaning # they were the same length try: next(iter1) except (IndexError, StopIteration): pass else: self.fail("storage1 has more records") try: next(iter2) except (IndexError, StopIteration): pass else: self.fail("storage2 has more records") iter1.close() iter2.close()