def copyTransactionsFrom(self, other):
# Just the interface, not the attribute, in case we have a
# partial proxy.
other_has_record_iternext = IRecordIter.providedBy(other) # pylint:disable=no-value-for-parameter
copier_factory = _HistoryFreeCopier
if self.tpc.keep_history or not other_has_record_iternext:
copier_factory = _HistoryPreservingCopier
logger.info(
"Copying transactions to %s "
"from %s (supports IStorageCurrentRecordIteration? %s) "
"using %s",
self.tpc,
other,
other_has_record_iternext,
copier_factory,
)
copier = copier_factory(other, self.blobhelper, self.tpc, self.restore)
try:
logger.info("Counting the %s to copy.", copier.units)
num_txns = len(copier)
logger.info("Copying %d %s%s", num_txns, copier.units, copier.initial_log_suffix)
progress = copier.ProgressLogger(num_txns, copier)
copier.copy(progress)
finally:
copier.close()
now = perf_counter()
logger.info(
"Copied transactions: %s",
progress.display_at(now))
def verify_zodb(obj, debug=False):
storage = obj._p_jar._db._storage
if not IStorageCurrentRecordIteration.providedBy(storage):
raise TypeError(
'ZODB storage {} does not implement record_iternext'.format(
storage))
logger.info('Scanning ZODB...')
next_ = None
count = 0
errors = 0
while True:
count += 1
oid, tid, data, next_ = storage.record_iternext(next_)
logger.debug('Verifying {}'.format(oid))
success = verify_record(oid, data, debug)
if not success:
errors += 1
if next_ is None:
break
logger.info('Done! Scanned {} records. '
'Found {} records that could not be loaded.'.format(
count, errors))
def verify_zodb(obj, debug=False):
storage = obj._p_jar._db._storage
if not IStorageCurrentRecordIteration.providedBy(storage):
raise TypeError(
'ZODB storage {} does not implement record_iternext'.format(
storage))
logger.info('Scanning ZODB...')
next_ = None
count = 0
errors = 0
while True:
count += 1
oid, tid, data, next_ = storage.record_iternext(next_)
logger.debug('Verifying {}'.format(oid))
success = verify_record(oid, data, debug)
if not success:
errors += 1
if next_ is None:
break
logger.info(
'Done! Scanned {} records. '
'Found {} records that could not be loaded.'.format(
count, errors)
)
def verify_zodb(storage, debug=False):
if not IStorageCurrentRecordIteration.providedBy(storage):
raise TypeError(
"ZODB storage {} does not implement record_iternext".format(
storage))
logger.info("Scanning ZODB...")
next_ = None
count = 0
errors = 0
issues = []
while True:
count += 1
oid, tid, data, next_ = storage.record_iternext(next_)
logger.debug("Verifying {}".format(oid))
success, msg = verify_record(oid, data, debug)
if not success:
errors += 1
issues.append(msg)
if next_ is None:
break
issues = Counter(sorted(issues))
msg = ""
for value, amount in issues.items():
msg += "{}: {}\n".format(value, amount)
logger.info("Done! Scanned {} records. \n"
"Found {} records that could not be loaded. \n"
"Exceptions and how often they happened: \n"
"{}".format(count, errors, msg))
def records(self):
next = ZODB.utils.repr_to_oid(self.start_at)
storage = self.storage
# If we've got a BlobStorage wrapper, let's
# actually iterate through the storage it wraps.
if isinstance(storage, BlobStorage):
storage = storage._BlobStorage__storage
if isinstance(storage, FileStorage):
# Custom iterator for FileStorage. This is used to be able
# to recover form a POSKey error.
index = storage._index
while True:
oid = index.minKey(next)
try:
data, tid = storage.load(oid, "")
except ZODB.POSException.POSKeyError as e:
logger.error(
'Warning: Jumping record {}, '
'referencing missing key in database: {}'.format(
ZODB.utils.oid_repr(oid), str(e)))
else:
yield oid, tid, io.BytesIO(data)
oid_as_long, = unpack(">Q", oid)
next = pack(">Q", oid_as_long + 1)
try:
next = index.minKey(next)
except ValueError:
# No more records
break
elif IStorageCurrentRecordIteration.providedBy(storage):
# Second best way to iterate through the lastest records.
while True:
oid, tid, data, next = storage.record_iternext(next)
yield oid, tid, io.BytesIO(data)
if next is None:
break
elif (IStorageIteration.providedBy(storage)
and (not IStorageUndoable.providedBy(storage)
or not storage.supportsUndo())):
# If we can't iterate only through the recent records,
# iterate on all. Of course doing a pack before help :).
for transaction_ in storage.iterator():
for rec in transaction_:
yield rec.oid, rec.tid, io.BytesIO(rec.data)
else:
raise SystemExit(
"Don't know how to iterate through this storage type")
def verify_oid(storage, oid, debug=False, app=None):
if not IStorageCurrentRecordIteration.providedBy(storage):
raise TypeError(
"ZODB storage {} does not implement record_iternext".format(storage)
)
try:
# by default expect a 8-byte string (e.g. '0x22d17d')
# transform to a 64-bit long integer (e.g. b'\x00\x00\x00\x00\x00"\xd1}')
as_int = int(oid, 0)
oid = p64(as_int)
except ValueError:
# probably already a 64-bit long integer
pass
if app:
# use exitsing zope instance.
# only available when used as ./bin/instance zodbverify -o XXX
connection = app._p_jar
else:
# connect to database to be able to load the object
db = ZODB.DB(storage)
connection = db.open()
try:
obj = connection.get(oid)
try:
logger.info("\nObject as dict:\n{}".format(vars(obj)))
except TypeError:
pass
if debug:
hint = "\nThe object is 'obj'"
if app:
hint += "\nThe Zope instance is 'app'"
logger.info(hint)
pdb.set_trace()
except Exception as e:
logger.info("Could not load object")
logger.info(traceback.format_exc())
if debug:
pdb.set_trace()
pickle, state = storage.load(oid)
success, msg = verify_record(oid, pickle, debug)
if not success:
logger.info('{}: {}'.format(msg, oid_repr(oid)))
def verify_zodb(storage, debug=False):
if not IStorageCurrentRecordIteration.providedBy(storage):
raise TypeError(
"ZODB storage {} does not implement record_iternext".format(
storage))
logger.info("Scanning ZODB...")
next_ = None
count = 0
errors = 0
issues = defaultdict(list)
oids = []
while True:
count += 1
oid, tid, data, next_ = storage.record_iternext(next_)
logger.debug("Verifying {}".format(oid))
success, msg = verify_record(oid, data, debug)
if not success:
errors += 1
issues[msg].append(oid_repr(oid))
# issues.append(msg)
oids.append(oid)
if next_ is None:
break
msg = ""
order = sorted(issues, key=lambda k: len(issues[k]), reverse=True)
for key in order:
oids = issues[key]
msg += "{}: {}\n{}\n\n".format(key, len(oids), ' '.join(oids))
logger.info(
"Done! Scanned {} records. \n"
"Found {} records that could not be loaded. \n"
"Exceptions, how often they happened and which oids are affected: \n\n"
"{}".format(count, errors, msg))
def main(argv=None):
# pylint:disable=too-many-branches,too-many-statements
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run", dest="dry_run", action="store_true",
default=False,
help="Attempt to open both storages, then explain what would be done.")
parser.add_argument(
"--clear", dest="clear", action="store_true",
default=False,
help="Clear the contents of the destination storage before copying."
" Only works if the destination is a RelStorage."
" WARNING: use this only if you are certain the destination has no useful data.")
parser.add_argument(
"--incremental", dest="incremental", action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. ")
log_group = parser.add_mutually_exclusive_group()
log_group.add_argument(
'--debug', dest="log_level", action='store_const',
const=logging.DEBUG,
default=logging.INFO,
help="Set the logging level to DEBUG instead of the default of INFO."
)
log_group.add_argument(
'--trace', dest="log_level", action='store_const',
const=loglevels.TRACE,
default=logging.INFO,
help="Set the logging level to TRACE instead of the default of INFO."
)
parser.add_argument("config_file", type=argparse.FileType('r'))
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=options.log_level,
format="%(asctime)s [%(name)s] %(levelname)-6s %(message)s"
)
source, destination = open_storages(options)
def cleanup_and_exit(exit_msg=None):
source.close()
destination.close()
if exit_msg:
sys.exit(exit_msg)
log.info("Storages opened successfully.")
if options.dry_run and options.clear:
cleanup_and_exit("Cannot clear a storage during a dry-run")
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else: # pragma: no cover
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
cleanup_and_exit(msg)
log.info("Done clearing old data.")
if options.incremental:
assert hasattr(destination, 'lastTransaction'), (
"Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
if storage_has_data(destination):
# This requires that the storage produce a valid (not z64) value before
# anything is loaded with it.
last_tid = destination.lastTransaction()
if isinstance(last_tid, bytes):
# This *should* be a byte string.
last_tid = u64(last_tid)
next_tid = p64(last_tid + 1)
# Compensate for the RelStorage bug(?) and get a reusable iterator
# that starts where we want it to. There's no harm in wrapping it for
# other sources like FileStorage too.
#
# Note that this disables access to ``record_iternext``,
# defeating some of the optimizations our own copyTransactionsFrom
# would like to do.
# XXX: Figure out an incremental way to do this.
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
else:
log.warning("Destination empty; Forcing "
"incremental conversion from the beginning. "
"If the destination is a history-free RelStorage, this "
"will take much longer.")
# Use the DefaultStartStorageIteration for its side-effect of disabling
# record_iternext
next_tid = None
source = _DefaultStartStorageIteration(source, next_tid)
assert not IStorageCurrentRecordIteration.providedBy(source) # pylint:disable=no-value-for-parameter
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s',
readable_tid_repr(txn.tid), txn.user, txn.description)
count += 1
log.info("Would copy %d transactions.", count)
cleanup_and_exit()
else:
if storage_has_data(destination) and not options.incremental:
msg = "Error: the destination storage has data. Try --clear."
cleanup_and_exit(msg)
try:
destination.copyTransactionsFrom(source)
finally:
cleanup_and_exit()
except ZODB.POSException.POSKeyError, e:
logger.error(
u'Warning: Jumping record %s, '
u'referencing missing key in database: %s' %
(ZODB.utils.oid_repr(oid), str(e)))
else:
yield oid, tid, cStringIO.StringIO(data)
oid_as_long, = unpack(">Q", oid)
next = pack(">Q", oid_as_long + 1)
try:
next = index.minKey(next)
except ValueError:
# No more records
break
elif IStorageCurrentRecordIteration.providedBy(self.storage):
# Second best way to iterate through the lastest records.
while True:
oid, tid, data, next = self.storage.record_iternext(next)
yield oid, tid, cStringIO.StringIO(data)
if next is None:
break
elif (IStorageIteration.providedBy(self.storage) and
not self.storage.supportsUndo()):
# If we can't iterate only through the recent records,
# iterate on all. Of course doing a pack before help :).
for transaction in self.storage.iterator():
for rec in transaction:
yield rec.oid, rec.tid, cStringIO.StringIO(rec.data)
else:
raise SystemExit(