def __str__(self):
extras = []
if self.oid:
extras.append("oid %s" % oid_repr(self.oid))
if self.class_name:
extras.append("class %s" % self.class_name)
if self.serials:
current, old = self.serials
extras.append("serial this txn started with %s" % readable_tid_repr(old))
extras.append("serial currently committed %s" % readable_tid_repr(current))
if extras:
return "%s (%s)" % (self.message, ", ".join(extras))
else:
return self.message
def __str__(self):
extras = []
if self.oid:
extras.append("oid %s" % oid_repr(self.oid))
if self.class_name:
extras.append("class %s" % self.class_name)
if self.serials:
current, old = self.serials
extras.append("serial this txn started with %s" %
readable_tid_repr(old))
extras.append("serial currently committed %s" %
readable_tid_repr(current))
if extras:
return "%s (%s)" % (self.message, ", ".join(extras))
else:
return self.message
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
help="Attempt to open the storages, then explain what would be done")
parser.add_option(
"--clear",
dest="clear",
action="store_true",
help="Clear the contents of the destination storage before copying")
parser.add_option(
"--incremental",
dest="incremental",
action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. "
"Currently only supports RelStorage destinations.")
parser.set_defaults(dry_run=False, clear=False)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
source = config.source.open()
destination = config.destination.open()
log.info("Storages opened successfully.")
if options.incremental:
if not hasattr(destination, 'lastTransaction'):
msg = ("Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
sys.exit(msg)
if not storage_has_data(destination):
log.warning(
"Destination empty, start conversion from the beginning.")
else:
# This requires that the storage produce a valid (not z64) value before
# anything is loaded with it.
last_tid = destination.lastTransaction()
if isinstance(last_tid, bytes):
# This *should* be a byte string.
last_tid = u64(last_tid)
next_tid = p64(last_tid + 1)
# Compensate for the RelStorage bug(?) and get a reusable iterator
# that starts where we want it to. There's no harm in wrapping it for
# other sources like FileStorage too.
source = _DefaultStartStorageIteration(source, next_tid)
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s' %
(TimeStamp(txn.tid), txn.user, txn.description))
count += 1
log.info("Would copy %d transactions.", count)
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
sys.exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination) and not options.incremental:
msg = "Error: the destination storage has data. Try --clear."
sys.exit(msg)
destination.copyTransactionsFrom(source)
source.close()
destination.close()
def main(argv=None):
# pylint:disable=too-many-branches,too-many-statements
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run", dest="dry_run", action="store_true",
default=False,
help="Attempt to open both storages, then explain what would be done.")
parser.add_argument(
"--clear", dest="clear", action="store_true",
default=False,
help="Clear the contents of the destination storage before copying."
" Only works if the destination is a RelStorage."
" WARNING: use this only if you are certain the destination has no useful data.")
parser.add_argument(
"--incremental", dest="incremental", action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. ")
log_group = parser.add_mutually_exclusive_group()
log_group.add_argument(
'--debug', dest="log_level", action='store_const',
const=logging.DEBUG,
default=logging.INFO,
help="Set the logging level to DEBUG instead of the default of INFO."
)
log_group.add_argument(
'--trace', dest="log_level", action='store_const',
const=loglevels.TRACE,
default=logging.INFO,
help="Set the logging level to TRACE instead of the default of INFO."
)
parser.add_argument("config_file", type=argparse.FileType('r'))
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=options.log_level,
format="%(asctime)s [%(name)s] %(levelname)-6s %(message)s"
)
source, destination = open_storages(options)
def cleanup_and_exit(exit_msg=None):
source.close()
destination.close()
if exit_msg:
sys.exit(exit_msg)
log.info("Storages opened successfully.")
if options.dry_run and options.clear:
cleanup_and_exit("Cannot clear a storage during a dry-run")
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else: # pragma: no cover
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
cleanup_and_exit(msg)
log.info("Done clearing old data.")
if options.incremental:
assert hasattr(destination, 'lastTransaction'), (
"Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
if storage_has_data(destination):
# This requires that the storage produce a valid (not z64) value before
# anything is loaded with it.
last_tid = destination.lastTransaction()
if isinstance(last_tid, bytes):
# This *should* be a byte string.
last_tid = u64(last_tid)
next_tid = p64(last_tid + 1)
# Compensate for the RelStorage bug(?) and get a reusable iterator
# that starts where we want it to. There's no harm in wrapping it for
# other sources like FileStorage too.
#
# Note that this disables access to ``record_iternext``,
# defeating some of the optimizations our own copyTransactionsFrom
# would like to do.
# XXX: Figure out an incremental way to do this.
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
else:
log.warning("Destination empty; Forcing "
"incremental conversion from the beginning. "
"If the destination is a history-free RelStorage, this "
"will take much longer.")
# Use the DefaultStartStorageIteration for its side-effect of disabling
# record_iternext
next_tid = None
source = _DefaultStartStorageIteration(source, next_tid)
assert not IStorageCurrentRecordIteration.providedBy(source) # pylint:disable=no-value-for-parameter
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s',
readable_tid_repr(txn.tid), txn.user, txn.description)
count += 1
log.info("Would copy %d transactions.", count)
cleanup_and_exit()
else:
if storage_has_data(destination) and not options.incremental:
msg = "Error: the destination storage has data. Try --clear."
cleanup_and_exit(msg)
try:
destination.copyTransactionsFrom(source)
finally:
cleanup_and_exit()
def main(argv=None):
# pylint:disable=too-many-branches,too-many-statements
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run",
dest="dry_run",
action="store_true",
default=False,
help="Attempt to open both storages, then explain what would be done.")
parser.add_argument(
"--clear",
dest="clear",
action="store_true",
default=False,
help="Clear the contents of the destination storage before copying."
" Only works if the destination is a RelStorage."
" WARNING: use this only if you are certain the destination has no useful data."
)
parser.add_argument(
"--incremental",
dest="incremental",
action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. ")
parser.add_argument("config_file", type=argparse.FileType('r'))
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
source, destination = open_storages(options)
def cleanup_and_exit(exit_msg=None):
source.close()
destination.close()
if exit_msg:
sys.exit(msg)
log.info("Storages opened successfully.")
if options.incremental:
assert hasattr(destination, 'lastTransaction'), (
"Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
if not storage_has_data(destination):
log.warning(
"Destination empty, start conversion from the beginning.")
else:
# This requires that the storage produce a valid (not z64) value before
# anything is loaded with it.
last_tid = destination.lastTransaction()
if isinstance(last_tid, bytes):
# This *should* be a byte string.
last_tid = u64(last_tid)
next_tid = p64(last_tid + 1)
# Compensate for the RelStorage bug(?) and get a reusable iterator
# that starts where we want it to. There's no harm in wrapping it for
# other sources like FileStorage too.
source = _DefaultStartStorageIteration(source, next_tid)
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s', TimeStamp(txn.tid), txn.user,
txn.description)
count += 1
log.info("Would copy %d transactions.", count)
cleanup_and_exit()
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
cleanup_and_exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination) and not options.incremental:
msg = "Error: the destination storage has data. Try --clear."
cleanup_and_exit(msg)
destination.copyTransactionsFrom(source)
cleanup_and_exit()
def __transaction_display(self, trans, copy_result):
num_txn_records, txn_byte_size, num_txn_blobs = copy_result
return 'transaction %s <%4d records, %3d blobs, %9s>' % (
readable_tid_repr(trans.tid), num_txn_records, num_txn_blobs,
byte_display(txn_byte_size))
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run", dest="dry_run", action="store_true",
default=False,
help="Attempt to open both storages, then explain what would be done.")
parser.add_argument(
"--clear", dest="clear", action="store_true",
default=False,
help="Clear the contents of the destination storage before copying. Only works if the destination is a RelStorage."
" WARNING: use this only if you are certain the destination has no useful data.")
parser.add_argument(
"--incremental", dest="incremental", action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. ")
parser.add_argument("config_file")
options = parser.parse_args(argv[1:])
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, _ = ZConfig.loadConfig(schema, options.config_file)
source = config.source.open()
destination = config.destination.open()
def cleanup_and_exit(exit_msg=None):
source.close()
destination.close()
if exit_msg:
sys.exit(msg)
log.info("Storages opened successfully.")
if options.incremental:
assert hasattr(destination, 'lastTransaction'), ("Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
if not storage_has_data(destination):
log.warning("Destination empty, start conversion from the beginning.")
else:
# This requires that the storage produce a valid (not z64) value before
# anything is loaded with it.
last_tid = destination.lastTransaction()
if isinstance(last_tid, bytes):
# This *should* be a byte string.
last_tid = u64(last_tid)
next_tid = p64(last_tid+1)
# Compensate for the RelStorage bug(?) and get a reusable iterator
# that starts where we want it to. There's no harm in wrapping it for
# other sources like FileStorage too.
source = _DefaultStartStorageIteration(source, next_tid)
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s',
TimeStamp(txn.tid), txn.user, txn.description)
count += 1
log.info("Would copy %d transactions.", count)
cleanup_and_exit()
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
cleanup_and_exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination) and not options.incremental:
msg = "Error: the destination storage has data. Try --clear."
cleanup_and_exit(msg)
destination.copyTransactionsFrom(source)
cleanup_and_exit()
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"--dry-run", dest="dry_run", action="store_true",
help="Attempt to open the storages, then explain what would be done")
parser.add_option(
"--clear", dest="clear", action="store_true",
help="Clear the contents of the destination storage before copying")
parser.add_option(
"--incremental", dest="incremental", action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. "
"Currently only supports RelStorage destinations.")
parser.set_defaults(dry_run=False, clear=False)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
source = config.source.open()
destination = config.destination.open()
log.info("Storages opened successfully.")
if options.incremental:
if not hasattr(destination, '_adapter'):
msg = ("Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
sys.exit(msg)
if not storage_has_data(destination):
log.warning("Destination empty, start conversion from the beginning.")
else:
last_tid = destination._adapter.txncontrol.get_tid(
destination._load_cursor)
next_tid = p64(last_tid+1)
source = source.iterator(start=next_tid)
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s' % (
TimeStamp(txn.tid), txn.user, txn.description))
count += 1
log.info("Would copy %d transactions.", count)
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
sys.exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination):
msg = "Error: the destination storage has data. Try --clear."
sys.exit(msg)
destination.copyTransactionsFrom(source)
source.close()
destination.close()
