def __str__(self): extras = [] if self.oid: extras.append("oid %s" % oid_repr(self.oid)) if self.class_name: extras.append("class %s" % self.class_name) if self.serials: current, old = self.serials extras.append("serial this txn started with %s" % readable_tid_repr(old)) extras.append("serial currently committed %s" % readable_tid_repr(current)) if extras: return "%s (%s)" % (self.message, ", ".join(extras)) else: return self.message
def main(argv=sys.argv): parser = optparse.OptionParser(description=__doc__, usage="%prog [options] config_file") parser.add_option( "--dry-run", dest="dry_run", action="store_true", help="Attempt to open the storages, then explain what would be done") parser.add_option( "--clear", dest="clear", action="store_true", help="Clear the contents of the destination storage before copying") parser.add_option( "--incremental", dest="incremental", action="store_true", help="Assume the destination contains a partial copy of the source " "and resume copying from the last transaction. WARNING: no " "effort is made to verify that the destination holds the same " "transaction data before this point! Use at your own risk. " "Currently only supports RelStorage destinations.") parser.set_defaults(dry_run=False, clear=False) options, args = parser.parse_args(argv[1:]) if len(args) != 1: parser.error("The name of one configuration file is required.") logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, handler = ZConfig.loadConfig(schema, args[0]) source = config.source.open() destination = config.destination.open() log.info("Storages opened successfully.") if options.incremental: if not hasattr(destination, 'lastTransaction'): msg = ("Error: no API is known for determining the last committed " "transaction of the destination storage. Aborting " "conversion.") sys.exit(msg) if not storage_has_data(destination): log.warning( "Destination empty, start conversion from the beginning.") else: # This requires that the storage produce a valid (not z64) value before # anything is loaded with it. last_tid = destination.lastTransaction() if isinstance(last_tid, bytes): # This *should* be a byte string. last_tid = u64(last_tid) next_tid = p64(last_tid + 1) # Compensate for the RelStorage bug(?) and get a reusable iterator # that starts where we want it to. There's no harm in wrapping it for # other sources like FileStorage too. source = _DefaultStartStorageIteration(source, next_tid) log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid)) if options.dry_run: log.info("Dry run mode: not changing the destination.") if storage_has_data(destination): log.warning("The destination storage has data.") count = 0 for txn in source.iterator(): log.info('%s user=%s description=%s' % (TimeStamp(txn.tid), txn.user, txn.description)) count += 1 log.info("Would copy %d transactions.", count) else: if options.clear: log.info("Clearing old data...") if hasattr(destination, 'zap_all'): destination.zap_all() else: msg = ("Error: no API is known for clearing this type " "of storage. Use another method.") sys.exit(msg) log.info("Done clearing old data.") if storage_has_data(destination) and not options.incremental: msg = "Error: the destination storage has data. Try --clear." sys.exit(msg) destination.copyTransactionsFrom(source) source.close() destination.close()
def main(argv=None): # pylint:disable=too-many-branches,too-many-statements if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--dry-run", dest="dry_run", action="store_true", default=False, help="Attempt to open both storages, then explain what would be done.") parser.add_argument( "--clear", dest="clear", action="store_true", default=False, help="Clear the contents of the destination storage before copying." " Only works if the destination is a RelStorage." " WARNING: use this only if you are certain the destination has no useful data.") parser.add_argument( "--incremental", dest="incremental", action="store_true", help="Assume the destination contains a partial copy of the source " "and resume copying from the last transaction. WARNING: no " "effort is made to verify that the destination holds the same " "transaction data before this point! Use at your own risk. ") log_group = parser.add_mutually_exclusive_group() log_group.add_argument( '--debug', dest="log_level", action='store_const', const=logging.DEBUG, default=logging.INFO, help="Set the logging level to DEBUG instead of the default of INFO." ) log_group.add_argument( '--trace', dest="log_level", action='store_const', const=loglevels.TRACE, default=logging.INFO, help="Set the logging level to TRACE instead of the default of INFO." ) parser.add_argument("config_file", type=argparse.FileType('r')) options = parser.parse_args(argv[1:]) logging.basicConfig( level=options.log_level, format="%(asctime)s [%(name)s] %(levelname)-6s %(message)s" ) source, destination = open_storages(options) def cleanup_and_exit(exit_msg=None): source.close() destination.close() if exit_msg: sys.exit(exit_msg) log.info("Storages opened successfully.") if options.dry_run and options.clear: cleanup_and_exit("Cannot clear a storage during a dry-run") if options.clear: log.info("Clearing old data...") if hasattr(destination, 'zap_all'): destination.zap_all() else: # pragma: no cover msg = ("Error: no API is known for clearing this type " "of storage. Use another method.") cleanup_and_exit(msg) log.info("Done clearing old data.") if options.incremental: assert hasattr(destination, 'lastTransaction'), ( "Error: no API is known for determining the last committed " "transaction of the destination storage. Aborting " "conversion.") if storage_has_data(destination): # This requires that the storage produce a valid (not z64) value before # anything is loaded with it. last_tid = destination.lastTransaction() if isinstance(last_tid, bytes): # This *should* be a byte string. last_tid = u64(last_tid) next_tid = p64(last_tid + 1) # Compensate for the RelStorage bug(?) and get a reusable iterator # that starts where we want it to. There's no harm in wrapping it for # other sources like FileStorage too. # # Note that this disables access to ``record_iternext``, # defeating some of the optimizations our own copyTransactionsFrom # would like to do. # XXX: Figure out an incremental way to do this. log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid)) else: log.warning("Destination empty; Forcing " "incremental conversion from the beginning. " "If the destination is a history-free RelStorage, this " "will take much longer.") # Use the DefaultStartStorageIteration for its side-effect of disabling # record_iternext next_tid = None source = _DefaultStartStorageIteration(source, next_tid) assert not IStorageCurrentRecordIteration.providedBy(source) # pylint:disable=no-value-for-parameter if options.dry_run: log.info("Dry run mode: not changing the destination.") if storage_has_data(destination): log.warning("The destination storage has data.") count = 0 for txn in source.iterator(): log.info('%s user=%s description=%s', readable_tid_repr(txn.tid), txn.user, txn.description) count += 1 log.info("Would copy %d transactions.", count) cleanup_and_exit() else: if storage_has_data(destination) and not options.incremental: msg = "Error: the destination storage has data. Try --clear." cleanup_and_exit(msg) try: destination.copyTransactionsFrom(source) finally: cleanup_and_exit()
def main(argv=None): # pylint:disable=too-many-branches,too-many-statements if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--dry-run", dest="dry_run", action="store_true", default=False, help="Attempt to open both storages, then explain what would be done.") parser.add_argument( "--clear", dest="clear", action="store_true", default=False, help="Clear the contents of the destination storage before copying." " Only works if the destination is a RelStorage." " WARNING: use this only if you are certain the destination has no useful data." ) parser.add_argument( "--incremental", dest="incremental", action="store_true", help="Assume the destination contains a partial copy of the source " "and resume copying from the last transaction. WARNING: no " "effort is made to verify that the destination holds the same " "transaction data before this point! Use at your own risk. ") parser.add_argument("config_file", type=argparse.FileType('r')) options = parser.parse_args(argv[1:]) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") source, destination = open_storages(options) def cleanup_and_exit(exit_msg=None): source.close() destination.close() if exit_msg: sys.exit(msg) log.info("Storages opened successfully.") if options.incremental: assert hasattr(destination, 'lastTransaction'), ( "Error: no API is known for determining the last committed " "transaction of the destination storage. Aborting " "conversion.") if not storage_has_data(destination): log.warning( "Destination empty, start conversion from the beginning.") else: # This requires that the storage produce a valid (not z64) value before # anything is loaded with it. last_tid = destination.lastTransaction() if isinstance(last_tid, bytes): # This *should* be a byte string. last_tid = u64(last_tid) next_tid = p64(last_tid + 1) # Compensate for the RelStorage bug(?) and get a reusable iterator # that starts where we want it to. There's no harm in wrapping it for # other sources like FileStorage too. source = _DefaultStartStorageIteration(source, next_tid) log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid)) if options.dry_run: log.info("Dry run mode: not changing the destination.") if storage_has_data(destination): log.warning("The destination storage has data.") count = 0 for txn in source.iterator(): log.info('%s user=%s description=%s', TimeStamp(txn.tid), txn.user, txn.description) count += 1 log.info("Would copy %d transactions.", count) cleanup_and_exit() else: if options.clear: log.info("Clearing old data...") if hasattr(destination, 'zap_all'): destination.zap_all() else: msg = ("Error: no API is known for clearing this type " "of storage. Use another method.") cleanup_and_exit(msg) log.info("Done clearing old data.") if storage_has_data(destination) and not options.incremental: msg = "Error: the destination storage has data. Try --clear." cleanup_and_exit(msg) destination.copyTransactionsFrom(source) cleanup_and_exit()
def __transaction_display(self, trans, copy_result): num_txn_records, txn_byte_size, num_txn_blobs = copy_result return 'transaction %s <%4d records, %3d blobs, %9s>' % ( readable_tid_repr(trans.tid), num_txn_records, num_txn_blobs, byte_display(txn_byte_size))
def main(argv=None): if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--dry-run", dest="dry_run", action="store_true", default=False, help="Attempt to open both storages, then explain what would be done.") parser.add_argument( "--clear", dest="clear", action="store_true", default=False, help="Clear the contents of the destination storage before copying. Only works if the destination is a RelStorage." " WARNING: use this only if you are certain the destination has no useful data.") parser.add_argument( "--incremental", dest="incremental", action="store_true", help="Assume the destination contains a partial copy of the source " "and resume copying from the last transaction. WARNING: no " "effort is made to verify that the destination holds the same " "transaction data before this point! Use at your own risk. ") parser.add_argument("config_file") options = parser.parse_args(argv[1:]) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, _ = ZConfig.loadConfig(schema, options.config_file) source = config.source.open() destination = config.destination.open() def cleanup_and_exit(exit_msg=None): source.close() destination.close() if exit_msg: sys.exit(msg) log.info("Storages opened successfully.") if options.incremental: assert hasattr(destination, 'lastTransaction'), ("Error: no API is known for determining the last committed " "transaction of the destination storage. Aborting " "conversion.") if not storage_has_data(destination): log.warning("Destination empty, start conversion from the beginning.") else: # This requires that the storage produce a valid (not z64) value before # anything is loaded with it. last_tid = destination.lastTransaction() if isinstance(last_tid, bytes): # This *should* be a byte string. last_tid = u64(last_tid) next_tid = p64(last_tid+1) # Compensate for the RelStorage bug(?) and get a reusable iterator # that starts where we want it to. There's no harm in wrapping it for # other sources like FileStorage too. source = _DefaultStartStorageIteration(source, next_tid) log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid)) if options.dry_run: log.info("Dry run mode: not changing the destination.") if storage_has_data(destination): log.warning("The destination storage has data.") count = 0 for txn in source.iterator(): log.info('%s user=%s description=%s', TimeStamp(txn.tid), txn.user, txn.description) count += 1 log.info("Would copy %d transactions.", count) cleanup_and_exit() else: if options.clear: log.info("Clearing old data...") if hasattr(destination, 'zap_all'): destination.zap_all() else: msg = ("Error: no API is known for clearing this type " "of storage. Use another method.") cleanup_and_exit(msg) log.info("Done clearing old data.") if storage_has_data(destination) and not options.incremental: msg = "Error: the destination storage has data. Try --clear." cleanup_and_exit(msg) destination.copyTransactionsFrom(source) cleanup_and_exit()
def main(argv=sys.argv): parser = optparse.OptionParser(description=__doc__, usage="%prog [options] config_file") parser.add_option( "--dry-run", dest="dry_run", action="store_true", help="Attempt to open the storages, then explain what would be done") parser.add_option( "--clear", dest="clear", action="store_true", help="Clear the contents of the destination storage before copying") parser.add_option( "--incremental", dest="incremental", action="store_true", help="Assume the destination contains a partial copy of the source " "and resume copying from the last transaction. WARNING: no " "effort is made to verify that the destination holds the same " "transaction data before this point! Use at your own risk. " "Currently only supports RelStorage destinations.") parser.set_defaults(dry_run=False, clear=False) options, args = parser.parse_args(argv[1:]) if len(args) != 1: parser.error("The name of one configuration file is required.") logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") schema = ZConfig.loadSchemaFile(StringIO(schema_xml)) config, handler = ZConfig.loadConfig(schema, args[0]) source = config.source.open() destination = config.destination.open() log.info("Storages opened successfully.") if options.incremental: if not hasattr(destination, '_adapter'): msg = ("Error: no API is known for determining the last committed " "transaction of the destination storage. Aborting " "conversion.") sys.exit(msg) if not storage_has_data(destination): log.warning("Destination empty, start conversion from the beginning.") else: last_tid = destination._adapter.txncontrol.get_tid( destination._load_cursor) next_tid = p64(last_tid+1) source = source.iterator(start=next_tid) log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid)) if options.dry_run: log.info("Dry run mode: not changing the destination.") if storage_has_data(destination): log.warning("The destination storage has data.") count = 0 for txn in source.iterator(): log.info('%s user=%s description=%s' % ( TimeStamp(txn.tid), txn.user, txn.description)) count += 1 log.info("Would copy %d transactions.", count) else: if options.clear: log.info("Clearing old data...") if hasattr(destination, 'zap_all'): destination.zap_all() else: msg = ("Error: no API is known for clearing this type " "of storage. Use another method.") sys.exit(msg) log.info("Done clearing old data.") if storage_has_data(destination): msg = "Error: the destination storage has data. Try --clear." sys.exit(msg) destination.copyTransactionsFrom(source) source.close() destination.close()