def main(argv=sys.argv): parser = optparse.OptionParser(description=__doc__, usage="%prog [options] config_file") parser.add_option( "-d", "--days", dest="days", default="0", help="Days of history to keep (default 0)", ) parser.add_option( "--prepack", dest="prepack", default=False, action="store_true", help="Perform only the pre-pack preparation stage of a pack. " "(Only works with some storage types)", ) parser.add_option( "--use-prepack-state", dest="reuse_prepack", default=False, action="store_true", help="Skip the preparation stage and go straight to packing. " "Requires that a pre-pack has been run, or that packing was aborted " "before it was completed.", ) options, args = parser.parse_args(argv[1:]) if len(args) != 1: parser.error("The name of one configuration file is required.") logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") schema = ZConfig.loadSchemaFile(BytesIO(schema_xml)) config, handler = ZConfig.loadConfig(schema, args[0]) t = time.time() - float(options.days) * 86400.0 for s in config.storages: name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__) log.info("Opening %s...", name) storage = s.open() log.info("Packing %s.", name) if options.prepack or options.reuse_prepack: storage.pack(t, ZODB.serialize.referencesf, prepack_only=options.prepack, skip_prepack=options.reuse_prepack) else: # Be non-relstorage Storages friendly storage.pack(t, ZODB.serialize.referencesf) storage.close() log.info("Packed %s.", name)
def __init__(self, threshold=10 * 1024 * 1024): self._threshold = threshold self._f = BytesIO()
class AutoTemporaryFile(object): """Initially a BytesIO, but becomes a TemporaryFile if it grows large. Not thread safe. """ def __init__(self, threshold=10 * 1024 * 1024): self._threshold = threshold self._f = BytesIO() def read(self, n=None): if n is not None: return self._f.read(n) else: return self._f.read() def seek(self, pos, mode=0): self._f.seek(pos, mode) def tell(self): return self._f.tell() def close(self): self._f.close() def write(self, data): threshold = self._threshold if threshold and self._f.tell() + len(data) >= threshold: # convert to TemporaryFile self._threshold = 0 f = tempfile.TemporaryFile() f.write(self._f.getvalue()) f.seek(self._f.tell()) self._f = f self._f.write(data)
def main(argv=sys.argv): parser = optparse.OptionParser(description=__doc__, usage="%prog [options] config_file") parser.add_option( "--dry-run", dest="dry_run", action="store_true", help="Attempt to open the storages, then explain what would be done") parser.add_option( "--clear", dest="clear", action="store_true", help="Clear the contents of the destination storage before copying") parser.add_option( "--incremental", dest="incremental", action="store_true", help="Assume the destination contains a partial copy of the source " "and resume copying from the last transaction. WARNING: no " "effort is made to verify that the destination holds the same " "transaction data before this point! Use at your own risk. " "Currently only supports RelStorage destinations.") parser.set_defaults(dry_run=False, clear=False) options, args = parser.parse_args(argv[1:]) if len(args) != 1: parser.error("The name of one configuration file is required.") logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s %(message)s") schema = ZConfig.loadSchemaFile(BytesIO(schema_xml)) config, handler = ZConfig.loadConfig(schema, args[0]) source = config.source.open() destination = config.destination.open() log.info("Storages opened successfully.") if options.incremental: if not hasattr(destination, '_adapter'): msg = ("Error: no API is known for determining the last committed " "transaction of the destination storage. Aborting " "conversion.") sys.exit(msg) if not storage_has_data(destination): log.warning( "Destination empty, start conversion from the beginning.") else: last_tid = destination._adapter.txncontrol.get_tid( destination._load_cursor) next_tid = p64(last_tid + 1) source = source.iterator(start=next_tid) log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid)) if options.dry_run: log.info("Dry run mode: not changing the destination.") if storage_has_data(destination): log.warning("The destination storage has data.") count = 0 for txn in source.iterator(): log.info('%s user=%s description=%s' % (TimeStamp(txn.tid), txn.user, txn.description)) count += 1 log.info("Would copy %d transactions.", count) else: if options.clear: log.info("Clearing old data...") if hasattr(destination, 'zap_all'): destination.zap_all() else: msg = ("Error: no API is known for clearing this type " "of storage. Use another method.") sys.exit(msg) log.info("Done clearing old data.") if storage_has_data(destination): msg = "Error: the destination storage has data. Try --clear." sys.exit(msg) destination.copyTransactionsFrom(source) source.close() destination.close()