def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"-d",
"--days",
dest="days",
default="0",
help="Days of history to keep (default 0)",
)
parser.add_option(
"--prepack",
dest="prepack",
default=False,
action="store_true",
help="Perform only the pre-pack preparation stage of a pack. "
"(Only works with some storage types)",
)
parser.add_option(
"--use-prepack-state",
dest="reuse_prepack",
default=False,
action="store_true",
help="Skip the preparation stage and go straight to packing. "
"Requires that a pre-pack has been run, or that packing was aborted "
"before it was completed.",
)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(BytesIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
t = time.time() - float(options.days) * 86400.0
for s in config.storages:
name = '%s (%s)' % ((s.name or 'storage'), s.__class__.__name__)
log.info("Opening %s...", name)
storage = s.open()
log.info("Packing %s.", name)
if options.prepack or options.reuse_prepack:
storage.pack(t,
ZODB.serialize.referencesf,
prepack_only=options.prepack,
skip_prepack=options.reuse_prepack)
else:
# Be non-relstorage Storages friendly
storage.pack(t, ZODB.serialize.referencesf)
storage.close()
log.info("Packed %s.", name)
def __init__(self, threshold=10 * 1024 * 1024):
self._threshold = threshold
self._f = BytesIO()
class AutoTemporaryFile(object):
"""Initially a BytesIO, but becomes a TemporaryFile if it grows large.
Not thread safe.
"""
def __init__(self, threshold=10 * 1024 * 1024):
self._threshold = threshold
self._f = BytesIO()
def read(self, n=None):
if n is not None:
return self._f.read(n)
else:
return self._f.read()
def seek(self, pos, mode=0):
self._f.seek(pos, mode)
def tell(self):
return self._f.tell()
def close(self):
self._f.close()
def write(self, data):
threshold = self._threshold
if threshold and self._f.tell() + len(data) >= threshold:
# convert to TemporaryFile
self._threshold = 0
f = tempfile.TemporaryFile()
f.write(self._f.getvalue())
f.seek(self._f.tell())
self._f = f
self._f.write(data)
def __init__(self, threshold=10 * 1024 * 1024):
self._threshold = threshold
self._f = BytesIO()
class AutoTemporaryFile(object):
"""Initially a BytesIO, but becomes a TemporaryFile if it grows large.
Not thread safe.
"""
def __init__(self, threshold=10 * 1024 * 1024):
self._threshold = threshold
self._f = BytesIO()
def read(self, n=None):
if n is not None:
return self._f.read(n)
else:
return self._f.read()
def seek(self, pos, mode=0):
self._f.seek(pos, mode)
def tell(self):
return self._f.tell()
def close(self):
self._f.close()
def write(self, data):
threshold = self._threshold
if threshold and self._f.tell() + len(data) >= threshold:
# convert to TemporaryFile
self._threshold = 0
f = tempfile.TemporaryFile()
f.write(self._f.getvalue())
f.seek(self._f.tell())
self._f = f
self._f.write(data)
def main(argv=sys.argv):
parser = optparse.OptionParser(description=__doc__,
usage="%prog [options] config_file")
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
help="Attempt to open the storages, then explain what would be done")
parser.add_option(
"--clear",
dest="clear",
action="store_true",
help="Clear the contents of the destination storage before copying")
parser.add_option(
"--incremental",
dest="incremental",
action="store_true",
help="Assume the destination contains a partial copy of the source "
"and resume copying from the last transaction. WARNING: no "
"effort is made to verify that the destination holds the same "
"transaction data before this point! Use at your own risk. "
"Currently only supports RelStorage destinations.")
parser.set_defaults(dry_run=False, clear=False)
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("The name of one configuration file is required.")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s %(message)s")
schema = ZConfig.loadSchemaFile(BytesIO(schema_xml))
config, handler = ZConfig.loadConfig(schema, args[0])
source = config.source.open()
destination = config.destination.open()
log.info("Storages opened successfully.")
if options.incremental:
if not hasattr(destination, '_adapter'):
msg = ("Error: no API is known for determining the last committed "
"transaction of the destination storage. Aborting "
"conversion.")
sys.exit(msg)
if not storage_has_data(destination):
log.warning(
"Destination empty, start conversion from the beginning.")
else:
last_tid = destination._adapter.txncontrol.get_tid(
destination._load_cursor)
next_tid = p64(last_tid + 1)
source = source.iterator(start=next_tid)
log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
if options.dry_run:
log.info("Dry run mode: not changing the destination.")
if storage_has_data(destination):
log.warning("The destination storage has data.")
count = 0
for txn in source.iterator():
log.info('%s user=%s description=%s' %
(TimeStamp(txn.tid), txn.user, txn.description))
count += 1
log.info("Would copy %d transactions.", count)
else:
if options.clear:
log.info("Clearing old data...")
if hasattr(destination, 'zap_all'):
destination.zap_all()
else:
msg = ("Error: no API is known for clearing this type "
"of storage. Use another method.")
sys.exit(msg)
log.info("Done clearing old data.")
if storage_has_data(destination):
msg = "Error: the destination storage has data. Try --clear."
sys.exit(msg)
destination.copyTransactionsFrom(source)
source.close()
destination.close()
