def __copy_transaction(self, other, trans):
# Originally adapted from ZODB.blob.BlobStorageMixin
tpc = self.tpc
num_txn_records = 0
txn_data_size = 0
num_blobs = 0
tmp_blobs_to_rm = []
tpc.tpc_begin(trans, trans.tid, trans.status)
for record in trans:
num_txn_records += 1
if record.data:
txn_data_size += len(record.data)
blobfile = None
if is_blob_record(record.data):
try:
blobfile = other.openCommittedBlobFile(
record.oid, record.tid)
except POSKeyError:
logger.exception("Failed to open blob to copy")
if blobfile is not None:
fd, name = tempfile.mkstemp(
suffix='.tmp', dir=self.blobhelper.temporaryDirectory())
tmp_blobs_to_rm.append(name)
logger.log(TRACE,
"Copying %s to temporary blob file %s for upload",
blobfile, name)
with os.fdopen(fd, 'wb') as target:
# If we don't get the length, ``copy_blob`` will.
old_pos = blobfile.tell()
blobfile.seek(0, 2)
length = blobfile.tell()
blobfile.seek(old_pos)
copy_blob(blobfile, target, length)
txn_data_size += length
blobfile.close()
self.restore.restoreBlob(record.oid, record.tid, record.data,
name, record.data_txn, trans)
else:
self.restore.restore(record.oid, record.tid, record.data, '',
record.data_txn, trans)
tpc.tpc_vote(trans)
tpc.tpc_finish(trans)
num_blobs = len(tmp_blobs_to_rm)
if num_blobs:
for tmp_blob in tmp_blobs_to_rm:
logger.log(TRACE, "Removing temporary blob file %s", tmp_blob)
try:
os.unlink(tmp_blob)
except OSError:
pass
return num_txn_records, txn_data_size, num_blobs
def restore_one(self, active_txn_meta,
oid, tid, data):
# The signature for both ``restore`` and ``restoreBlob``
# is:
#
# (oid, serial, data, (blobfilename|prev_txn), version, txn)
#
# Where ``txn`` is the TransactionMetaData object
# originally passed to ``tpc_begin``. It is only used to
# check that the same object has been passed.
#
# ``prev_txn`` is not used but would come from ``record.data_txn``
txn_data_size = len(data) if data else 0
blobfile = None
if is_blob_record(data):
try:
blobfile = self.storage.openCommittedBlobFile(
oid, tid)
except POSKeyError: # pragma: no cover
logger.exception("Failed to open blob to copy")
# We may not be able to read the data after this.
data = self.restore._crs_transform_record_data(data)
if blobfile is not None:
fd, name = tempfile.mkstemp(
suffix='.tmp',
dir=self.blobhelper.temporaryDirectory()
)
self.temp_blobs_to_rm.append(name)
logger.log(
TRACE,
"Copying %s to temporary blob file %s for upload",
blobfile, name)
with os.fdopen(fd, 'wb') as target:
# If we don't get the length, ``copy_blob`` will.
old_pos = blobfile.tell()
blobfile.seek(0, 2)
length = blobfile.tell()
blobfile.seek(old_pos)
copy_blob(blobfile, target, length)
txn_data_size += length
blobfile.close()
self.restore.restoreBlob(oid, tid, data,
name, None, active_txn_meta)
else:
self.restore.restore(oid, tid, data,
'', None, active_txn_meta)
return txn_data_size, blobfile is not None
def copyTransactionsFrom(self, other):
# pylint:disable=too-many-locals
# adapted from ZODB.blob.BlobStorageMixin
begin_time = time.time()
txnum = 0
total_size = 0
blobhelper = self.blobhelper
tpc = self.tpc
restore = self.restore
logger.info("Counting the transactions to copy.")
num_txns = 0
for _ in other.iterator():
num_txns += 1
logger.info("Copying %d transactions", num_txns)
for trans in other.iterator():
txnum += 1
num_txn_records = 0
tpc.tpc_begin(trans, trans.tid, trans.status)
for record in trans:
blobfile = None
if is_blob_record(record.data):
try:
blobfile = other.openCommittedBlobFile(
record.oid, record.tid)
except POSKeyError:
pass
if blobfile is not None:
fd, name = tempfile.mkstemp(
suffix='.tmp', dir=blobhelper.temporaryDirectory())
os.close(fd)
with open(name, 'wb') as target:
copy_blob(blobfile, target)
blobfile.close()
restore.restoreBlob(record.oid, record.tid, record.data,
name, record.data_txn, trans)
else:
restore.restore(record.oid, record.tid, record.data, '',
record.data_txn, trans)
num_txn_records += 1
if record.data:
total_size += len(record.data)
tpc.tpc_vote(trans)
tpc.tpc_finish(trans)
pct_complete = '%1.2f%%' % (txnum * 100.0 / num_txns)
elapsed = time.time() - begin_time
if elapsed:
rate = total_size / 1e6 / elapsed
else:
rate = 0.0
rate_str = '%1.3f' % rate
logger.info("Copied tid %d,%5d records | %6s MB/s (%6d/%6d,%7s)",
bytes8_to_int64(trans.tid), num_txn_records, rate_str,
txnum, num_txns, pct_complete)
elapsed = time.time() - begin_time
logger.info(
"All %d transactions copied successfully in %4.1f minutes.", txnum,
elapsed / 60.0)
def copyTransactionsFrom(self, other):
# pylint:disable=too-many-locals,too-many-statements,too-many-branches
# adapted from ZODB.blob.BlobStorageMixin
begin_time = time.time()
log_at = begin_time + self.log_interval
txnum = 0
total_size = 0
blobhelper = self.blobhelper
tpc = self.tpc
restore = self.restore
logger.info("Counting the transactions to copy.")
other_it = other.iterator()
logger.debug("Opened the other iterator: %s", other_it)
try:
num_txns = len(other_it)
if num_txns == 0:
# Hmm, that shouldn't really be right, should it?
# Try the other path.
raise TypeError()
except TypeError:
logger.debug("Iterator %s doesn't support len()", other_it)
num_txns = 0
for _ in other_it:
num_txns += 1
other_it.close()
other_it = other.iterator()
logger.info("Copying %d transactions", num_txns)
tmp_blobs_to_rm = []
for trans in other_it:
txnum += 1
num_txn_records = 0
tpc.tpc_begin(trans, trans.tid, trans.status)
for record in trans:
blobfile = None
if is_blob_record(record.data):
try:
blobfile = other.openCommittedBlobFile(
record.oid, record.tid)
except POSKeyError:
pass
if blobfile is not None:
fd, name = tempfile.mkstemp(
suffix='.tmp', dir=blobhelper.temporaryDirectory())
tmp_blobs_to_rm.append(name)
logger.debug(
"Copying %s to temporary blob file %s for upload",
blobfile, name)
with os.fdopen(fd, 'wb') as target:
copy_blob(blobfile, target)
blobfile.close()
restore.restoreBlob(record.oid, record.tid, record.data,
name, record.data_txn, trans)
else:
restore.restore(record.oid, record.tid, record.data, '',
record.data_txn, trans)
num_txn_records += 1
if record.data:
total_size += len(record.data)
tpc.tpc_vote(trans)
tpc.tpc_finish(trans)
for tmp_blob in tmp_blobs_to_rm:
logger.debug("Removing temporary blob file %s", tmp_blob)
try:
os.unlink(tmp_blob)
except OSError:
pass
del tmp_blobs_to_rm[:]
if txnum % self.log_count == 0 and time.time() > log_at:
now = time.time()
log_at = now + self.log_interval
pct_complete = '%1.2f%%' % (txnum * 100.0 / num_txns)
elapsed = now - begin_time
if elapsed:
rate = total_size / 1e6 / elapsed
else:
rate = 0.0
rate_str = '%1.3f' % rate
logger.info(
"Copied tid %d,%5d records | %6s MB/s (%6d/%6d,%7s)",
bytes8_to_int64(trans.tid), num_txn_records, rate_str,
txnum, num_txns, pct_complete)
elapsed = time.time() - begin_time
logger.info(
"All %d transactions copied successfully in %4.1f minutes.", txnum,
elapsed / 60.0)