def checkFile(cache, crypt, checksum, size, basis, compressed, encrypted,
added, authCond):
fsize = cache.size(checksum)
if not cache.exists(checksum):
#print(f"{checksum}: does not exist")
missing.append(checksum)
elif fsize == 0:
print(f"{checksum} is empty")
zero.append(checksum)
else:
authenticate = (authCond == 'all')
if fsize != size:
print(
f"{checksum}: size mismatch Expected: {size}, found {fsize} ({fsize - size})-- {added} -- {basis is not None} "
)
mismatch.append((checksum, size, fsize))
sizes.setdefault((fsize - size), []).append(checksum)
if authCond != 'none':
authenticate = True
elif basis:
#print(f"{checksum} -- {compressed} {encrypted}", flush=True)
instream = decryptHeader(crypt, cache.open(checksum, "rb"))
uc = CompressedBuffer.UncompressedBufferedReader(
instream, compressor=compressed)
data = uc.read(256)
kind = magic.from_buffer(data)
if kind != 'rdiff network-delta data':
print(f"{checksum}: Not a delta: {kind}")
notdelta.append((checksum, kind))
if authenticate:
with cache.open(checksum, "rb") as f:
if not authenticateFile(f, fsize, crypt):
print(f"{checksum} did not authenticate")
notauth.append(checksum)
def processArgs():
parser = argparse.ArgumentParser(
description='Encrypt files for a backup database', add_help=False)
(_, remaining) = Config.parseConfigOptions(parser)
Config.addCommonOptions(parser)
Config.addPasswordOptions(parser)
parser.add_argument('--output',
'-o',
dest='output',
required=True,
help="Output directory")
parser.add_argument('--json',
'-j',
default=None,
dest='input',
help='JSON input file')
parser.add_argument('--signature',
'-s',
default=False,
action='store_true',
dest='signature',
help='Generate signature file')
parser.add_argument('--compress-data',
'-Z',
dest='compress',
const='zlib',
default=None,
nargs='?',
choices=CompressedBuffer.getCompressors(),
help='Compress files')
parser.add_argument('names',
nargs='*',
help="List of pathnames to decrypt")
parser.add_argument('--help', '-h', action='help')
Util.addGenCompletions(parser)
args = parser.parse_args(remaining)
return args
def recoverChecksum(self, cksum, authenticate=True, chain=None, basisFile=None):
self.logger.debug("Recovering checksum: %s", cksum)
cksInfo = None
if not chain:
chain = self.db.getChecksumInfoChain(cksum)
if chain:
cksInfo = chain.pop(0)
if cksInfo['checksum'] != cksum:
self.logger.error("Unexpected checksum: %s. Expected: %s", cksInfo['checksum'], cksum)
return None
else:
cksInfo = self.db.getChecksumInfo(cksum)
if cksInfo is None:
self.logger.error("Checksum %s not found", cksum)
return None
#self.logger.debug(" %s: %s", cksum, str(cksInfo))
try:
if not cksInfo['isfile']:
raise RegenerateException("{} is not a file".format(cksum))
if cksInfo['basis']:
if basisFile:
basis = basisFile
basis.seek(0)
else:
basis = self.recoverChecksum(cksInfo['basis'], authenticate, chain)
if cksInfo['encrypted']:
patchfile = self.decryptFile(cksum, cksInfo['disksize'], authenticate)
else:
patchfile = self.cacheDir.open(cksum, 'rb')
if cksInfo['compressed']:
self.logger.debug("Uncompressing %s", cksum)
temp = tempfile.TemporaryFile()
buf = CompressedBuffer.UncompressedBufferedReader(patchfile, compressor=cksInfo['compressed'])
shutil.copyfileobj(buf, temp)
temp.seek(0)
patchfile = temp
try:
output = librsync.patch(basis, patchfile)
#output.seek(0)
return output
except librsync.LibrsyncError as e:
self.logger.error("Recovering checksum: %s : %s", cksum, e)
raise RegenerateException("Checksum: {}: Error: {}".format(cksum, e))
else:
if cksInfo['encrypted']:
output = self.decryptFile(cksum, cksInfo['disksize'])
else:
output = self.cacheDir.open(cksum, "rb")
if cksInfo['compressed'] is not None and cksInfo['compressed'].lower() != 'none':
self.logger.debug("Uncompressing %s", cksum)
temp = tempfile.TemporaryFile()
buf = CompressedBuffer.UncompressedBufferedReader(output, compressor=cksInfo['compressed'])
shutil.copyfileobj(buf, temp)
temp.seek(0)
output = temp
return output
except RegenerateException:
raise
except Exception as e:
self.logger.error("Unable to recover checksum %s: %s", cksum, e)
#self.logger.exception(e)
raise RegenerateException("Checksum: {}: Error: {}".format(cksum, e))
def sendData(sender, data, encrypt, chunksize=(16 * 1024), hasher=None, compress=None, stats=None, signature=False, progress=None, progressPeriod=8*1024*1024):
"""
Send a block of data, optionally encrypt and/or compress it before sending
Compress should be either None, for no compression, or one of the known compression types (zlib, bzip, lzma)
"""
#logger = logging.getLogger('Data')
if isinstance(sender, Connection.Connection):
sender = sender.sender
size = 0
status = "OK"
ck = None
sig = None
start = time.time()
if progress:
# Set the chunksize
if progressPeriod % chunksize != 0:
progressPeriod -= progressPeriod % chunksize
if compress:
stream = CompressedBuffer.CompressedBufferedReader(data, hasher=hasher, signature=signature, compressor=compress)
else:
stream = CompressedBuffer.BufferedReader(data, hasher=hasher, signature=signature)
try:
if encrypt.iv:
sender.sendMessage(encrypt.iv, raw=True)
accumulateStat(stats, 'dataSent', len(encrypt.iv))
for chunk, eof in _chunks(stream, chunksize):
#print len(chunk), eof
if chunk:
data = encrypt.encrypt(chunk)
else:
data = b''
if eof:
data += encrypt.finish()
#chunkMessage = { "chunk" : num, "data": data }
if data:
sender.sendMessage(data, raw=True)
accumulateStat(stats, 'dataSent', len(data))
size += len(data)
if progress:
if (size % progressPeriod) == 0:
progress()
#num += 1
digest = encrypt.digest()
if digest:
sender.sendMessage(digest, raw=True)
accumulateStat(stats, 'dataSent', len(digest))
except Exception as e:
status = "Fail"
#logger = logging.getLogger('Data')
#logger.exception(e)
raise e
finally:
sender.sendMessage(b'', raw=True)
compressed = compress if stream.isCompressed() else "None"
size = stream.size()
accumulateStat(stats, 'dataBacked', size)
message = { "chunk": "done", "size": size, "status": status, "compressed": compressed }
if hasher:
ck = stream.checksum()
message["checksum"] = ck
if signature:
sig = stream.signatureFile()
#print message
sender.sendMessage(message)
stream = None
end = time.time()
global _transmissionTime
_transmissionTime += end - start
return size, ck, sig