def _post_process(self, source_rorp, dest_rorp, changed, success, inc):
"""Post process source_rorp and dest_rorp.
The point of this is to write statistics and metadata.
changed will be true if the files have changed. success will
be true if the files have been successfully updated (this is
always false for un-changed files).
"""
if Globals.preserve_hardlinks and source_rorp:
Hardlink.del_rorp(source_rorp)
if not changed or success:
if source_rorp:
self.statfileobj.add_source_file(source_rorp)
if dest_rorp:
self.statfileobj.add_dest_file(dest_rorp)
if success == 0:
metadata_rorp = dest_rorp
elif success == 1:
metadata_rorp = source_rorp
else:
metadata_rorp = None # in case deleted because of ListError
if success == 1 or success == 2:
self.statfileobj.add_changed(source_rorp, dest_rorp)
if metadata_rorp and metadata_rorp.lstat():
self.metawriter.write_object(metadata_rorp)
if Globals.file_statistics:
statistics.FileStats.update(source_rorp, dest_rorp, changed, inc)
def rbdir_equal(src_rorp, dest_rorp):
"""Like hardlink_equal, but make allowances for data directories"""
if not src_rorp.index and not dest_rorp.index: return 1
if (src_rorp.index and src_rorp.index[0] == 'rdiff-backup-data' and
src_rorp.index == dest_rorp.index):
# Don't compare dirs - they don't carry significant info
if dest_rorp.isdir() and src_rorp.isdir(): return 1
if dest_rorp.isreg() and src_rorp.isreg():
# Don't compare gzipped files because it is apparently
# non-deterministic.
if dest_rorp.index[-1].endswith('gz'): return 1
# Don't compare .missing increments because they don't matter
if dest_rorp.index[-1].endswith('.missing'): return 1
if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp):
Log("Different EAs in files %s and %s" %
(src_rorp.get_indexpath(), dest_rorp.get_indexpath()))
return None
if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp):
Log("Different ACLs in files %s and %s" %
(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
return None
if compare_hardlinks:
if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
elif src_rorp.equal_verbose(dest_rorp,
compare_ownership = compare_ownership):
return 1
Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
Log("%s: %s" % (dest_rorp.index,
Hardlink.get_indicies(dest_rorp, None)), 3)
return None
def equality_func(src_rorp, dest_rorp):
"""Combined eq func returns true iff two files compare same"""
if not src_rorp:
Log("Source rorp missing: " + str(dest_rorp), 3)
return 0
if not dest_rorp:
Log("Dest rorp missing: " + str(src_rorp), 3)
return 0
if not src_rorp.equal_verbose(dest_rorp,
compare_ownership = compare_ownership):
return 0
if compare_hardlinks and not Hardlink.rorp_eq(src_rorp, dest_rorp):
Log("Hardlink compare failure", 3)
Log("%s: %s" % (src_rorp.index,
Hardlink.get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index,
Hardlink.get_inode_key(dest_rorp)), 3)
return 0
if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp):
Log("Different EAs in files %s and %s" %
(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
return 0
if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp):
Log("Different ACLs in files %s and %s" %
(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
return 0
return 1
def testBuildingDict(self):
"""See if the partial inode dictionary is correct"""
Globals.preserve_hardlinks = 1
reset_hardlink_dicts()
for dsrp in selection.Select(self.hlinks_rp3).set_iter():
Hardlink.add_rorp(dsrp)
self.assertEqual(len(list(Hardlink._inode_index.keys())), 3)
def testBuildingDict(self): """See if the partial inode dictionary is correct""" Globals.preserve_hardlinks = 1 reset_hardlink_dicts() for dsrp in selection.Select(self.hardlink_dir3).set_iter(): Hardlink.add_rorp(dsrp) assert len(Hardlink._inode_index.keys()) == 3, \ Hardlink._inode_index
def testBuildingDict(self):
"""See if the partial inode dictionary is correct"""
Globals.preserve_hardlinks = 1
reset_hardlink_dicts()
for dsrp in selection.Select(self.hardlink_dir3).set_iter():
Hardlink.add_rorp(dsrp)
assert len(Hardlink._inode_index.keys()) == 3, \
Hardlink._inode_index
def _pre_process(self, source_rorp, dest_rorp):
"""Do initial processing on source_rorp and dest_rorp
It will not be clear whether source_rorp and dest_rorp have
errors at this point, so don't do anything which assumes they
will be backed up correctly.
"""
if Globals.preserve_hardlinks and source_rorp:
Hardlink.add_rorp(source_rorp, dest_rorp)
if (dest_rorp and dest_rorp.isdir() and Globals.process_uid != 0
and dest_rorp.getperms() % 0o1000 < 0o700):
self._unreadable_dir_init(source_rorp, dest_rorp)
def _patch_to_temp(self, basis_rp, diff_rorp, new):
"""Patch basis_rp, writing output in new, which doesn't exist yet"""
if diff_rorp.isflaglinked():
Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
return
if diff_rorp.get_attached_filetype() == 'snapshot':
copy_report = rpath.copy(diff_rorp, new)
else:
assert diff_rorp.get_attached_filetype() == 'diff', (
"File '{drp}' must be of type '{dtype}'.".format(drp=diff_rorp,
dtype='diff'))
copy_report = Rdiff.patch_local(basis_rp, diff_rorp, new)
self._check_hash(copy_report, diff_rorp)
if new.lstat():
rpath.copy_attribs(diff_rorp, new)
def update_hardlink_hash(self, diff_rorp):
"""Tag associated source_rorp with same hash diff_rorp points to"""
sha1sum = Hardlink.get_sha1(diff_rorp)
if not sha1sum:
return
source_rorp = self.get_source_rorp(diff_rorp.index)
source_rorp.set_sha1(sha1sum)
def _get_one_sig(cls, dest_base_rpath, index, src_rorp, dest_rorp):
"""Return a signature given source and destination rorps"""
if (Globals.preserve_hardlinks and src_rorp
and Hardlink.is_linked(src_rorp)):
dest_sig = rpath.RORPath(index)
dest_sig.flaglinked(Hardlink.get_link_index(src_rorp))
elif dest_rorp:
dest_sig = dest_rorp.getRORPath()
if dest_rorp.isreg():
dest_rp = longname.get_mirror_rp(dest_base_rpath, dest_rorp)
sig_fp = cls._get_one_sig_fp(dest_rp)
if sig_fp is None:
return None
dest_sig.setfile(sig_fp)
else:
dest_sig = rpath.RORPath(index)
return dest_sig
def testCompletedDict(self):
"""See if the hardlink dictionaries are built correctly"""
reset_hardlink_dicts()
for dsrp in selection.Select(self.hlinks_rp1).set_iter():
Hardlink.add_rorp(dsrp)
Hardlink.del_rorp(dsrp)
self.assertEqual(Hardlink._inode_index, {})
reset_hardlink_dicts()
for dsrp in selection.Select(self.hlinks_rp2).set_iter():
Hardlink.add_rorp(dsrp)
Hardlink.del_rorp(dsrp)
self.assertEqual(Hardlink._inode_index, {})
def testCompletedDict(self):
"""See if the hardlink dictionaries are built correctly"""
reset_hardlink_dicts()
for dsrp in selection.Select(self.hardlink_dir1).set_iter():
Hardlink.add_rorp(dsrp)
Hardlink.del_rorp(dsrp)
assert Hardlink._inode_index == {}, Hardlink._inode_index
reset_hardlink_dicts()
for dsrp in selection.Select(self.hardlink_dir2).set_iter():
Hardlink.add_rorp(dsrp)
Hardlink.del_rorp(dsrp)
assert Hardlink._inode_index == {}, Hardlink._inode_index
def hashes_changed(src_rp, mir_rorp):
"""Return 0 if their data hashes same, 1 otherwise"""
verify_sha1 = Hardlink.get_hash(mir_rorp)
if not verify_sha1:
log.Log("Metadata file has no digest for mirror file {mf}, "
"unable to compare.".format(mf=mir_rorp), log.WARNING)
return 0
elif (src_rp.getsize() == mir_rorp.getsize()
and hash.compute_sha1(src_rp) == verify_sha1):
return 0
return 1
def testCompletedDict(self):
"""See if the hardlink dictionaries are built correctly"""
reset_hardlink_dicts()
for dsrp in selection.Select(self.hardlink_dir1).set_iter():
Hardlink.add_rorp(dsrp)
Hardlink.del_rorp(dsrp)
assert Hardlink._inode_index == {}, Hardlink._inode_index
reset_hardlink_dicts()
for dsrp in selection.Select(self.hardlink_dir2).set_iter():
Hardlink.add_rorp(dsrp)
Hardlink.del_rorp(dsrp)
assert Hardlink._inode_index == {}, Hardlink._inode_index
def Verify(mirror_rp, inc_rp, verify_time):
"""Compute SHA1 sums of repository files and check against metadata"""
assert mirror_rp.conn is Globals.local_connection, (
"Only verify mirror locally, not remotely over '{conn}'.".format(
conn=mirror_rp.conn))
repo_iter = RepoSide.init_and_get_iter(mirror_rp, inc_rp, verify_time)
base_index = RepoSide.mirror_base.index
bad_files = 0
no_hash = 0
for repo_rorp in repo_iter:
if not repo_rorp.isreg():
continue
verify_sha1 = Hardlink.get_hash(repo_rorp)
if not verify_sha1:
log.Log("Cannot find SHA1 digest for file {fi}, "
"perhaps because this feature was added in v1.1.1".format(
fi=repo_rorp), log.WARNING)
no_hash += 1
continue
fp = RepoSide.rf_cache.get_fp(base_index + repo_rorp.index, repo_rorp)
computed_hash = hash.compute_sha1_fp(fp)
if computed_hash == verify_sha1:
log.Log("Verified SHA1 digest of file {fi}".format(fi=repo_rorp),
log.INFO)
else:
bad_files += 1
log.Log("Computed SHA1 digest of file {fi} '{cd}' "
"doesn't match recorded digest of '{rd}'. "
"Your backup repository may be corrupted!".format(
fi=repo_rorp, cd=computed_hash, rd=verify_sha1),
log.WARNING)
RepoSide.close_rf_cache()
if bad_files:
log.Log("Verification found {cf} potentially corrupted files".format(
cf=bad_files), log.ERROR)
return 2
if no_hash:
log.Log("Verification found {fi} files without hash, all others "
"could be verified successfully".format(fi=no_hash), log.NOTE)
else:
log.Log("All files verified successfully", log.NOTE)
return 0
def get_sigs(cls, dest_base_rpath):
"""
Yield signatures of any changed destination files
"""
flush_threshold = Globals.pipeline_max_length - 2
num_rorps_seen = 0
for src_rorp, dest_rorp in cls.CCPP:
# If we are backing up across a pipe, we must flush the pipeline
# every so often so it doesn't get congested on destination end.
if (Globals.backup_reader is not Globals.backup_writer):
num_rorps_seen += 1
if (num_rorps_seen > flush_threshold):
num_rorps_seen = 0
yield iterfile.MiscIterFlushRepeat
if not (src_rorp and dest_rorp and src_rorp == dest_rorp
and (not Globals.preserve_hardlinks
or Hardlink.rorp_eq(src_rorp, dest_rorp))):
index = src_rorp and src_rorp.index or dest_rorp.index
sig = cls._get_one_sig(dest_base_rpath, index, src_rorp,
dest_rorp)
if sig:
cls.CCPP.flag_changed(index)
yield sig
def hardlink_rorp_eq(src_rorp, dest_rorp):
Hardlink.add_rorp(dest_rorp)
Hardlink.add_rorp(src_rorp, dest_rorp)
rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
if not src_rorp.isreg() or not dest_rorp.isreg(
) or src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1:
if not rorp_eq:
Log("Hardlink compare error with when no links exist exist", 3)
Log(
"%s: %s" %
(src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
Log(
"%s: %s" %
(dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
return 0
elif src_rorp.getnumlinks() > 1 and not Hardlink.islinked(src_rorp):
if rorp_eq:
Log(
"Hardlink compare error with first linked src_rorp and no dest_rorp sha1",
3)
Log(
"%s: %s" %
(src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
Log(
"%s: %s" %
(dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
return 0
hash.compute_sha1(dest_rorp)
rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
if src_rorp.getnumlinks() != dest_rorp.getnumlinks():
if rorp_eq:
Log(
"Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with differing link counts",
3)
Log(
"%s: %s" %
(src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
Log(
"%s: %s" %
(dest_rorp.index, Hardlink.get_inode_key(dest_rorp)),
3)
return 0
elif not rorp_eq:
Log(
"Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with equal link counts",
3)
Log(
"%s: %s" %
(src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
Log(
"%s: %s" %
(dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
return 0
elif src_rorp.getnumlinks() != dest_rorp.getnumlinks():
if rorp_eq:
Log(
"Hardlink compare error with non-first linked src_rorp and with differing link counts",
3)
Log(
"%s: %s" %
(src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
Log(
"%s: %s" %
(dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
return 0
elif not rorp_eq:
Log(
"Hardlink compare error with non-first linked src_rorp and with equal link counts",
3)
Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)),
3)
Log(
"%s: %s" %
(dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
return 0
Hardlink.del_rorp(src_rorp)
Hardlink.del_rorp(dest_rorp)
return 1
def _hardlink_rorp_eq(src_rorp, dest_rorp):
"""Compare two files for hardlink equality, encompassing being hard-linked,
having the same hashsum, and the same number of link counts."""
Hardlink.add_rorp(dest_rorp)
Hardlink.add_rorp(src_rorp, dest_rorp)
rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
if not src_rorp.isreg() or not dest_rorp.isreg() or src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1:
if not rorp_eq:
Log("Hardlink compare error with when no links exist", 3)
Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
return False
elif src_rorp.getnumlinks() > 1 and not Hardlink.is_linked(src_rorp):
if rorp_eq:
Log("Hardlink compare error with first linked src_rorp and no dest_rorp sha1", 3)
Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
return False
hash.compute_sha1(dest_rorp)
rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
if src_rorp.getnumlinks() != dest_rorp.getnumlinks():
if rorp_eq:
Log("Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with differing link counts", 3)
Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
return False
elif not rorp_eq:
Log("Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with equal link counts", 3)
Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
return False
elif src_rorp.getnumlinks() != dest_rorp.getnumlinks():
if rorp_eq:
Log("Hardlink compare error with non-first linked src_rorp and with differing link counts", 3)
Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
return False
elif not rorp_eq:
Log("Hardlink compare error with non-first linked src_rorp and with equal link counts", 3)
Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
return False
Hardlink.del_rorp(src_rorp)
Hardlink.del_rorp(dest_rorp)
return True
def postprocess(src_rorp, dest_rorp): """After comparison, process src_rorp and dest_rorp""" if compare_hardlinks and src_rorp: Hardlink.del_rorp(src_rorp)
def preprocess(src_rorp, dest_rorp): """Initially process src and dest_rorp""" if compare_hardlinks and src_rorp: Hardlink.add_rorp(src_rorp, dest_rorp)
def _patch_hardlink_to_temp(self, diff_rorp, new):
"""Hardlink diff_rorp to temp, update hash if necessary"""
Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
self.CCPP.update_hardlink_hash(diff_rorp)