Example #1
0
    def _post_process(self, source_rorp, dest_rorp, changed, success, inc):
        """Post process source_rorp and dest_rorp.

        The point of this is to write statistics and metadata.

        changed will be true if the files have changed.  success will
        be true if the files have been successfully updated (this is
        always false for un-changed files).

        """
        if Globals.preserve_hardlinks and source_rorp:
            Hardlink.del_rorp(source_rorp)

        if not changed or success:
            if source_rorp:
                self.statfileobj.add_source_file(source_rorp)
            if dest_rorp:
                self.statfileobj.add_dest_file(dest_rorp)
        if success == 0:
            metadata_rorp = dest_rorp
        elif success == 1:
            metadata_rorp = source_rorp
        else:
            metadata_rorp = None  # in case deleted because of ListError
        if success == 1 or success == 2:
            self.statfileobj.add_changed(source_rorp, dest_rorp)

        if metadata_rorp and metadata_rorp.lstat():
            self.metawriter.write_object(metadata_rorp)
        if Globals.file_statistics:
            statistics.FileStats.update(source_rorp, dest_rorp, changed, inc)
Example #2
0
	def rbdir_equal(src_rorp, dest_rorp):
		"""Like hardlink_equal, but make allowances for data directories"""
		if not src_rorp.index and not dest_rorp.index: return 1
		if (src_rorp.index and src_rorp.index[0] == 'rdiff-backup-data' and
			src_rorp.index == dest_rorp.index):
			# Don't compare dirs - they don't carry significant info
			if dest_rorp.isdir() and src_rorp.isdir(): return 1
			if dest_rorp.isreg() and src_rorp.isreg():
				# Don't compare gzipped files because it is apparently
				# non-deterministic.
				if dest_rorp.index[-1].endswith('gz'): return 1
				# Don't compare .missing increments because they don't matter
				if dest_rorp.index[-1].endswith('.missing'): return 1
		if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp):
			Log("Different EAs in files %s and %s" %
				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()))
			return None
		if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp):
			Log("Different ACLs in files %s and %s" %
				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
			return None
		if compare_hardlinks:
			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
		elif src_rorp.equal_verbose(dest_rorp,
									compare_ownership = compare_ownership):
			return 1
		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
		Log("%s: %s" % (dest_rorp.index,
						Hardlink.get_indicies(dest_rorp, None)), 3)
		return None
Example #3
0
	def equality_func(src_rorp, dest_rorp):
		"""Combined eq func returns true iff two files compare same"""
		if not src_rorp:
			Log("Source rorp missing: " + str(dest_rorp), 3)
			return 0
		if not dest_rorp:
			Log("Dest rorp missing: " + str(src_rorp), 3)
			return 0
		if not src_rorp.equal_verbose(dest_rorp,
									  compare_ownership = compare_ownership):
			return 0
		if compare_hardlinks and not Hardlink.rorp_eq(src_rorp, dest_rorp):
			Log("Hardlink compare failure", 3)
			Log("%s: %s" % (src_rorp.index,
							Hardlink.get_inode_key(src_rorp)), 3)
			Log("%s: %s" % (dest_rorp.index,
							Hardlink.get_inode_key(dest_rorp)), 3)
			return 0
		if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp):
			Log("Different EAs in files %s and %s" %
				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
			return 0
		if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp):
			Log("Different ACLs in files %s and %s" %
				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
			return 0
		return 1
Example #4
0
    def testBuildingDict(self):
        """See if the partial inode dictionary is correct"""
        Globals.preserve_hardlinks = 1
        reset_hardlink_dicts()
        for dsrp in selection.Select(self.hlinks_rp3).set_iter():
            Hardlink.add_rorp(dsrp)

        self.assertEqual(len(list(Hardlink._inode_index.keys())), 3)
Example #5
0
	def testBuildingDict(self):
		"""See if the partial inode dictionary is correct"""
		Globals.preserve_hardlinks = 1
		reset_hardlink_dicts()
		for dsrp in selection.Select(self.hardlink_dir3).set_iter():
			Hardlink.add_rorp(dsrp)
		
		assert len(Hardlink._inode_index.keys()) == 3, \
			   Hardlink._inode_index
Example #6
0
    def testBuildingDict(self):
        """See if the partial inode dictionary is correct"""
        Globals.preserve_hardlinks = 1
        reset_hardlink_dicts()
        for dsrp in selection.Select(self.hardlink_dir3).set_iter():
            Hardlink.add_rorp(dsrp)

        assert len(Hardlink._inode_index.keys()) == 3, \
            Hardlink._inode_index
Example #7
0
    def _pre_process(self, source_rorp, dest_rorp):
        """Do initial processing on source_rorp and dest_rorp

        It will not be clear whether source_rorp and dest_rorp have
        errors at this point, so don't do anything which assumes they
        will be backed up correctly.

        """
        if Globals.preserve_hardlinks and source_rorp:
            Hardlink.add_rorp(source_rorp, dest_rorp)
        if (dest_rorp and dest_rorp.isdir() and Globals.process_uid != 0
                and dest_rorp.getperms() % 0o1000 < 0o700):
            self._unreadable_dir_init(source_rorp, dest_rorp)
Example #8
0
 def _patch_to_temp(self, basis_rp, diff_rorp, new):
     """Patch basis_rp, writing output in new, which doesn't exist yet"""
     if diff_rorp.isflaglinked():
         Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
         return
     if diff_rorp.get_attached_filetype() == 'snapshot':
         copy_report = rpath.copy(diff_rorp, new)
     else:
         assert diff_rorp.get_attached_filetype() == 'diff', (
             "File '{drp}' must be of type '{dtype}'.".format(drp=diff_rorp,
                                                              dtype='diff'))
         copy_report = Rdiff.patch_local(basis_rp, diff_rorp, new)
     self._check_hash(copy_report, diff_rorp)
     if new.lstat():
         rpath.copy_attribs(diff_rorp, new)
Example #9
0
 def update_hardlink_hash(self, diff_rorp):
     """Tag associated source_rorp with same hash diff_rorp points to"""
     sha1sum = Hardlink.get_sha1(diff_rorp)
     if not sha1sum:
         return
     source_rorp = self.get_source_rorp(diff_rorp.index)
     source_rorp.set_sha1(sha1sum)
Example #10
0
 def _get_one_sig(cls, dest_base_rpath, index, src_rorp, dest_rorp):
     """Return a signature given source and destination rorps"""
     if (Globals.preserve_hardlinks and src_rorp
             and Hardlink.is_linked(src_rorp)):
         dest_sig = rpath.RORPath(index)
         dest_sig.flaglinked(Hardlink.get_link_index(src_rorp))
     elif dest_rorp:
         dest_sig = dest_rorp.getRORPath()
         if dest_rorp.isreg():
             dest_rp = longname.get_mirror_rp(dest_base_rpath, dest_rorp)
             sig_fp = cls._get_one_sig_fp(dest_rp)
             if sig_fp is None:
                 return None
             dest_sig.setfile(sig_fp)
     else:
         dest_sig = rpath.RORPath(index)
     return dest_sig
Example #11
0
    def testCompletedDict(self):
        """See if the hardlink dictionaries are built correctly"""
        reset_hardlink_dicts()
        for dsrp in selection.Select(self.hlinks_rp1).set_iter():
            Hardlink.add_rorp(dsrp)
            Hardlink.del_rorp(dsrp)
        self.assertEqual(Hardlink._inode_index, {})

        reset_hardlink_dicts()
        for dsrp in selection.Select(self.hlinks_rp2).set_iter():
            Hardlink.add_rorp(dsrp)
            Hardlink.del_rorp(dsrp)
        self.assertEqual(Hardlink._inode_index, {})
Example #12
0
	def testCompletedDict(self):
		"""See if the hardlink dictionaries are built correctly"""
		reset_hardlink_dicts()
		for dsrp in selection.Select(self.hardlink_dir1).set_iter():
			Hardlink.add_rorp(dsrp)
			Hardlink.del_rorp(dsrp)
		assert Hardlink._inode_index == {}, Hardlink._inode_index

		reset_hardlink_dicts()
		for dsrp in selection.Select(self.hardlink_dir2).set_iter():
			Hardlink.add_rorp(dsrp)
			Hardlink.del_rorp(dsrp)
		assert Hardlink._inode_index == {}, Hardlink._inode_index
Example #13
0
 def hashes_changed(src_rp, mir_rorp):
     """Return 0 if their data hashes same, 1 otherwise"""
     verify_sha1 = Hardlink.get_hash(mir_rorp)
     if not verify_sha1:
         log.Log("Metadata file has no digest for mirror file {mf}, "
                 "unable to compare.".format(mf=mir_rorp), log.WARNING)
         return 0
     elif (src_rp.getsize() == mir_rorp.getsize()
           and hash.compute_sha1(src_rp) == verify_sha1):
         return 0
     return 1
Example #14
0
    def testCompletedDict(self):
        """See if the hardlink dictionaries are built correctly"""
        reset_hardlink_dicts()
        for dsrp in selection.Select(self.hardlink_dir1).set_iter():
            Hardlink.add_rorp(dsrp)
            Hardlink.del_rorp(dsrp)
        assert Hardlink._inode_index == {}, Hardlink._inode_index

        reset_hardlink_dicts()
        for dsrp in selection.Select(self.hardlink_dir2).set_iter():
            Hardlink.add_rorp(dsrp)
            Hardlink.del_rorp(dsrp)
        assert Hardlink._inode_index == {}, Hardlink._inode_index
Example #15
0
def Verify(mirror_rp, inc_rp, verify_time):
    """Compute SHA1 sums of repository files and check against metadata"""
    assert mirror_rp.conn is Globals.local_connection, (
        "Only verify mirror locally, not remotely over '{conn}'.".format(
            conn=mirror_rp.conn))
    repo_iter = RepoSide.init_and_get_iter(mirror_rp, inc_rp, verify_time)
    base_index = RepoSide.mirror_base.index

    bad_files = 0
    no_hash = 0
    for repo_rorp in repo_iter:
        if not repo_rorp.isreg():
            continue
        verify_sha1 = Hardlink.get_hash(repo_rorp)
        if not verify_sha1:
            log.Log("Cannot find SHA1 digest for file {fi}, "
                    "perhaps because this feature was added in v1.1.1".format(
                        fi=repo_rorp), log.WARNING)
            no_hash += 1
            continue
        fp = RepoSide.rf_cache.get_fp(base_index + repo_rorp.index, repo_rorp)
        computed_hash = hash.compute_sha1_fp(fp)
        if computed_hash == verify_sha1:
            log.Log("Verified SHA1 digest of file {fi}".format(fi=repo_rorp),
                    log.INFO)
        else:
            bad_files += 1
            log.Log("Computed SHA1 digest of file {fi} '{cd}' "
                    "doesn't match recorded digest of '{rd}'. "
                    "Your backup repository may be corrupted!".format(
                        fi=repo_rorp, cd=computed_hash, rd=verify_sha1),
                    log.WARNING)
    RepoSide.close_rf_cache()
    if bad_files:
        log.Log("Verification found {cf} potentially corrupted files".format(
            cf=bad_files), log.ERROR)
        return 2
    if no_hash:
        log.Log("Verification found {fi} files without hash, all others "
                "could be verified successfully".format(fi=no_hash), log.NOTE)
    else:
        log.Log("All files verified successfully", log.NOTE)
    return 0
Example #16
0
    def get_sigs(cls, dest_base_rpath):
        """
        Yield signatures of any changed destination files
        """
        flush_threshold = Globals.pipeline_max_length - 2
        num_rorps_seen = 0
        for src_rorp, dest_rorp in cls.CCPP:
            # If we are backing up across a pipe, we must flush the pipeline
            # every so often so it doesn't get congested on destination end.
            if (Globals.backup_reader is not Globals.backup_writer):
                num_rorps_seen += 1
                if (num_rorps_seen > flush_threshold):
                    num_rorps_seen = 0
                    yield iterfile.MiscIterFlushRepeat
            if not (src_rorp and dest_rorp and src_rorp == dest_rorp
                    and (not Globals.preserve_hardlinks
                         or Hardlink.rorp_eq(src_rorp, dest_rorp))):

                index = src_rorp and src_rorp.index or dest_rorp.index
                sig = cls._get_one_sig(dest_base_rpath, index, src_rorp,
                                       dest_rorp)
                if sig:
                    cls.CCPP.flag_changed(index)
                    yield sig
Example #17
0
 def hardlink_rorp_eq(src_rorp, dest_rorp):
     Hardlink.add_rorp(dest_rorp)
     Hardlink.add_rorp(src_rorp, dest_rorp)
     rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
     if not src_rorp.isreg() or not dest_rorp.isreg(
     ) or src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1:
         if not rorp_eq:
             Log("Hardlink compare error with when no links exist exist", 3)
             Log(
                 "%s: %s" %
                 (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
             Log(
                 "%s: %s" %
                 (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
             return 0
     elif src_rorp.getnumlinks() > 1 and not Hardlink.islinked(src_rorp):
         if rorp_eq:
             Log(
                 "Hardlink compare error with first linked src_rorp and no dest_rorp sha1",
                 3)
             Log(
                 "%s: %s" %
                 (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
             Log(
                 "%s: %s" %
                 (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
             return 0
         hash.compute_sha1(dest_rorp)
         rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
         if src_rorp.getnumlinks() != dest_rorp.getnumlinks():
             if rorp_eq:
                 Log(
                     "Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with differing link counts",
                     3)
                 Log(
                     "%s: %s" %
                     (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
                 Log(
                     "%s: %s" %
                     (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)),
                     3)
                 return 0
         elif not rorp_eq:
             Log(
                 "Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with equal link counts",
                 3)
             Log(
                 "%s: %s" %
                 (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
             Log(
                 "%s: %s" %
                 (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
             return 0
     elif src_rorp.getnumlinks() != dest_rorp.getnumlinks():
         if rorp_eq:
             Log(
                 "Hardlink compare error with non-first linked src_rorp and with differing link counts",
                 3)
             Log(
                 "%s: %s" %
                 (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3)
             Log(
                 "%s: %s" %
                 (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
             return 0
     elif not rorp_eq:
         Log(
             "Hardlink compare error with non-first linked src_rorp and with equal link counts",
             3)
         Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)),
             3)
         Log(
             "%s: %s" %
             (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3)
         return 0
     Hardlink.del_rorp(src_rorp)
     Hardlink.del_rorp(dest_rorp)
     return 1
Example #18
0
def _hardlink_rorp_eq(src_rorp, dest_rorp):
    """Compare two files for hardlink equality, encompassing being hard-linked,
    having the same hashsum, and the same number of link counts."""
    Hardlink.add_rorp(dest_rorp)
    Hardlink.add_rorp(src_rorp, dest_rorp)
    rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
    if not src_rorp.isreg() or not dest_rorp.isreg() or src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1:
        if not rorp_eq:
            Log("Hardlink compare error with when no links exist", 3)
            Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
            Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
            return False
    elif src_rorp.getnumlinks() > 1 and not Hardlink.is_linked(src_rorp):
        if rorp_eq:
            Log("Hardlink compare error with first linked src_rorp and no dest_rorp sha1", 3)
            Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
            Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
            return False
        hash.compute_sha1(dest_rorp)
        rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp)
        if src_rorp.getnumlinks() != dest_rorp.getnumlinks():
            if rorp_eq:
                Log("Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with differing link counts", 3)
                Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
                Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
                return False
        elif not rorp_eq:
            Log("Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with equal link counts", 3)
            Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
            Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
            return False
    elif src_rorp.getnumlinks() != dest_rorp.getnumlinks():
        if rorp_eq:
            Log("Hardlink compare error with non-first linked src_rorp and with differing link counts", 3)
            Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
            Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
            return False
    elif not rorp_eq:
        Log("Hardlink compare error with non-first linked src_rorp and with equal link counts", 3)
        Log("%s: %s" % (src_rorp.index, Hardlink._get_inode_key(src_rorp)), 3)
        Log("%s: %s" % (dest_rorp.index, Hardlink._get_inode_key(dest_rorp)), 3)
        return False
    Hardlink.del_rorp(src_rorp)
    Hardlink.del_rorp(dest_rorp)
    return True
Example #19
0
	def postprocess(src_rorp, dest_rorp):
		"""After comparison, process src_rorp and dest_rorp"""
		if compare_hardlinks and src_rorp:
			Hardlink.del_rorp(src_rorp)
Example #20
0
	def preprocess(src_rorp, dest_rorp):
		"""Initially process src and dest_rorp"""
		if compare_hardlinks and src_rorp:
			Hardlink.add_rorp(src_rorp, dest_rorp)
Example #21
0
 def _patch_hardlink_to_temp(self, diff_rorp, new):
     """Hardlink diff_rorp to temp, update hash if necessary"""
     Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
     self.CCPP.update_hardlink_hash(diff_rorp)