def test_diff(self): """Test making a diff""" sel1 = selection.Select(Path("testfiles/dir1")) diffdir.write_block_iter(diffdir.SigTarBlockIter(sel1.set_iter()), "testfiles/output/dir1.sigtar") sigtar_fp = open("testfiles/output/dir1.sigtar", "rb") sel2 = selection.Select(Path("testfiles/dir2")) delta_tar = diffdir.DirDelta(sel2.set_iter(), sigtar_fp) diffdir.write_block_iter(delta_tar, "testfiles/output/dir1dir2.difftar") changed_files = ["diff/changeable_permission", "diff/regular_file", "snapshot/symbolic_link/", "deleted/deleted_file", "snapshot/directory_to_file", "snapshot/file_to_directory/"] for tarinfo in tarfile.TarFile("testfiles/output/dir1dir2.difftar", "r"): tiname = util.get_tarinfo_name(tarinfo) if tiname in changed_files: changed_files.remove(tiname) assert not changed_files, ("Following files not found:\n" + "\n".join(changed_files))
def test_diff2(self): """Another diff test - this one involves multivol support (requires rdiff to be installed to pass)""" sel1 = selection.Select(Path("testfiles/dir2")) diffdir.write_block_iter(diffdir.SigTarBlockIter(sel1.set_iter()), "testfiles/output/dir2.sigtar") sigtar_fp = open("testfiles/output/dir2.sigtar", "rb") sel2 = selection.Select(Path("testfiles/dir3")) delta_tar = diffdir.DirDelta(sel2.set_iter(), sigtar_fp) diffdir.write_block_iter(delta_tar, "testfiles/output/dir2dir3.difftar") buffer = "" tf = tarfile.TarFile("testfiles/output/dir2dir3.difftar", "r") for tarinfo in tf: if tarinfo.name.startswith("multivol_diff/"): buffer += tf.extractfile(tarinfo).read() assert 3000000 < len(buffer) < 4000000 fout = open("testfiles/output/largefile.delta", "wb") fout.write(buffer) fout.close() assert not os.system("rdiff patch testfiles/dir2/largefile " "testfiles/output/largefile.delta " "testfiles/output/largefile.patched") dir3large = open("testfiles/dir3/largefile", "rb").read() patchedlarge = open("testfiles/output/largefile.patched", "rb").read() assert dir3large == patchedlarge
def patch_diff_tarfile( base_path, diff_tarfile, restrict_index=() ): """Patch given Path object using delta tarfile (as in tarfile.TarFile) If restrict_index is set, ignore any deltas in diff_tarfile that don't start with restrict_index. """ if base_path.exists(): path_iter = selection.Select( base_path ).set_iter() else: path_iter = empty_iter() # probably untarring full backup diff_path_iter = difftar2path_iter( diff_tarfile ) if restrict_index: diff_path_iter = filter_path_iter( diff_path_iter, restrict_index ) collated = diffdir.collate2iters( path_iter, diff_path_iter ) ITR = IterTreeReducer( PathPatcher, [base_path] ) for basis_path, diff_ropath in collated: if basis_path: log.Info( _( "Patching %s" ) % ( basis_path.get_relative_path(), ), log.InfoCode.patch_file_patching, util.escape( basis_path.get_relative_path() ) ) ITR( basis_path.index, basis_path, diff_ropath ) else: log.Info( _( "Patching %s" ) % ( diff_ropath.get_relative_path(), ), log.InfoCode.patch_file_patching, util.escape( diff_ropath.get_relative_path() ) ) ITR( diff_ropath.index, basis_path, diff_ropath ) ITR.Finish() base_path.setdata()
def test_empty_diff(self): """Test producing a diff against same sig; should be len 0""" self.empty_diff_schema("testfiles/various_file_types") select = selection.Select(Path("testfiles/various_file_types")) select.set_iter() sigtar = diffdir.SigTarBlockIter(select) diffdir.write_block_iter(sigtar, "testfiles/output/sigtar") sigtar_fp = open("testfiles/output/sigtar") select2 = selection.Select(Path("testfiles/various_file_types")) select2.set_iter() diffdir.write_block_iter(diffdir.DirDelta(select2, sigtar_fp), "testfiles/output/difftar") size = os.stat("testfiles/output/difftar").st_size # @UnusedVariable
def test_empty_diff(self): u"""Test producing a diff against same sig; should be len 0""" self.empty_diff_schema(u"{0}/testfiles/various_file_types".format(_runtest_dir)) select = selection.Select(Path(u"{0}/testfiles/various_file_types".format(_runtest_dir))) select.set_iter() sigtar = diffdir.SigTarBlockIter(select) diffdir.write_block_iter(sigtar, u"{0}/testfiles/output/sigtar".format(_runtest_dir)) sigtar_fp = open(u"{0}/testfiles/output/sigtar".format(_runtest_dir), u"rb") select2 = selection.Select(Path(u"{0}/testfiles/various_file_types".format(_runtest_dir))) select2.set_iter() diffdir.write_block_iter(diffdir.DirDelta(select2, sigtar_fp), u"{0}/testfiles/output/difftar".format(_runtest_dir)) size = os.stat(u"{0}/testfiles/output/difftar".format(_runtest_dir)).st_size
def deltree(self): """Remove self by recursively deleting files under it""" from duplicity import selection # todo: avoid circ. dep. issue log.Info(_("Deleting tree %s") % util.ufn(self.name)) itr = IterTreeReducer(PathDeleter, []) for path in selection.Select(self).set_iter(): itr(path.index, path) itr.Finish() self.setdata()
def empty_diff_schema(self, dirname): """Given directory name, make sure can tell when nothing changes""" select = selection.Select(Path(dirname)) select.set_iter() sigtar = diffdir.SigTarBlockIter(select) diffdir.write_block_iter(sigtar, "testfiles/output/sigtar") sigtar_fp = open("testfiles/output/sigtar") select2 = selection.Select(Path(dirname)) select2.set_iter() diffdir.write_block_iter(diffdir.DirDelta(select2, sigtar_fp), "testfiles/output/difftar") size = os.stat("testfiles/output/difftar").st_size assert size == 0 or size == 10240, size # 10240 is size of one record if size != 0: fin = open("testfiles/output/difftar", "rb") diff_buf = fin.read() assert not fin.close() assert diff_buf == '\0' * 10240
def testsig(self): """Test producing tar signature of various file types""" select = selection.Select(Path("testfiles/various_file_types")) select.set_iter() sigtar = diffdir.SigTarBlockIter(select) diffdir.write_block_iter(sigtar, "testfiles/output/sigtar") i = 0 for tarinfo in tarfile.TarFile("testfiles/output/sigtar", "r"): # @UnusedVariable i += 1 assert i >= 5, "There should be at least 5 files in sigtar"
def testsig(self): u"""Test producing tar signature of various file types""" select = selection.Select(Path(u"{0}/testfiles/various_file_types".format(_runtest_dir))) select.set_iter() sigtar = diffdir.SigTarBlockIter(select) diffdir.write_block_iter(sigtar, u"{0}/testfiles/output/sigtar".format(_runtest_dir)) i = 0 for tarinfo in tarfile.TarFile(u"{0}/testfiles/output/sigtar".format(_runtest_dir), u"r"): i += 1 assert i >= 5, u"There should be at least 5 files in sigtar"
def total_sequence(self, filelist): """Test signatures, diffing, and patching on directory list""" assert len(filelist) >= 2 self.deltmp() assert not os.system("cp -pR %s testfiles/output/sequence" % (filelist[0], )) seq_path = Path("testfiles/output/sequence") sig = Path("testfiles/output/sig.tar") diff = Path("testfiles/output/diff.tar") for dirname in filelist[1:]: new_path = Path(dirname) diffdir.write_block_iter( diffdir.DirSig(selection.Select(seq_path).set_iter()), sig) diffdir.write_block_iter( diffdir.DirDelta( selection.Select(new_path).set_iter(), sig.open("rb")), diff) patchdir.Patch(seq_path, diff.open("rb")) assert seq_path.compare_recursive(new_path, 1)
def test_dirdelta_write_sig(self): """Test simultaneous delta and sig generation Generate signatures and deltas of dirs1, 2, 3, 4 and compare those produced by DirDelta_WriteSig and other methods. """ deltadir1 = Path("testfiles/output/dir.deltatar1") # @UnusedVariable deltadir2 = Path("testfiles/output/dir.deltatar2") # @UnusedVariable cur_full_sigs = Path("testfiles/output/fullsig.dir1") cur_dir = Path("testfiles/dir1") get_sel = lambda cur_dir: selection.Select(cur_dir).set_iter() diffdir.write_block_iter(diffdir.SigTarBlockIter(get_sel(cur_dir)), cur_full_sigs) sigstack = [cur_full_sigs] for dirname in ['dir2', 'dir3', 'dir4']: # print "Processing ", dirname old_dir = cur_dir # @UnusedVariable cur_dir = Path("testfiles/" + dirname) old_full_sigs = cur_full_sigs cur_full_sigs = Path("testfiles/output/fullsig." + dirname) delta1 = Path("testfiles/output/delta1." + dirname) delta2 = Path("testfiles/output/delta2." + dirname) incsig = Path("testfiles/output/incsig." + dirname) # Write old-style delta to deltadir1 diffdir.write_block_iter(diffdir.DirDelta(get_sel(cur_dir), old_full_sigs.open("rb")), delta1) # Write new signature and delta to deltadir2 and sigdir2, compare block_iter = diffdir.DirDelta_WriteSig( get_sel(cur_dir), [p.open("rb") for p in sigstack], incsig.open("wb")) sigstack.append(incsig) diffdir.write_block_iter(block_iter, delta2) # print delta1.name, delta2.name compare_tar(delta1.open("rb"), delta2.open("rb")) assert not os.system("cmp %s %s" % (delta1.name, delta2.name)) # Write old-style signature to cur_full_sigs diffdir.write_block_iter(diffdir.SigTarBlockIter(get_sel(cur_dir)), cur_full_sigs)
def set_selection(): """Return selection iter starting at filename with arguments applied""" global select_opts, select_files sel = selection.Select(globals.local_path) sel.ParseArgs(select_opts, select_files) globals.select = sel.set_iter()
def get_sel(self, path): u"""Get selection iter over the given directory""" return selection.Select(path).set_iter()
def compare_recursive(self, other, verbose=None): """Compare self to other Path, descending down directories""" from duplicity import selection # todo: avoid circ. dep. issue selfsel = selection.Select(self).set_iter() othersel = selection.Select(other).set_iter() return Iter.equal(selfsel, othersel, verbose)