def test_optdict(): with no_lingering_errors(): d = options.OptDict({ 'x': ('x', False), 'y': ('y', False), 'z': ('z', False), 'other_thing': ('other_thing', False), 'no_other_thing': ('other_thing', True), 'no_z': ('z', True), 'no_smart': ('smart', True), 'smart': ('smart', False), 'stupid': ('smart', True), 'no_smart': ('smart', False), }) WVPASS('foo') d['x'] = 5 d['y'] = 4 d['z'] = 99 d['no_other_thing'] = 5 WVPASSEQ(d.x, 5) WVPASSEQ(d.y, 4) WVPASSEQ(d.z, 99) WVPASSEQ(d.no_z, False) WVPASSEQ(d.no_other_thing, True) WVEXCEPT(KeyError, lambda: d.p)
def test_item_mode(): with no_lingering_errors(): mode = S_IFDIR | 0o755 meta = metadata.from_path('.') oid = '\0' * 20 wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode))) wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
def test_check_repo_or_die(): with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir: os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" orig_cwd = os.getcwd() try: os.chdir(tmpdir) git.init_repo(bupdir) git.check_repo_or_die() WVPASS('check_repo_or_die') # if we reach this point the call above passed os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp') open(bupdir + '/objects/pack', 'w').close() try: git.check_repo_or_die() except SystemExit as e: WVPASSEQ(e.code, 14) else: WVFAIL() os.unlink(bupdir + '/objects/pack') os.rename(bupdir + '/objects/pack.tmp', bupdir + '/objects/pack') try: git.check_repo_or_die('nonexistantbup.tmp') except SystemExit as e: WVPASSEQ(e.code, 15) else: WVFAIL() finally: os.chdir(orig_cwd)
def test_options(): with no_lingering_errors(): o = options.Options(optspec) (opt,flags,extra) = o.parse(['-tttqp', 7, '--longoption', '19', 'hanky', '--onlylong', '-7']) WVPASSEQ(flags[0], ('-t', '')) WVPASSEQ(flags[1], ('-t', '')) WVPASSEQ(flags[2], ('-t', '')) WVPASSEQ(flags[3], ('-q', '')) WVPASSEQ(flags[4], ('-p', 7)) WVPASSEQ(flags[5], ('--longoption', '19')) WVPASSEQ(extra, ['hanky']) WVPASSEQ((opt.t, opt.q, opt.p, opt.l, opt.onlylong, opt.neveropt), (3,1,7,19,1,None)) WVPASSEQ((opt.deftest1, opt.deftest2, opt.deftest3, opt.deftest4, opt.deftest5), (1,2,None,None,'[square')) WVPASSEQ((opt.stupid, opt.no_stupid), (True, None)) WVPASSEQ((opt.smart, opt.no_smart), (None, True)) WVPASSEQ((opt.x, opt.extended, opt.no_simple), (2,2,2)) WVPASSEQ((opt.no_x, opt.no_extended, opt.simple), (False,False,False)) WVPASSEQ(opt['#'], 7) WVPASSEQ(opt.compress, 7) (opt,flags,extra) = o.parse(['--onlylong', '-t', '--no-onlylong', '--smart', '--simple']) WVPASSEQ((opt.t, opt.q, opt.onlylong), (1, None, 0)) WVPASSEQ((opt.stupid, opt.no_stupid), (False, True)) WVPASSEQ((opt.smart, opt.no_smart), (True, False)) WVPASSEQ((opt.x, opt.extended, opt.no_simple), (0,0,0)) WVPASSEQ((opt.no_x, opt.no_extended, opt.simple), (True,True,True))
def test_shquote(): with no_lingering_errors(): WVPASSEQ(qst(""" this is basic \t\n\r text """), ["this", "is", "basic", "text"]) WVPASSEQ(qst(r""" \"x\" "help" 'yelp' """), ['"x"', "help", "yelp"]) WVPASSEQ(qst(r""" "'\"\"'" '\"\'' """), ["'\"\"'", "\\\"'"]) WVPASSEQ(shquote.quotesplit(' this is "unfinished'), [(2, "this"), (7, "is"), (10, "unfinished")]) WVPASSEQ(shquote.quotesplit('"silly"\'will'), [(0, "silly"), (7, "will")]) WVPASSEQ(shquote.unfinished_word('this is a "billy" "goat'), ('"', "goat")) WVPASSEQ(shquote.unfinished_word("'x"), ("'", "x")) WVPASSEQ(shquote.unfinished_word("abra cadabra "), (None, "")) WVPASSEQ(shquote.unfinished_word("abra cadabra"), (None, "cadabra")) (qtype, word) = shquote.unfinished_word("this is /usr/loc") WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", True), "al") (qtype, word) = shquote.unfinished_word("this is '/usr/loc") WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", True), "al'") (qtype, word) = shquote.unfinished_word('this is "/usr/loc') WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", True), 'al"') (qtype, word) = shquote.unfinished_word('this is "/usr/loc') WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", False), "al") (qtype, word) = shquote.unfinished_word('this is \\ hammer\\ "') WVPASSEQ(word, ' hammer "') WVPASSEQ(shquote.what_to_add(qtype, word, ' hammer "time"', True), 'time\\"') WVPASSEQ( shquote.quotify_list(["a", "", '"word"', "'third'", "'", "x y"]), "a '' '\"word\"' \"'third'\" \"'\" 'x y'" )
def test_multiple_suggestions(): with no_lingering_errors(): with test_tempdir('bup-tclient-') as tmpdir: os.environ['BUP_MAIN_EXE'] = '../../../bup' os.environ['BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) lw = git.PackWriter() lw.new_blob(s1) lw.close() lw = git.PackWriter() lw.new_blob(s2) lw.close() WVPASSEQ(len(glob.glob(git.repo('objects/pack'+IDX_PAT))), 2) c = client.Client(bupdir, create=True) WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 0) rw = c.new_packwriter() s1sha = rw.new_blob(s1) WVPASS(rw.exists(s1sha)) s2sha = rw.new_blob(s2) # This is a little hacky, but ensures that we test the # code under test while (len(glob.glob(c.cachedir+IDX_PAT)) < 2 and not c.conn.has_input()): pass rw.new_blob(s2) WVPASS(rw.objcache.exists(s1sha)) WVPASS(rw.objcache.exists(s2sha)) rw.new_blob(s3) WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 2) rw.close() WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 3)
def test_uncache_ours_upto(): history = [] def mock_fadvise_pages_done(f, ofs, len): history.append((f, ofs, len)) with no_lingering_errors(): uncache_upto = hashsplit._uncache_ours_upto page_size = helpers.sc_page_size orig_pages_done = hashsplit._fadvise_pages_done try: hashsplit._fadvise_pages_done = mock_fadvise_pages_done history = [] uncache_upto(42, 0, (0, 1), iter([])) WVPASSEQ([], history) uncache_upto(42, page_size, (0, 1), iter([])) WVPASSEQ([(42, 0, 1)], history) history = [] uncache_upto(42, page_size, (0, 3), iter([(5, 2)])) WVPASSEQ([], history) uncache_upto(42, 2 * page_size, (0, 3), iter([(5, 2)])) WVPASSEQ([], history) uncache_upto(42, 3 * page_size, (0, 3), iter([(5, 2)])) WVPASSEQ([(42, 0, 3)], history) history = [] uncache_upto(42, 5 * page_size, (0, 3), iter([(5, 2)])) WVPASSEQ([(42, 0, 3)], history) history = [] uncache_upto(42, 6 * page_size, (0, 3), iter([(5, 2)])) WVPASSEQ([(42, 0, 3)], history) history = [] uncache_upto(42, 7 * page_size, (0, 3), iter([(5, 2)])) WVPASSEQ([(42, 0, 3), (42, 5, 2)], history) finally: hashsplit._fadvise_pages_done = orig_pages_done
def test_grafted_path_components(): with no_lingering_errors(): WVPASSEQ(grafted_path_components([('/chroot', '/')], '/foo'), [('', '/'), ('foo', '/foo')]) WVPASSEQ(grafted_path_components([('/foo/bar', '/')], '/foo/bar/baz/bax'), [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ('bax', '/foo/bar/baz/bax')]) WVPASSEQ(grafted_path_components([('/foo/bar/baz', '/bax')], '/foo/bar/baz/1/2'), [('', None), ('bax', '/foo/bar/baz'), ('1', '/foo/bar/baz/1'), ('2', '/foo/bar/baz/1/2')]) WVPASSEQ(grafted_path_components([('/foo', '/bar/baz/bax')], '/foo/bar'), [('', None), ('bar', None), ('baz', None), ('bax', '/foo'), ('bar', '/foo/bar')]) WVPASSEQ(grafted_path_components([('/foo/bar/baz', '/a/b/c')], '/foo/bar/baz'), [('', None), ('a', None), ('b', None), ('c', '/foo/bar/baz')]) WVPASSEQ(grafted_path_components([('/', '/a/b/c/')], '/foo/bar'), [('', None), ('a', None), ('b', None), ('c', '/'), ('foo', '/foo'), ('bar', '/foo/bar')]) WVEXCEPT(Exception, grafted_path_components, 'foo', [])
def test_duplicate_save_dates(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir environ['TZ'] = 'UTC' git.repodir = bup_dir data_path = tmpdir + '/src' os.mkdir(data_path) with open(data_path + '/file', 'w+') as tmpfile: tmpfile.write(b'canary\n') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) for i in range(11): ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path)) repo = LocalRepo() res = vfs.resolve(repo, '/test') wvpasseq(2, len(res)) name, revlist = res[-1] wvpasseq('test', name) wvpasseq(('.', '1970-01-02-034640-00', '1970-01-02-034640-01', '1970-01-02-034640-02', '1970-01-02-034640-03', '1970-01-02-034640-04', '1970-01-02-034640-05', '1970-01-02-034640-06', '1970-01-02-034640-07', '1970-01-02-034640-08', '1970-01-02-034640-09', '1970-01-02-034640-10', 'latest'), tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
def testmangle(): with no_lingering_errors(): afile = 0100644 afile2 = 0100770 alink = 0120000 adir = 0040000 adir2 = 0040777 WVPASSEQ(git.mangle_name("a", adir2, adir), "a") WVPASSEQ(git.mangle_name(".bup", adir2, adir), ".bup.bupl") WVPASSEQ(git.mangle_name("a.bupa", adir2, adir), "a.bupa.bupl") WVPASSEQ(git.mangle_name("b.bup", alink, alink), "b.bup.bupl") WVPASSEQ(git.mangle_name("b.bu", alink, alink), "b.bu") WVPASSEQ(git.mangle_name("f", afile, afile2), "f") WVPASSEQ(git.mangle_name("f.bup", afile, afile2), "f.bup.bupl") WVPASSEQ(git.mangle_name("f.bup", afile, adir), "f.bup.bup") WVPASSEQ(git.mangle_name("f", afile, adir), "f.bup") WVPASSEQ(git.demangle_name("f.bup", afile), ("f", git.BUP_CHUNKED)) WVPASSEQ(git.demangle_name("f.bupl", afile), ("f", git.BUP_NORMAL)) WVPASSEQ(git.demangle_name("f.bup.bupl", afile), ("f.bup", git.BUP_NORMAL)) WVPASSEQ(git.demangle_name(".bupm", afile), ('', git.BUP_NORMAL)) WVPASSEQ(git.demangle_name(".bupm", adir), ('', git.BUP_CHUNKED)) # for safety, we ignore .bup? suffixes we don't recognize. Future # versions might implement a .bup[a-z] extension as something other # than BUP_NORMAL. WVPASSEQ(git.demangle_name("f.bupa", afile), ("f.bupa", git.BUP_NORMAL))
def test_metadata_method(): with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir: bup_dir = tmpdir + '/bup' data_path = tmpdir + '/foo' os.mkdir(data_path) ex('touch', data_path + '/file') ex('ln', '-s', 'file', data_path + '/symlink') test_time1 = 13 * 1000000000 test_time2 = 42 * 1000000000 utime(data_path + '/file', (0, test_time1)) lutime(data_path + '/symlink', (0, 0)) utime(data_path, (0, test_time2)) ex(bup_path, '-d', bup_dir, 'init') ex(bup_path, '-d', bup_dir, 'index', '-v', data_path) ex(bup_path, '-d', bup_dir, 'save', '-tvvn', 'test', data_path) git.check_repo_or_die(bup_dir) top = vfs.RefList(None) n = top.lresolve('/test/latest' + resolve_parent(data_path)) m = n.metadata() WVPASS(m.mtime == test_time2) WVPASS(len(n.subs()) == 2) WVPASS(n.name == 'foo') WVPASS(set([x.name for x in n.subs()]) == set(['file', 'symlink'])) for sub in n: if sub.name == 'file': m = sub.metadata() WVPASS(m.mtime == test_time1) elif sub.name == 'symlink': m = sub.metadata() WVPASS(m.mtime == 0)
def test_restore_over_existing_target(): with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir: path = tmpdir + '/foo' os.mkdir(path) dir_m = metadata.from_path(path, archive_path=path, save_symlinks=True) os.rmdir(path) open(path, 'w').close() file_m = metadata.from_path(path, archive_path=path, save_symlinks=True) # Restore dir over file. WVPASSEQ(dir_m.create_path(path, create_symlinks=True), None) WVPASS(stat.S_ISDIR(os.stat(path).st_mode)) # Restore dir over dir. WVPASSEQ(dir_m.create_path(path, create_symlinks=True), None) WVPASS(stat.S_ISDIR(os.stat(path).st_mode)) # Restore file over dir. WVPASSEQ(file_m.create_path(path, create_symlinks=True), None) WVPASS(stat.S_ISREG(os.stat(path).st_mode)) # Restore file over file. WVPASSEQ(file_m.create_path(path, create_symlinks=True), None) WVPASS(stat.S_ISREG(os.stat(path).st_mode)) # Restore file over non-empty dir. os.remove(path) os.mkdir(path) open(path + '/bar', 'w').close() WVEXCEPT(Exception, file_m.create_path, path, create_symlinks=True) # Restore dir over non-empty dir. os.remove(path + '/bar') os.mkdir(path + '/bar') WVEXCEPT(Exception, dir_m.create_path, path, create_symlinks=True)
def test_utc_offset_str(): with no_lingering_errors(): tz = os.environ.get('TZ') try: os.environ['TZ'] = 'FOO+0:00' WVPASSEQ(utc_offset_str(0), '+0000') os.environ['TZ'] = 'FOO+1:00' WVPASSEQ(utc_offset_str(0), '-0100') os.environ['TZ'] = 'FOO-1:00' WVPASSEQ(utc_offset_str(0), '+0100') os.environ['TZ'] = 'FOO+3:3' WVPASSEQ(utc_offset_str(0), '-0303') os.environ['TZ'] = 'FOO-3:3' WVPASSEQ(utc_offset_str(0), '+0303') # Offset is not an integer number of minutes os.environ['TZ'] = 'FOO+3:3:3' WVPASSEQ(utc_offset_str(1), '-0303') os.environ['TZ'] = 'FOO-3:3:3' WVPASSEQ(utc_offset_str(1), '+0303') WVPASSEQ(utc_offset_str(314159), '+0303') finally: if tz: os.environ['TZ'] = tz else: try: del os.environ['TZ'] except KeyError: pass
def test_batchpipe(): with no_lingering_errors(): for chunk in batchpipe(['echo'], []): WVPASS(False) out = '' for chunk in batchpipe(['echo'], ['42']): out += chunk WVPASSEQ(out, '42\n') try: batchpipe(['bash', '-c'], ['exit 42']) except Exception as ex: WVPASSEQ(str(ex), "subprocess 'bash -c exit 42' failed with status 42") args = [str(x) for x in range(6)] # Force batchpipe to break the args into batches of 3. This # approach assumes all args are the same length. arg_max = \ helpers._argmax_base(['echo']) + helpers._argmax_args_size(args[:3]) batches = batchpipe(['echo'], args, arg_max=arg_max) WVPASSEQ(next(batches), '0 1 2\n') WVPASSEQ(next(batches), '3 4 5\n') WVPASSEQ(next(batches, None), None) batches = batchpipe(['echo'], [str(x) for x in range(5)], arg_max=arg_max) WVPASSEQ(next(batches), '0 1 2\n') WVPASSEQ(next(batches), '3 4\n') WVPASSEQ(next(batches, None), None)
def test_apply_to_path_restricted_access(): if is_superuser() or detect_fakeroot(): return if sys.platform.startswith('cygwin'): return # chmod 000 isn't effective. with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir: parent = tmpdir + '/foo' path = parent + '/bar' os.mkdir(parent) os.mkdir(path) clear_errors() m = metadata.from_path(path, archive_path=path, save_symlinks=True) WVPASSEQ(m.path, path) os.chmod(parent, 000) m.apply_to_path(path) print >> sys.stderr, 'saved_errors:', helpers.saved_errors expected_errors = ['utime: '] if m.linux_attr and _linux_attr_supported(tmpdir): expected_errors.append('Linux chattr: ') if metadata.xattr and m.linux_xattr: expected_errors.append("xattr.set '") WVPASS(len(helpers.saved_errors) == len(expected_errors)) for i in xrange(len(expected_errors)): WVPASS(str(helpers.saved_errors[i]).startswith(expected_errors[i])) clear_errors()
def test_long_index(): with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir: os.environ['BUP_MAIN_EXE'] = bup_exe os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" git.init_repo(bupdir) w = git.PackWriter() obj_bin = struct.pack('!IIIII', 0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899) obj2_bin = struct.pack('!IIIII', 0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900) obj3_bin = struct.pack('!IIIII', 0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011) pack_bin = struct.pack('!IIIII', 0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100) idx = list(list() for i in xrange(256)) idx[0].append((obj_bin, 1, 0xfffffffff)) idx[0x11].append((obj2_bin, 2, 0xffffffffff)) idx[0x22].append((obj3_bin, 3, 0xff)) w.count = 3 name = tmpdir + '/tmp.idx' r = w._write_pack_idx_v2(name, idx, pack_bin) i = git.PackIdxV2(name, open(name, 'rb')) WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff) WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff) WVPASSEQ(i.find_offset(obj3_bin), 0xff)
def test_new_commit(): with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir: os.environ['BUP_MAIN_EXE'] = bup_exe os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" git.init_repo(bupdir) git.verbose = 1 w = git.PackWriter() tree = os.urandom(20) parent = os.urandom(20) author_name = 'Author' author_mail = 'author@somewhere' adate_sec = 1439657836 cdate_sec = adate_sec + 1 committer_name = 'Committer' committer_mail = 'committer@somewhere' adate_tz_sec = cdate_tz_sec = None commit = w.new_commit(tree, parent, '%s <%s>' % (author_name, author_mail), adate_sec, adate_tz_sec, '%s <%s>' % (committer_name, committer_mail), cdate_sec, cdate_tz_sec, 'There is a small mailbox here') adate_tz_sec = -60 * 60 cdate_tz_sec = 120 * 60 commit_off = w.new_commit(tree, parent, '%s <%s>' % (author_name, author_mail), adate_sec, adate_tz_sec, '%s <%s>' % (committer_name, committer_mail), cdate_sec, cdate_tz_sec, 'There is a small mailbox here') w.close() commit_items = git.get_commit_items(commit.encode('hex'), git.cp()) local_author_offset = localtime(adate_sec).tm_gmtoff local_committer_offset = localtime(cdate_sec).tm_gmtoff WVPASSEQ(tree, commit_items.tree.decode('hex')) WVPASSEQ(1, len(commit_items.parents)) WVPASSEQ(parent, commit_items.parents[0].decode('hex')) WVPASSEQ(author_name, commit_items.author_name) WVPASSEQ(author_mail, commit_items.author_mail) WVPASSEQ(adate_sec, commit_items.author_sec) WVPASSEQ(local_author_offset, commit_items.author_offset) WVPASSEQ(committer_name, commit_items.committer_name) WVPASSEQ(committer_mail, commit_items.committer_mail) WVPASSEQ(cdate_sec, commit_items.committer_sec) WVPASSEQ(local_committer_offset, commit_items.committer_offset) commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp()) WVPASSEQ(tree, commit_items.tree.decode('hex')) WVPASSEQ(1, len(commit_items.parents)) WVPASSEQ(parent, commit_items.parents[0].decode('hex')) WVPASSEQ(author_name, commit_items.author_name) WVPASSEQ(author_mail, commit_items.author_mail) WVPASSEQ(adate_sec, commit_items.author_sec) WVPASSEQ(adate_tz_sec, commit_items.author_offset) WVPASSEQ(committer_name, commit_items.committer_name) WVPASSEQ(committer_mail, commit_items.committer_mail) WVPASSEQ(cdate_sec, commit_items.committer_sec) WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def test_nonresident_page_regions(): with no_lingering_errors(): WVPASSEQ(nr_regions([]), []) WVPASSEQ(nr_regions([1]), []) WVPASSEQ(nr_regions([0]), [(0, 1)]) WVPASSEQ(nr_regions([1, 0]), [(1, 1)]) WVPASSEQ(nr_regions([0, 0]), [(0, 2)]) WVPASSEQ(nr_regions([1, 0, 1]), [(1, 1)]) WVPASSEQ(nr_regions([1, 0, 0]), [(1, 2)]) WVPASSEQ(nr_regions([0, 1, 0]), [(0, 1), (2, 1)]) WVPASSEQ(nr_regions([0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0]), [(0, 2), (5, 3), (9, 2)]) WVPASSEQ(nr_regions([2, 42, 3, 101]), [(0, 2)]) # Test limit WVPASSEQ(nr_regions([0, 0, 0], None), [(0, 3)]) WVPASSEQ(nr_regions([0, 0, 0], 1), [(0, 1), (1, 1), (2, 1)]) WVPASSEQ(nr_regions([0, 0, 0], 2), [(0, 2), (2, 1)]) WVPASSEQ(nr_regions([0, 0, 0], 3), [(0, 3)]) WVPASSEQ(nr_regions([0, 0, 0], 4), [(0, 3)]) WVPASSEQ(nr_regions([0, 0, 1], None), [(0, 2)]) WVPASSEQ(nr_regions([0, 0, 1], 1), [(0, 1), (1, 1)]) WVPASSEQ(nr_regions([0, 0, 1], 2), [(0, 2)]) WVPASSEQ(nr_regions([0, 0, 1], 3), [(0, 2)]) WVPASSEQ(nr_regions([1, 0, 0], None), [(1, 2)]) WVPASSEQ(nr_regions([1, 0, 0], 1), [(1, 1), (2, 1)]) WVPASSEQ(nr_regions([1, 0, 0], 2), [(1, 2)]) WVPASSEQ(nr_regions([1, 0, 0], 3), [(1, 2)]) WVPASSEQ(nr_regions([1, 0, 0, 0, 1], None), [(1, 3)]) WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 1), [(1, 1), (2, 1), (3, 1)]) WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 2), [(1, 2), (3, 1)]) WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 3), [(1, 3)]) WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 4), [(1, 3)])
def test_item_read_write(): with no_lingering_errors(): x = vfs.Root(meta=13) stream = BytesIO() vfs.write_item(stream, x) print('stream:', repr(stream.getvalue()), stream.tell(), file=sys.stderr) stream.seek(0) wvpasseq(x, vfs.read_item(stream))
def test_path_components(): with no_lingering_errors(): WVPASSEQ(path_components('/'), [('', '/')]) WVPASSEQ(path_components('/foo'), [('', '/'), ('foo', '/foo')]) WVPASSEQ(path_components('/foo/'), [('', '/'), ('foo', '/foo')]) WVPASSEQ(path_components('/foo/bar'), [('', '/'), ('foo', '/foo'), ('bar', '/foo/bar')]) WVEXCEPT(Exception, path_components, 'foo')
def test_vint(): with no_lingering_errors(): values = (0, 1, 42, 64, 10**16) for x in values: WVPASSEQ(encode_and_decode_vint(x), x) for x in [-x for x in values]: WVPASSEQ(encode_and_decode_vint(x), x) WVEXCEPT(EOFError, vint.read_vint, BytesIO())
def test_parse_num(): with no_lingering_errors(): pn = parse_num WVPASSEQ(pn('1'), 1) WVPASSEQ(pn('0'), 0) WVPASSEQ(pn('1.5k'), 1536) WVPASSEQ(pn('2 gb'), 2*1024*1024*1024) WVPASSEQ(pn('1e+9 k'), 1000000000 * 1024) WVPASSEQ(pn('-3e-3mb'), int(-0.003 * 1024 * 1024))
def test_fallback_next(): with no_lingering_errors(): global next orig = next next = helpers._fallback_next try: test_next() finally: next = orig
def test_readpipe(): with no_lingering_errors(): x = readpipe(['echo', '42']) WVPASSEQ(x, '42\n') try: readpipe(['bash', '-c', 'exit 42']) except Exception as ex: WVPASSEQ(str(ex), "subprocess 'bash -c exit 42' failed with status 42")
def test_bloom(): with no_lingering_errors(): with test_tempdir('bup-tbloom-') as tmpdir: hashes = [os.urandom(20) for i in range(100)] class Idx: pass ix = Idx() ix.name = 'dummy.idx' ix.shatable = ''.join(hashes) for k in (4, 5): b = bloom.create(tmpdir + '/pybuptest.bloom', expected=100, k=k) b.add_idx(ix) WVPASSLT(b.pfalse_positive(), .1) b.close() b = bloom.ShaBloom(tmpdir + '/pybuptest.bloom') all_present = True for h in hashes: all_present &= b.exists(h) WVPASS(all_present) false_positives = 0 for h in [os.urandom(20) for i in range(1000)]: if b.exists(h): false_positives += 1 WVPASSLT(false_positives, 5) os.unlink(tmpdir + '/pybuptest.bloom') tf = tempfile.TemporaryFile(dir=tmpdir) b = bloom.create('bup.bloom', f=tf, expected=100) WVPASSEQ(b.rwfile, tf) WVPASSEQ(b.k, 5) # Test large (~1GiB) filter. This may fail on s390 (31-bit # architecture), and anywhere else where the address space is # sufficiently limited. tf = tempfile.TemporaryFile(dir=tmpdir) skip_test = False try: b = bloom.create('bup.bloom', f=tf, expected=2**28, delaywrite=False) except EnvironmentError as ex: (ptr_width, linkage) = platform.architecture() if ptr_width == '32bit' and ex.errno == errno.ENOMEM: WVMSG( 'skipping large bloom filter test (mmap probably failed) ' + str(ex)) skip_test = True else: raise if not skip_test: WVPASSEQ(b.k, 4)
def test_misc(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir git.repodir = bup_dir data_path = tmpdir + '/src' os.mkdir(data_path) with open(data_path + '/file', 'w+') as tmpfile: tmpfile.write(b'canary\n') symlink('file', data_path + '/symlink') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', data_path)) repo = LocalRepo() wvstart('readlink') ls_tree = exo(('git', 'ls-tree', 'test', 'symlink')).out mode, typ, oidx, name = ls_tree.strip().split(None, 3) assert name == 'symlink' link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8)) wvpasseq('file', vfs.readlink(repo, link_item)) ls_tree = exo(('git', 'ls-tree', 'test', 'file')).out mode, typ, oidx, name = ls_tree.strip().split(None, 3) assert name == 'file' file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8)) wvexcept(Exception, vfs.readlink, repo, file_item) wvstart('item_size') wvpasseq(4, vfs.item_size(repo, link_item)) wvpasseq(7, vfs.item_size(repo, file_item)) meta = metadata.from_path(__file__) meta.size = 42 fake_item = file_item._replace(meta=meta) wvpasseq(42, vfs.item_size(repo, fake_item)) wvstart('augment_item_meta') run_augment_item_meta_tests(repo, '/test/latest/file', 7, '/test/latest/symlink', 'file') wvstart('copy_item') # FIXME: this caused StopIteration #_, file_item = vfs.resolve(repo, '/file')[-1] _, file_item = vfs.resolve(repo, '/test/latest/file')[-1] file_copy = vfs.copy_item(file_item) wvpass(file_copy is not file_item) wvpass(file_copy.meta is not file_item.meta) wvpass(isinstance(file_copy, tuple)) wvpass(file_item.meta.user) wvpass(file_copy.meta.user) file_copy.meta.user = None wvpass(file_item.meta.user)
def test_midx_close(): fddir = b'/proc/self/fd' try: os.listdir(fddir) except Exception: # not supported, not Linux, I guess return def openfiles(): for fd in os.listdir(fddir): try: yield os.readlink(os.path.join(fddir, fd)) except OSError: pass def force_midx(objdir): args = [path.exe(), b'midx', b'--auto', b'--dir', objdir] check_call(args) with no_lingering_errors(), \ test_tempdir(b'bup-tgit-') as tmpdir: environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup' git.init_repo(bupdir) # create a few dummy idxes for i in range(10): _create_idx(tmpdir, i) git.auto_midx(tmpdir) l = git.PackIdxList(tmpdir) # this doesn't exist (yet) WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0))) for i in range(10, 15): _create_idx(tmpdir, i) # delete the midx ... # TODO: why do we need to? git.auto_midx() below doesn't?! for fn in os.listdir(tmpdir): if fn.endswith(b'.midx'): os.unlink(os.path.join(tmpdir, fn)) # and make a new one git.auto_midx(tmpdir) # check it still doesn't exist - we haven't refreshed WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0))) # check that we still have the midx open, this really # just checks more for the kernel API ('deleted' string) for fn in openfiles(): if not b'midx-' in fn: continue WVPASSEQ(True, b'deleted' in fn) # refresh the PackIdxList l.refresh() # and check that an object in pack 10 exists now WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0))) for fn in openfiles(): if not b'midx-' in fn: continue # check that we don't have it open anymore WVPASSEQ(False, b'deleted' in fn)
def prep_and_test_repo(name, create_repo, test_repo): with no_lingering_errors(): with test_tempdir(b'bup-t' + name) as tmpdir: bup_dir = tmpdir + b'/bup' environ[b'GIT_DIR'] = bup_dir environ[b'BUP_DIR'] = bup_dir ex((bup_path, b'init')) git.repodir = bup_dir with create_repo(bup_dir) as repo: test_repo(repo, tmpdir)
def test_parse_num(): with no_lingering_errors(): pn = parse_num WVPASSEQ(pn(b'1'), 1) WVPASSEQ(pn('1'), 1) WVPASSEQ(pn('0'), 0) WVPASSEQ(pn('1.5k'), 1536) WVPASSEQ(pn('2 gb'), 2 * 1024 * 1024 * 1024) WVPASSEQ(pn('1e+9 k'), 1000000000 * 1024) WVPASSEQ(pn('-3e-3mb'), int(-0.003 * 1024 * 1024))
def test_readpipe(): with no_lingering_errors(): x = readpipe([b'echo', b'42']) WVPASSEQ(x, b'42\n') try: readpipe([b'bash', b'-c', b'exit 42']) except Exception as ex: rx = '^subprocess b?"bash -c \'exit 42\'" failed with status 42$' if not re.match(rx, str(ex)): WVPASSEQ(str(ex), rx)
def test_misc(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir git.repodir = bup_dir data_path = tmpdir + '/src' os.mkdir(data_path) with open(data_path + '/file', 'w+') as tmpfile: tmpfile.write(b'canary\n') symlink('file', data_path + '/symlink') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', data_path)) repo = LocalRepo() wvstart('readlink') ls_tree = exo(('git', 'ls-tree', 'test', 'symlink')) mode, typ, oidx, name = ls_tree[0].strip().split(None, 3) assert name == 'symlink' link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8)) wvpasseq('file', vfs.readlink(repo, link_item)) ls_tree = exo(('git', 'ls-tree', 'test', 'file')) mode, typ, oidx, name = ls_tree[0].strip().split(None, 3) assert name == 'file' file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8)) wvexcept(Exception, vfs.readlink, repo, file_item) wvstart('item_size') wvpasseq(4, vfs.item_size(repo, link_item)) wvpasseq(7, vfs.item_size(repo, file_item)) meta = metadata.from_path(__file__) meta.size = 42 fake_item = file_item._replace(meta=meta) wvpasseq(42, vfs.item_size(repo, fake_item)) wvstart('augment_item_meta') run_augment_item_meta_tests(repo, '/test/latest/file', 7, '/test/latest/symlink', 'file') wvstart('copy_item') # FIXME: this caused StopIteration #_, file_item = vfs.resolve(repo, '/file')[-1] _, file_item = vfs.resolve(repo, '/test/latest/file')[-1] file_copy = vfs.copy_item(file_item) wvpass(file_copy is not file_item) wvpass(file_copy.meta is not file_item.meta) wvpass(isinstance(file_copy, tuple)) wvpass(file_item.meta.user) wvpass(file_copy.meta.user) file_copy.meta.user = None wvpass(file_item.meta.user)
def test_encrypted_container(): with no_lingering_errors(), create_test_config() as (tmpdir, store): secret = libnacl.public.SecretKey() p = encrypted.EncryptedContainer(store, b'test.pack', 'w', Kind.DATA, compression=9, key=secret.pk) p.finish() pfile = open(os.path.join(tmpdir, b'repo', b'test.pack'), 'rb') # minimum file size with header and footer wvpasseq(len(pfile.read()), 92) p = encrypted.EncryptedContainer(store, b'test2.pack', 'w', Kind.DATA, compression=9, key=secret.pk) offsets = {} offsets[b'A'] = p.write(3, None, b'A' * 1000) offsets[b'B'] = p.write(3, None, b'B' * 1000) offsets[b'C'] = p.write(3, None, b'C' * 1000) offsets[b'D'] = p.write(3, None, b'D' * 1000) offsets[b'ABCD'] = p.write(3, None, b'ABCD' * 250) sk = p.box.sk p.finish() pfile = open(os.path.join(tmpdir, b'repo', b'test2.pack'), 'rb') pdata = pfile.read() # the simple stuff above compresses well wvpasseq(len(pdata), 265) # check header wvpasseq(struct.unpack('<4sBBH', pdata[:8]), (b'BUPe', 1, 0, 84)) # check secret header eh = libnacl.sealed.SealedBox(secret).decrypt(pdata[8:84 + 8]) wvpasseq(struct.unpack('<BBBB', eh[:4]), (1, 1, 1, 1)) # ignore vuint_key here, it's random wvpasseq(sk, eh[4:]) # read the objects and check if they're fine p = encrypted.EncryptedContainer(store, b'test2.pack', 'r', Kind.DATA, key=secret) for k in sorted(offsets.keys()): wvpasseq(p.read(offsets[k]), (3, k * (1000 // len(k)))) p.close() # this does some extra checks - do it explicitly store.close()
def prep_and_test_repo(name, create_repo, test_repo): with no_lingering_errors(): with test_tempdir('bup-t' + name) as tmpdir: bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir environ['BUP_MAIN_EXE'] = bup_path ex((bup_path, 'init')) git.repodir = bup_dir with create_repo(bup_dir) as repo: test_repo(repo, tmpdir)
def index_basic(): with no_lingering_errors(): cd = os.path.realpath('../../../t') WVPASS(cd) sd = os.path.realpath(cd + '/sampledata') WVPASSEQ(resolve_parent(cd + '/sampledata'), sd) WVPASSEQ(os.path.realpath(cd + '/sampledata/x'), sd + '/x') WVPASSEQ(os.path.realpath(cd + '/sampledata/var/abs-symlink'), sd + '/var/abs-symlink-target') WVPASSEQ(resolve_parent(cd + '/sampledata/var/abs-symlink'), sd + '/var/abs-symlink')
def test_abbreviate(): with no_lingering_errors(): l1 = [b"1234", b"1235", b"1236"] WVPASSEQ(tree._tree_names_abbreviate(l1), l1) l2 = [b"aaaa", b"bbbb", b"cccc"] WVPASSEQ(tree._tree_names_abbreviate(l2), [b'a', b'b', b'c']) l3 = [b".bupm"] WVPASSEQ(tree._tree_names_abbreviate(l3), [b'.b']) l4 = [b"..strange..name"] WVPASSEQ(tree._tree_names_abbreviate(l4), [b'..s']) l5 = [b"justone"] WVPASSEQ(tree._tree_names_abbreviate(l5), [b'j'])
def test_fanout_behaviour(): with no_lingering_errors(): global hashbits global fanout levels = lambda data: [(len(b), l) for b, l in hashsplit.hashsplit_iter( [BytesIO(data)], True, None, fanout=fanout)] def hslevels(data): global hashbits global fanout return [(len(b), l) for b, l in HashSplitter([BytesIO(data)], bits=hashbits, fanbits=int(math.log(fanout, 2)))] # This is a tuple of max blob size (4 << 13 bytes) and expected level (0) # Return tuple with split content and expected level (from table) def sb(pfx, n): global fanout needed = hashbits + int(math.log(fanout, 2)) * n # internal algorithm ignores one bit after the split bits, # adjust for that (if n > 0): if n: needed += 1 return (b'\x00' * pfx + split_test_objs[needed], n) def end(n): return (b'\x00' * n, 0) # check a given sequence is handled correctly def check(objs): # old API allows only hashbits == 13 if hashbits == 13: WVPASSEQ(levels(b''.join([x[0] for x in objs])), [(len(x[0]), x[1]) for x in objs]) WVPASSEQ(hslevels(b''.join([x[0] for x in objs])), [(len(x[0]), x[1]) for x in objs]) for hashbits in (13, 14, 15): max_blob = (b'\x00' * (4 << hashbits), 0) for fanout in (2, 4): # never split - just max blobs check([max_blob] * 4) check([sb(0, 0)]) check([max_blob, sb(1, 3), max_blob]) check([sb(13, 1)]) check([sb(13, 1), end(200)]) fanout = 2 check([sb(0, 1), sb(30, 2), sb(20, 0), sb(10, 5)]) check([sb(0, 1), sb(30, 2), sb(20, 0), sb(10, 5), end(10)])
def test_list_refs(): with no_lingering_errors(): with test_tempdir(b'bup-tgit-') as tmpdir: environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup' src = tmpdir + b'/src' mkdirp(src) with open(src + b'/1', 'wb+') as f: f.write(b'something\n') with open(src + b'/2', 'wb+') as f: f.write(b'something else\n') git.init_repo(bupdir) emptyset = frozenset() WVPASSEQ(frozenset(git.list_refs()), emptyset) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset) exc(bup_exe, b'index', src) exc(bup_exe, b'save', b'-n', b'src', b'--strip', src) src_hash = exo(b'git', b'--git-dir', bupdir, b'rev-parse', b'src').strip().split(b'\n') assert (len(src_hash) == 1) src_hash = unhexlify(src_hash[0]) tree_hash = unhexlify( exo(b'git', b'--git-dir', bupdir, b'rev-parse', b'src:').strip().split(b'\n')[0]) blob_hash = unhexlify( exo(b'git', b'--git-dir', bupdir, b'rev-parse', b'src:1').strip().split(b'\n')[0]) WVPASSEQ(frozenset(git.list_refs()), frozenset([(b'refs/heads/src', src_hash)])) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([(b'refs/heads/src', src_hash)])) exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src') WVPASSEQ( frozenset(git.list_refs()), frozenset([(b'refs/heads/src', src_hash), (b'refs/tags/commit-tag', src_hash)])) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), frozenset([(b'refs/tags/commit-tag', src_hash)])) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([(b'refs/heads/src', src_hash)])) exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:') exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1') os.unlink(bupdir + b'/refs/heads/src') expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash), (b'refs/tags/tree-tag', tree_hash), (b'refs/tags/blob-tag', blob_hash)]) WVPASSEQ(frozenset(git.list_refs()), expected_tags) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([])) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
def test_list_refs(): with no_lingering_errors(): with test_tempdir('bup-tgit-') as tmpdir: os.environ['BUP_MAIN_EXE'] = bup_exe os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" src = tmpdir + '/src' mkdirp(src) with open(src + '/1', 'w+') as f: print f, 'something' with open(src + '/2', 'w+') as f: print f, 'something else' git.init_repo(bupdir) emptyset = frozenset() WVPASSEQ(frozenset(git.list_refs()), emptyset) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset) exc(bup_exe, 'index', src) exc(bup_exe, 'save', '-n', 'src', '--strip', src) src_hash = exo('git', '--git-dir', bupdir, 'rev-parse', 'src').strip().split('\n') assert (len(src_hash) == 1) src_hash = src_hash[0].decode('hex') tree_hash = exo('git', '--git-dir', bupdir, 'rev-parse', 'src:').strip().split('\n')[0].decode('hex') blob_hash = exo('git', '--git-dir', bupdir, 'rev-parse', 'src:1').strip().split('\n')[0].decode('hex') WVPASSEQ(frozenset(git.list_refs()), frozenset([('refs/heads/src', src_hash)])) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([('refs/heads/src', src_hash)])) exc('git', '--git-dir', bupdir, 'tag', 'commit-tag', 'src') WVPASSEQ( frozenset(git.list_refs()), frozenset([('refs/heads/src', src_hash), ('refs/tags/commit-tag', src_hash)])) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), frozenset([('refs/tags/commit-tag', src_hash)])) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([('refs/heads/src', src_hash)])) exc('git', '--git-dir', bupdir, 'tag', 'tree-tag', 'src:') exc('git', '--git-dir', bupdir, 'tag', 'blob-tag', 'src:1') os.unlink(bupdir + '/refs/heads/src') expected_tags = frozenset([('refs/tags/commit-tag', src_hash), ('refs/tags/tree-tag', tree_hash), ('refs/tags/blob-tag', blob_hash)]) WVPASSEQ(frozenset(git.list_refs()), expected_tags) WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([])) WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
def test_bvec(): with no_lingering_errors(): values = (b'', b'x', b'foo', b'\0', b'\0foo', b'foo\0bar\0') for x in values: WVPASSEQ(encode_and_decode_bvec(x), x) WVEXCEPT(EOFError, vint.read_bvec, BytesIO()) outf = BytesIO() for x in (b'foo', b'bar', b'baz', b'bax'): vint.write_bvec(outf, x) inf = BytesIO(outf.getvalue()) WVPASSEQ(vint.read_bvec(inf), b'foo') WVPASSEQ(vint.read_bvec(inf), b'bar') vint.skip_bvec(inf) WVPASSEQ(vint.read_bvec(inf), b'bax')
def test_next(): with no_lingering_errors(): # Test whatever you end up with for next() after import '*'. WVPASSEQ(next(iter([]), None), None) x = iter([1]) WVPASSEQ(next(x, None), 1) WVPASSEQ(next(x, None), None) x = iter([1]) WVPASSEQ(next(x, 'x'), 1) WVPASSEQ(next(x, 'x'), 'x') WVEXCEPT(StopIteration, next, iter([])) x = iter([1]) WVPASSEQ(next(x), 1) WVEXCEPT(StopIteration, next, x)
def test_bvec(): with no_lingering_errors(): values = ('', 'x', 'foo', '\0', '\0foo', 'foo\0bar\0') for x in values: WVPASSEQ(encode_and_decode_bvec(x), x) WVEXCEPT(EOFError, vint.read_bvec, BytesIO()) outf = BytesIO() for x in ('foo', 'bar', 'baz', 'bax'): vint.write_bvec(outf, x) inf = BytesIO(outf.getvalue()) WVPASSEQ(vint.read_bvec(inf), 'foo') WVPASSEQ(vint.read_bvec(inf), 'bar') vint.skip_bvec(inf) WVPASSEQ(vint.read_bvec(inf), 'bax')
def test_bloom(): with no_lingering_errors(): with test_tempdir('bup-tbloom-') as tmpdir: hashes = [os.urandom(20) for i in range(100)] class Idx: pass ix = Idx() ix.name='dummy.idx' ix.shatable = ''.join(hashes) for k in (4, 5): b = bloom.create(tmpdir + '/pybuptest.bloom', expected=100, k=k) b.add_idx(ix) WVPASSLT(b.pfalse_positive(), .1) b.close() b = bloom.ShaBloom(tmpdir + '/pybuptest.bloom') all_present = True for h in hashes: all_present &= b.exists(h) WVPASS(all_present) false_positives = 0 for h in [os.urandom(20) for i in range(1000)]: if b.exists(h): false_positives += 1 WVPASSLT(false_positives, 5) os.unlink(tmpdir + '/pybuptest.bloom') tf = tempfile.TemporaryFile(dir=tmpdir) b = bloom.create('bup.bloom', f=tf, expected=100) WVPASSEQ(b.rwfile, tf) WVPASSEQ(b.k, 5) # Test large (~1GiB) filter. This may fail on s390 (31-bit # architecture), and anywhere else where the address space is # sufficiently limited. tf = tempfile.TemporaryFile(dir=tmpdir) skip_test = False try: b = bloom.create('bup.bloom', f=tf, expected=2**28, delaywrite=False) except EnvironmentError as ex: (ptr_width, linkage) = platform.architecture() if ptr_width == '32bit' and ex.errno == errno.ENOMEM: WVMSG('skipping large bloom filter test (mmap probably failed) ' + str(ex)) skip_test = True else: raise if not skip_test: WVPASSEQ(b.k, 4)
def test_bup_lutimes(): if not xstat._bup_lutimes: return with no_lingering_errors(), test_tempdir('bup-txstat-') as tmpdir: path = tmpdir + '/foo' open(path, 'w').close() frac_ts = (0, 10**6 / 2) xstat._bup_lutimes(path, (frac_ts, frac_ts)) st = _helpers.stat(path) atime_ts = st[8] mtime_ts = st[9] WVPASSEQ(atime_ts[0], 0) WVPASS(atime_ts[1] == 0 or atime_ts[1] == frac_ts[1] * 1000) WVPASSEQ(mtime_ts[0], 0) WVPASS(mtime_ts[1] == 0 or mtime_ts[1] == frac_ts[1] * 1000)
def testencode(): with no_lingering_errors(): s = 'hello world' looseb = ''.join(git._encode_looseobj('blob', s)) looset = ''.join(git._encode_looseobj('tree', s)) loosec = ''.join(git._encode_looseobj('commit', s)) packb = ''.join(git._encode_packobj('blob', s)) packt = ''.join(git._encode_packobj('tree', s)) packc = ''.join(git._encode_packobj('commit', s)) WVPASSEQ(git._decode_looseobj(looseb), ('blob', s)) WVPASSEQ(git._decode_looseobj(looset), ('tree', s)) WVPASSEQ(git._decode_looseobj(loosec), ('commit', s)) WVPASSEQ(git._decode_packobj(packb), ('blob', s)) WVPASSEQ(git._decode_packobj(packt), ('tree', s)) WVPASSEQ(git._decode_packobj(packc), ('commit', s))
def test_pack_and_unpack(): with no_lingering_errors(): tests = [('', []), ('s', ['foo']), ('ss', ['foo', 'bar']), ('sV', ['foo', 0]), ('sv', ['foo', -1]), ('V', [0]), ('Vs', [0, 'foo']), ('VV', [0, 1]), ('Vv', [0, -1]), ('v', [0]), ('vs', [0, 'foo']), ('vV', [0, 1]), ('vv', [0, -1])] for test in tests: (types, values) = test WVPASSEQ(pack_and_unpack(types, *values), values) WVEXCEPT(Exception, vint.pack, 's') WVEXCEPT(Exception, vint.pack, 's', 'foo', 'bar') WVEXCEPT(Exception, vint.pack, 'x', 1) WVEXCEPT(Exception, vint.unpack, 's', '') WVEXCEPT(Exception, vint.unpack, 'x', '')
def test_fstime(): with no_lingering_errors(): WVPASSEQ(xstat.timespec_to_nsecs((0, 0)), 0) WVPASSEQ(xstat.timespec_to_nsecs((1, 0)), 10**9) WVPASSEQ(xstat.timespec_to_nsecs((0, 10**9 / 2)), 500000000) WVPASSEQ(xstat.timespec_to_nsecs((1, 10**9 / 2)), 1500000000) WVPASSEQ(xstat.timespec_to_nsecs((-1, 0)), -10**9) WVPASSEQ(xstat.timespec_to_nsecs((-1, 10**9 / 2)), -500000000) WVPASSEQ(xstat.timespec_to_nsecs((-2, 10**9 / 2)), -1500000000) WVPASSEQ(xstat.timespec_to_nsecs((0, -1)), -1) WVPASSEQ(type(xstat.timespec_to_nsecs((2, 22222222))), type(0)) WVPASSEQ(type(xstat.timespec_to_nsecs((-2, 22222222))), type(0)) WVPASSEQ(xstat.nsecs_to_timespec(0), (0, 0)) WVPASSEQ(xstat.nsecs_to_timespec(10**9), (1, 0)) WVPASSEQ(xstat.nsecs_to_timespec(500000000), (0, 10**9 / 2)) WVPASSEQ(xstat.nsecs_to_timespec(1500000000), (1, 10**9 / 2)) WVPASSEQ(xstat.nsecs_to_timespec(-10**9), (-1, 0)) WVPASSEQ(xstat.nsecs_to_timespec(-500000000), (-1, 10**9 / 2)) WVPASSEQ(xstat.nsecs_to_timespec(-1500000000), (-2, 10**9 / 2)) x = xstat.nsecs_to_timespec(1977777778) WVPASSEQ(type(x[0]), type(0)) WVPASSEQ(type(x[1]), type(0)) x = xstat.nsecs_to_timespec(-1977777778) WVPASSEQ(type(x[0]), type(0)) WVPASSEQ(type(x[1]), type(0)) WVPASSEQ(xstat.nsecs_to_timeval(0), (0, 0)) WVPASSEQ(xstat.nsecs_to_timeval(10**9), (1, 0)) WVPASSEQ(xstat.nsecs_to_timeval(500000000), (0, (10**9 / 2) / 1000)) WVPASSEQ(xstat.nsecs_to_timeval(1500000000), (1, (10**9 / 2) / 1000)) WVPASSEQ(xstat.nsecs_to_timeval(-10**9), (-1, 0)) WVPASSEQ(xstat.nsecs_to_timeval(-500000000), (-1, (10**9 / 2) / 1000)) WVPASSEQ(xstat.nsecs_to_timeval(-1500000000), (-2, (10**9 / 2) / 1000)) x = xstat.nsecs_to_timeval(1977777778) WVPASSEQ(type(x[0]), type(0)) WVPASSEQ(type(x[1]), type(0)) x = xstat.nsecs_to_timeval(-1977777778) WVPASSEQ(type(x[0]), type(0)) WVPASSEQ(type(x[1]), type(0)) WVPASSEQ(xstat.fstime_floor_secs(0), 0) WVPASSEQ(xstat.fstime_floor_secs(10**9 / 2), 0) WVPASSEQ(xstat.fstime_floor_secs(10**9), 1) WVPASSEQ(xstat.fstime_floor_secs(-10**9 / 2), -1) WVPASSEQ(xstat.fstime_floor_secs(-10**9), -1) WVPASSEQ(type(xstat.fstime_floor_secs(10**9 / 2)), type(0)) WVPASSEQ(type(xstat.fstime_floor_secs(-10**9 / 2)), type(0))
def test_fanout_behaviour(): # Drop in replacement for bupsplit, but splitting if the int value of a # byte >= BUP_BLOBBITS basebits = _helpers.blobbits() def splitbuf(buf): ofs = 0 for b in buf: b = byte_int(b) ofs += 1 if b >= basebits: return ofs, b return 0, 0 with no_lingering_errors(): old_splitbuf = _helpers.splitbuf _helpers.splitbuf = splitbuf old_BLOB_MAX = hashsplit.BLOB_MAX hashsplit.BLOB_MAX = 4 old_BLOB_READ_SIZE = hashsplit.BLOB_READ_SIZE hashsplit.BLOB_READ_SIZE = 10 old_fanout = hashsplit.fanout hashsplit.fanout = 2 levels = lambda f: [ (len(b), l) for b, l in hashsplit.hashsplit_iter([f], True, None) ] # Return a string of n null bytes z = lambda n: b'\x00' * n # Return a byte which will be split with a level of n sb = lambda n: bytes_from_uint(basebits + n) split_never = BytesIO(z(16)) split_first = BytesIO(z(1) + sb(3) + z(14)) split_end = BytesIO(z(13) + sb(1) + z(2)) split_many = BytesIO( sb(1) + z(3) + sb(2) + z(4) + sb(0) + z(4) + sb(5) + z(1)) WVPASSEQ(levels(split_never), [(4, 0), (4, 0), (4, 0), (4, 0)]) WVPASSEQ(levels(split_first), [(2, 3), (4, 0), (4, 0), (4, 0), (2, 0)]) WVPASSEQ(levels(split_end), [(4, 0), (4, 0), (4, 0), (2, 1), (2, 0)]) WVPASSEQ(levels(split_many), [(1, 1), (4, 2), (4, 0), (1, 0), (4, 0), (1, 5), (1, 0)]) _helpers.splitbuf = old_splitbuf hashsplit.BLOB_MAX = old_BLOB_MAX hashsplit.BLOB_READ_SIZE = old_BLOB_READ_SIZE hashsplit.fanout = old_fanout
def test_server_split_with_indexes(): with no_lingering_errors(): with test_tempdir(b'bup-tclient-') as tmpdir: environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) lw = git.PackWriter() c = client.Client(bupdir, create=True) rw = c.new_packwriter() lw.new_blob(s1) lw.close() rw.new_blob(s2) rw.breakpoint() rw.new_blob(s1) rw.close()
def test_bup_utimensat(): if not xstat._bup_utimensat: return with no_lingering_errors(): with test_tempdir('bup-txstat-') as tmpdir: path = tmpdir + '/foo' open(path, 'w').close() frac_ts = (0, 10**9 / 2) xstat._bup_utimensat(_helpers.AT_FDCWD, path, (frac_ts, frac_ts), 0) st = _helpers.stat(path) atime_ts = st[8] mtime_ts = st[9] WVPASSEQ(atime_ts[0], 0) WVPASS(atime_ts[1] == 0 or atime_ts[1] == frac_ts[1]) WVPASSEQ(mtime_ts[0], 0) WVPASS(mtime_ts[1] == 0 or mtime_ts[1] == frac_ts[1])
def test_from_path_error(): if is_superuser() or detect_fakeroot(): return with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir: path = tmpdir + '/foo' os.mkdir(path) m = metadata.from_path(path, archive_path=path, save_symlinks=True) WVPASSEQ(m.path, path) os.chmod(path, 000) metadata.from_path(path, archive_path=path, save_symlinks=True) if metadata.get_linux_file_attr: print >> sys.stderr, 'saved_errors:', helpers.saved_errors WVPASS(len(helpers.saved_errors) == 1) errmsg = _first_err() WVPASS(errmsg.startswith('read Linux attr')) clear_errors()
def test_read_and_seek(): # Write a set of randomly sized files containing random data whose # names are their sizes, and then verify that what we get back # from the vfs when seeking and reading with various block sizes # matches the original content. with no_lingering_errors(): with test_tempdir(b'bup-tvfs-read-') as tmpdir: resolve = vfs.resolve bup_dir = tmpdir + b'/bup' environ[b'GIT_DIR'] = bup_dir environ[b'BUP_DIR'] = bup_dir git.repodir = bup_dir repo = LocalRepo() data_path = tmpdir + b'/src' os.mkdir(data_path) seed = randint(-(1 << 31), (1 << 31) - 1) rand = Random() rand.seed(seed) print('test_read seed:', seed, file=sys.stderr) max_size = 2 * 1024 * 1024 sizes = set((rand.randint(1, max_size) for _ in range(5))) sizes.add(1) sizes.add(max_size) for size in sizes: write_sized_random_content(data_path, size, seed) ex((bup_path, b'init')) ex((bup_path, b'index', b'-v', data_path)) ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test', b'--strip', data_path)) read_sizes = set((rand.randint(1, max_size) for _ in range(10))) sizes.add(1) sizes.add(max_size) print('test_read src sizes:', sizes, file=sys.stderr) print('test_read read sizes:', read_sizes, file=sys.stderr) for size in sizes: res = resolve(repo, b'/test/latest/' + str(size).encode('ascii')) _, item = res[-1] wvpasseq(size, vfs.item_size(repo, res[-1][1])) validate_vfs_streaming_read(repo, item, b'%s/%d' % (data_path, size), read_sizes) validate_vfs_seeking_read(repo, item, b'%s/%d' % (data_path, size), read_sizes)
def test_risky_path(): with no_lingering_errors(): risky = metadata._risky_path WVPASS(risky('/foo')) WVPASS(risky('///foo')) WVPASS(risky('/../foo')) WVPASS(risky('../foo')) WVPASS(risky('foo/..')) WVPASS(risky('foo/../')) WVPASS(risky('foo/../bar')) WVFAIL(risky('foo')) WVFAIL(risky('foo/')) WVFAIL(risky('foo///')) WVFAIL(risky('./foo')) WVFAIL(risky('foo/.')) WVFAIL(risky('./foo/.')) WVFAIL(risky('foo/bar')) WVFAIL(risky('foo/./bar'))
def index_writer(): with no_lingering_errors(), test_tempdir('bup-tindex-') as tmpdir: orig_cwd = os.getcwd() try: os.chdir(tmpdir) ds = xstat.stat('.') fs = xstat.stat(lib_t_dir + '/tindex.py') ms = index.MetaStoreWriter('index.meta.tmp') tmax = (time.time() - 1) * 10**9 w = index.Writer('index.tmp', ms, tmax) w.add('/var/tmp/sporky', fs, 0) w.add('/etc/passwd', fs, 0) w.add('/etc/', ds, 0) w.add('/', ds, 0) ms.close() w.close() finally: os.chdir(orig_cwd)
def test_stripped_path_components(): with no_lingering_errors(): WVPASSEQ(stripped_path_components('/', []), [('', '/')]) WVPASSEQ(stripped_path_components('/', ['']), [('', '/')]) WVPASSEQ(stripped_path_components('/', ['/']), [('', '/')]) WVPASSEQ(stripped_path_components('/foo', ['/']), [('', '/'), ('foo', '/foo')]) WVPASSEQ(stripped_path_components('/', ['/foo']), [('', '/')]) WVPASSEQ(stripped_path_components('/foo', ['/bar']), [('', '/'), ('foo', '/foo')]) WVPASSEQ(stripped_path_components('/foo', ['/foo']), [('', '/foo')]) WVPASSEQ(stripped_path_components('/foo/bar', ['/foo']), [('', '/foo'), ('bar', '/foo/bar')]) WVPASSEQ(stripped_path_components('/foo/bar', ['/bar', '/foo', '/baz']), [('', '/foo'), ('bar', '/foo/bar')]) WVPASSEQ(stripped_path_components('/foo/bar/baz', ['/foo/bar/baz']), [('', '/foo/bar/baz')]) WVEXCEPT(Exception, stripped_path_components, 'foo', [])
def test_resolve_loop(): with no_lingering_errors(): with test_tempdir('bup-tvfs-resloop-') as tmpdir: resolve = vfs.resolve lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir git.repodir = bup_dir repo = LocalRepo() data_path = tmpdir + '/src' os.mkdir(data_path) symlink('loop', data_path + '/loop') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', data_path)) wvexcept(vfs.Loop, resolve, repo, '/test/latest/loop')
def test_remote_parsing(): with no_lingering_errors(): tests = ( (':/bup', ('file', None, None, '/bup')), ('file:///bup', ('file', None, None, '/bup')), ('192.168.1.1:/bup', ('ssh', '192.168.1.1', None, '/bup')), ('ssh://192.168.1.1:2222/bup', ('ssh', '192.168.1.1', '2222', '/bup')), ('ssh://[ff:fe::1]:2222/bup', ('ssh', 'ff:fe::1', '2222', '/bup')), ('bup://foo.com:1950', ('bup', 'foo.com', '1950', None)), ('bup://foo.com:1950/bup', ('bup', 'foo.com', '1950', '/bup')), ('bup://[ff:fe::1]/bup', ('bup', 'ff:fe::1', None, '/bup')),) for remote, values in tests: WVPASSEQ(client.parse_remote(remote), values) try: client.parse_remote('http://asdf.com/bup') WVFAIL() except client.ClientError: WVPASS()
def test_dumb_client_server(): with no_lingering_errors(): with test_tempdir(b'bup-tclient-') as tmpdir: environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) open(git.repo(b'bup-dumb-server'), 'w').close() lw = git.PackWriter() lw.new_blob(s1) lw.close() c = client.Client(bupdir, create=True) rw = c.new_packwriter() WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1) rw.new_blob(s1) WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1) rw.new_blob(s2) rw.close() WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 2)