def test_optdict():
with no_lingering_errors():
d = options.OptDict({
'x': ('x', False),
'y': ('y', False),
'z': ('z', False),
'other_thing': ('other_thing', False),
'no_other_thing': ('other_thing', True),
'no_z': ('z', True),
'no_smart': ('smart', True),
'smart': ('smart', False),
'stupid': ('smart', True),
'no_smart': ('smart', False),
})
WVPASS('foo')
d['x'] = 5
d['y'] = 4
d['z'] = 99
d['no_other_thing'] = 5
WVPASSEQ(d.x, 5)
WVPASSEQ(d.y, 4)
WVPASSEQ(d.z, 99)
WVPASSEQ(d.no_z, False)
WVPASSEQ(d.no_other_thing, True)
WVEXCEPT(KeyError, lambda: d.p)
def test_item_mode():
with no_lingering_errors():
mode = S_IFDIR | 0o755
meta = metadata.from_path('.')
oid = '\0' * 20
wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
def test_check_repo_or_die():
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
git.init_repo(bupdir)
git.check_repo_or_die()
WVPASS('check_repo_or_die') # if we reach this point the call above passed
os.rename(bupdir + '/objects/pack', bupdir + '/objects/pack.tmp')
open(bupdir + '/objects/pack', 'w').close()
try:
git.check_repo_or_die()
except SystemExit as e:
WVPASSEQ(e.code, 14)
else:
WVFAIL()
os.unlink(bupdir + '/objects/pack')
os.rename(bupdir + '/objects/pack.tmp', bupdir + '/objects/pack')
try:
git.check_repo_or_die('nonexistantbup.tmp')
except SystemExit as e:
WVPASSEQ(e.code, 15)
else:
WVFAIL()
finally:
os.chdir(orig_cwd)
def test_options():
with no_lingering_errors():
o = options.Options(optspec)
(opt,flags,extra) = o.parse(['-tttqp', 7, '--longoption', '19',
'hanky', '--onlylong', '-7'])
WVPASSEQ(flags[0], ('-t', ''))
WVPASSEQ(flags[1], ('-t', ''))
WVPASSEQ(flags[2], ('-t', ''))
WVPASSEQ(flags[3], ('-q', ''))
WVPASSEQ(flags[4], ('-p', 7))
WVPASSEQ(flags[5], ('--longoption', '19'))
WVPASSEQ(extra, ['hanky'])
WVPASSEQ((opt.t, opt.q, opt.p, opt.l, opt.onlylong,
opt.neveropt), (3,1,7,19,1,None))
WVPASSEQ((opt.deftest1, opt.deftest2, opt.deftest3, opt.deftest4,
opt.deftest5), (1,2,None,None,'[square'))
WVPASSEQ((opt.stupid, opt.no_stupid), (True, None))
WVPASSEQ((opt.smart, opt.no_smart), (None, True))
WVPASSEQ((opt.x, opt.extended, opt.no_simple), (2,2,2))
WVPASSEQ((opt.no_x, opt.no_extended, opt.simple), (False,False,False))
WVPASSEQ(opt['#'], 7)
WVPASSEQ(opt.compress, 7)
(opt,flags,extra) = o.parse(['--onlylong', '-t', '--no-onlylong',
'--smart', '--simple'])
WVPASSEQ((opt.t, opt.q, opt.onlylong), (1, None, 0))
WVPASSEQ((opt.stupid, opt.no_stupid), (False, True))
WVPASSEQ((opt.smart, opt.no_smart), (True, False))
WVPASSEQ((opt.x, opt.extended, opt.no_simple), (0,0,0))
WVPASSEQ((opt.no_x, opt.no_extended, opt.simple), (True,True,True))
def test_shquote():
with no_lingering_errors():
WVPASSEQ(qst(""" this is basic \t\n\r text """), ["this", "is", "basic", "text"])
WVPASSEQ(qst(r""" \"x\" "help" 'yelp' """), ['"x"', "help", "yelp"])
WVPASSEQ(qst(r""" "'\"\"'" '\"\'' """), ["'\"\"'", "\\\"'"])
WVPASSEQ(shquote.quotesplit(' this is "unfinished'), [(2, "this"), (7, "is"), (10, "unfinished")])
WVPASSEQ(shquote.quotesplit('"silly"\'will'), [(0, "silly"), (7, "will")])
WVPASSEQ(shquote.unfinished_word('this is a "billy" "goat'), ('"', "goat"))
WVPASSEQ(shquote.unfinished_word("'x"), ("'", "x"))
WVPASSEQ(shquote.unfinished_word("abra cadabra "), (None, ""))
WVPASSEQ(shquote.unfinished_word("abra cadabra"), (None, "cadabra"))
(qtype, word) = shquote.unfinished_word("this is /usr/loc")
WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", True), "al")
(qtype, word) = shquote.unfinished_word("this is '/usr/loc")
WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", True), "al'")
(qtype, word) = shquote.unfinished_word('this is "/usr/loc')
WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", True), 'al"')
(qtype, word) = shquote.unfinished_word('this is "/usr/loc')
WVPASSEQ(shquote.what_to_add(qtype, word, "/usr/local", False), "al")
(qtype, word) = shquote.unfinished_word('this is \\ hammer\\ "')
WVPASSEQ(word, ' hammer "')
WVPASSEQ(shquote.what_to_add(qtype, word, ' hammer "time"', True), 'time\\"')
WVPASSEQ(
shquote.quotify_list(["a", "", '"word"', "'third'", "'", "x y"]), "a '' '\"word\"' \"'third'\" \"'\" 'x y'"
)
def test_multiple_suggestions():
with no_lingering_errors():
with test_tempdir('bup-tclient-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
lw = git.PackWriter()
lw.new_blob(s2)
lw.close()
WVPASSEQ(len(glob.glob(git.repo('objects/pack'+IDX_PAT))), 2)
c = client.Client(bupdir, create=True)
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 0)
rw = c.new_packwriter()
s1sha = rw.new_blob(s1)
WVPASS(rw.exists(s1sha))
s2sha = rw.new_blob(s2)
# This is a little hacky, but ensures that we test the
# code under test
while (len(glob.glob(c.cachedir+IDX_PAT)) < 2 and
not c.conn.has_input()):
pass
rw.new_blob(s2)
WVPASS(rw.objcache.exists(s1sha))
WVPASS(rw.objcache.exists(s2sha))
rw.new_blob(s3)
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 3)
def test_uncache_ours_upto():
history = []
def mock_fadvise_pages_done(f, ofs, len):
history.append((f, ofs, len))
with no_lingering_errors():
uncache_upto = hashsplit._uncache_ours_upto
page_size = helpers.sc_page_size
orig_pages_done = hashsplit._fadvise_pages_done
try:
hashsplit._fadvise_pages_done = mock_fadvise_pages_done
history = []
uncache_upto(42, 0, (0, 1), iter([]))
WVPASSEQ([], history)
uncache_upto(42, page_size, (0, 1), iter([]))
WVPASSEQ([(42, 0, 1)], history)
history = []
uncache_upto(42, page_size, (0, 3), iter([(5, 2)]))
WVPASSEQ([], history)
uncache_upto(42, 2 * page_size, (0, 3), iter([(5, 2)]))
WVPASSEQ([], history)
uncache_upto(42, 3 * page_size, (0, 3), iter([(5, 2)]))
WVPASSEQ([(42, 0, 3)], history)
history = []
uncache_upto(42, 5 * page_size, (0, 3), iter([(5, 2)]))
WVPASSEQ([(42, 0, 3)], history)
history = []
uncache_upto(42, 6 * page_size, (0, 3), iter([(5, 2)]))
WVPASSEQ([(42, 0, 3)], history)
history = []
uncache_upto(42, 7 * page_size, (0, 3), iter([(5, 2)]))
WVPASSEQ([(42, 0, 3), (42, 5, 2)], history)
finally:
hashsplit._fadvise_pages_done = orig_pages_done
def test_grafted_path_components():
with no_lingering_errors():
WVPASSEQ(grafted_path_components([('/chroot', '/')], '/foo'),
[('', '/'), ('foo', '/foo')])
WVPASSEQ(grafted_path_components([('/foo/bar', '/')],
'/foo/bar/baz/bax'),
[('', '/foo/bar'),
('baz', '/foo/bar/baz'),
('bax', '/foo/bar/baz/bax')])
WVPASSEQ(grafted_path_components([('/foo/bar/baz', '/bax')],
'/foo/bar/baz/1/2'),
[('', None),
('bax', '/foo/bar/baz'),
('1', '/foo/bar/baz/1'),
('2', '/foo/bar/baz/1/2')])
WVPASSEQ(grafted_path_components([('/foo', '/bar/baz/bax')],
'/foo/bar'),
[('', None),
('bar', None),
('baz', None),
('bax', '/foo'),
('bar', '/foo/bar')])
WVPASSEQ(grafted_path_components([('/foo/bar/baz', '/a/b/c')],
'/foo/bar/baz'),
[('', None), ('a', None), ('b', None), ('c', '/foo/bar/baz')])
WVPASSEQ(grafted_path_components([('/', '/a/b/c/')], '/foo/bar'),
[('', None), ('a', None), ('b', None), ('c', '/'),
('foo', '/foo'), ('bar', '/foo/bar')])
WVEXCEPT(Exception, grafted_path_components, 'foo', [])
def test_duplicate_save_dates():
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
environ['TZ'] = 'UTC'
git.repodir = bup_dir
data_path = tmpdir + '/src'
os.mkdir(data_path)
with open(data_path + '/file', 'w+') as tmpfile:
tmpfile.write(b'canary\n')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
for i in range(11):
ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
repo = LocalRepo()
res = vfs.resolve(repo, '/test')
wvpasseq(2, len(res))
name, revlist = res[-1]
wvpasseq('test', name)
wvpasseq(('.',
'1970-01-02-034640-00',
'1970-01-02-034640-01',
'1970-01-02-034640-02',
'1970-01-02-034640-03',
'1970-01-02-034640-04',
'1970-01-02-034640-05',
'1970-01-02-034640-06',
'1970-01-02-034640-07',
'1970-01-02-034640-08',
'1970-01-02-034640-09',
'1970-01-02-034640-10',
'latest'),
tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
def testmangle():
with no_lingering_errors():
afile = 0100644
afile2 = 0100770
alink = 0120000
adir = 0040000
adir2 = 0040777
WVPASSEQ(git.mangle_name("a", adir2, adir), "a")
WVPASSEQ(git.mangle_name(".bup", adir2, adir), ".bup.bupl")
WVPASSEQ(git.mangle_name("a.bupa", adir2, adir), "a.bupa.bupl")
WVPASSEQ(git.mangle_name("b.bup", alink, alink), "b.bup.bupl")
WVPASSEQ(git.mangle_name("b.bu", alink, alink), "b.bu")
WVPASSEQ(git.mangle_name("f", afile, afile2), "f")
WVPASSEQ(git.mangle_name("f.bup", afile, afile2), "f.bup.bupl")
WVPASSEQ(git.mangle_name("f.bup", afile, adir), "f.bup.bup")
WVPASSEQ(git.mangle_name("f", afile, adir), "f.bup")
WVPASSEQ(git.demangle_name("f.bup", afile), ("f", git.BUP_CHUNKED))
WVPASSEQ(git.demangle_name("f.bupl", afile), ("f", git.BUP_NORMAL))
WVPASSEQ(git.demangle_name("f.bup.bupl", afile), ("f.bup", git.BUP_NORMAL))
WVPASSEQ(git.demangle_name(".bupm", afile), ('', git.BUP_NORMAL))
WVPASSEQ(git.demangle_name(".bupm", adir), ('', git.BUP_CHUNKED))
# for safety, we ignore .bup? suffixes we don't recognize. Future
# versions might implement a .bup[a-z] extension as something other
# than BUP_NORMAL.
WVPASSEQ(git.demangle_name("f.bupa", afile), ("f.bupa", git.BUP_NORMAL))
def test_metadata_method():
with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir:
bup_dir = tmpdir + '/bup'
data_path = tmpdir + '/foo'
os.mkdir(data_path)
ex('touch', data_path + '/file')
ex('ln', '-s', 'file', data_path + '/symlink')
test_time1 = 13 * 1000000000
test_time2 = 42 * 1000000000
utime(data_path + '/file', (0, test_time1))
lutime(data_path + '/symlink', (0, 0))
utime(data_path, (0, test_time2))
ex(bup_path, '-d', bup_dir, 'init')
ex(bup_path, '-d', bup_dir, 'index', '-v', data_path)
ex(bup_path, '-d', bup_dir, 'save', '-tvvn', 'test', data_path)
git.check_repo_or_die(bup_dir)
top = vfs.RefList(None)
n = top.lresolve('/test/latest' + resolve_parent(data_path))
m = n.metadata()
WVPASS(m.mtime == test_time2)
WVPASS(len(n.subs()) == 2)
WVPASS(n.name == 'foo')
WVPASS(set([x.name for x in n.subs()]) == set(['file', 'symlink']))
for sub in n:
if sub.name == 'file':
m = sub.metadata()
WVPASS(m.mtime == test_time1)
elif sub.name == 'symlink':
m = sub.metadata()
WVPASS(m.mtime == 0)
def test_restore_over_existing_target():
with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir:
path = tmpdir + '/foo'
os.mkdir(path)
dir_m = metadata.from_path(path, archive_path=path, save_symlinks=True)
os.rmdir(path)
open(path, 'w').close()
file_m = metadata.from_path(path, archive_path=path, save_symlinks=True)
# Restore dir over file.
WVPASSEQ(dir_m.create_path(path, create_symlinks=True), None)
WVPASS(stat.S_ISDIR(os.stat(path).st_mode))
# Restore dir over dir.
WVPASSEQ(dir_m.create_path(path, create_symlinks=True), None)
WVPASS(stat.S_ISDIR(os.stat(path).st_mode))
# Restore file over dir.
WVPASSEQ(file_m.create_path(path, create_symlinks=True), None)
WVPASS(stat.S_ISREG(os.stat(path).st_mode))
# Restore file over file.
WVPASSEQ(file_m.create_path(path, create_symlinks=True), None)
WVPASS(stat.S_ISREG(os.stat(path).st_mode))
# Restore file over non-empty dir.
os.remove(path)
os.mkdir(path)
open(path + '/bar', 'w').close()
WVEXCEPT(Exception, file_m.create_path, path, create_symlinks=True)
# Restore dir over non-empty dir.
os.remove(path + '/bar')
os.mkdir(path + '/bar')
WVEXCEPT(Exception, dir_m.create_path, path, create_symlinks=True)
def test_utc_offset_str():
with no_lingering_errors():
tz = os.environ.get('TZ')
try:
os.environ['TZ'] = 'FOO+0:00'
WVPASSEQ(utc_offset_str(0), '+0000')
os.environ['TZ'] = 'FOO+1:00'
WVPASSEQ(utc_offset_str(0), '-0100')
os.environ['TZ'] = 'FOO-1:00'
WVPASSEQ(utc_offset_str(0), '+0100')
os.environ['TZ'] = 'FOO+3:3'
WVPASSEQ(utc_offset_str(0), '-0303')
os.environ['TZ'] = 'FOO-3:3'
WVPASSEQ(utc_offset_str(0), '+0303')
# Offset is not an integer number of minutes
os.environ['TZ'] = 'FOO+3:3:3'
WVPASSEQ(utc_offset_str(1), '-0303')
os.environ['TZ'] = 'FOO-3:3:3'
WVPASSEQ(utc_offset_str(1), '+0303')
WVPASSEQ(utc_offset_str(314159), '+0303')
finally:
if tz:
os.environ['TZ'] = tz
else:
try:
del os.environ['TZ']
except KeyError:
pass
def test_batchpipe():
with no_lingering_errors():
for chunk in batchpipe(['echo'], []):
WVPASS(False)
out = ''
for chunk in batchpipe(['echo'], ['42']):
out += chunk
WVPASSEQ(out, '42\n')
try:
batchpipe(['bash', '-c'], ['exit 42'])
except Exception as ex:
WVPASSEQ(str(ex),
"subprocess 'bash -c exit 42' failed with status 42")
args = [str(x) for x in range(6)]
# Force batchpipe to break the args into batches of 3. This
# approach assumes all args are the same length.
arg_max = \
helpers._argmax_base(['echo']) + helpers._argmax_args_size(args[:3])
batches = batchpipe(['echo'], args, arg_max=arg_max)
WVPASSEQ(next(batches), '0 1 2\n')
WVPASSEQ(next(batches), '3 4 5\n')
WVPASSEQ(next(batches, None), None)
batches = batchpipe(['echo'], [str(x) for x in range(5)], arg_max=arg_max)
WVPASSEQ(next(batches), '0 1 2\n')
WVPASSEQ(next(batches), '3 4\n')
WVPASSEQ(next(batches, None), None)
def test_apply_to_path_restricted_access():
if is_superuser() or detect_fakeroot():
return
if sys.platform.startswith('cygwin'):
return # chmod 000 isn't effective.
with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir:
parent = tmpdir + '/foo'
path = parent + '/bar'
os.mkdir(parent)
os.mkdir(path)
clear_errors()
m = metadata.from_path(path, archive_path=path, save_symlinks=True)
WVPASSEQ(m.path, path)
os.chmod(parent, 000)
m.apply_to_path(path)
print >> sys.stderr, 'saved_errors:', helpers.saved_errors
expected_errors = ['utime: ']
if m.linux_attr and _linux_attr_supported(tmpdir):
expected_errors.append('Linux chattr: ')
if metadata.xattr and m.linux_xattr:
expected_errors.append("xattr.set '")
WVPASS(len(helpers.saved_errors) == len(expected_errors))
for i in xrange(len(expected_errors)):
WVPASS(str(helpers.saved_errors[i]).startswith(expected_errors[i]))
clear_errors()
def test_long_index():
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
w = git.PackWriter()
obj_bin = struct.pack('!IIIII',
0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
obj2_bin = struct.pack('!IIIII',
0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
obj3_bin = struct.pack('!IIIII',
0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
pack_bin = struct.pack('!IIIII',
0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
idx = list(list() for i in xrange(256))
idx[0].append((obj_bin, 1, 0xfffffffff))
idx[0x11].append((obj2_bin, 2, 0xffffffffff))
idx[0x22].append((obj3_bin, 3, 0xff))
w.count = 3
name = tmpdir + '/tmp.idx'
r = w._write_pack_idx_v2(name, idx, pack_bin)
i = git.PackIdxV2(name, open(name, 'rb'))
WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
WVPASSEQ(i.find_offset(obj3_bin), 0xff)
def test_new_commit():
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = 'Author'
author_mail = 'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = 'Committer'
committer_mail = 'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(commit.encode('hex'), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp())
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def test_nonresident_page_regions():
with no_lingering_errors():
WVPASSEQ(nr_regions([]), [])
WVPASSEQ(nr_regions([1]), [])
WVPASSEQ(nr_regions([0]), [(0, 1)])
WVPASSEQ(nr_regions([1, 0]), [(1, 1)])
WVPASSEQ(nr_regions([0, 0]), [(0, 2)])
WVPASSEQ(nr_regions([1, 0, 1]), [(1, 1)])
WVPASSEQ(nr_regions([1, 0, 0]), [(1, 2)])
WVPASSEQ(nr_regions([0, 1, 0]), [(0, 1), (2, 1)])
WVPASSEQ(nr_regions([0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0]),
[(0, 2), (5, 3), (9, 2)])
WVPASSEQ(nr_regions([2, 42, 3, 101]), [(0, 2)])
# Test limit
WVPASSEQ(nr_regions([0, 0, 0], None), [(0, 3)])
WVPASSEQ(nr_regions([0, 0, 0], 1), [(0, 1), (1, 1), (2, 1)])
WVPASSEQ(nr_regions([0, 0, 0], 2), [(0, 2), (2, 1)])
WVPASSEQ(nr_regions([0, 0, 0], 3), [(0, 3)])
WVPASSEQ(nr_regions([0, 0, 0], 4), [(0, 3)])
WVPASSEQ(nr_regions([0, 0, 1], None), [(0, 2)])
WVPASSEQ(nr_regions([0, 0, 1], 1), [(0, 1), (1, 1)])
WVPASSEQ(nr_regions([0, 0, 1], 2), [(0, 2)])
WVPASSEQ(nr_regions([0, 0, 1], 3), [(0, 2)])
WVPASSEQ(nr_regions([1, 0, 0], None), [(1, 2)])
WVPASSEQ(nr_regions([1, 0, 0], 1), [(1, 1), (2, 1)])
WVPASSEQ(nr_regions([1, 0, 0], 2), [(1, 2)])
WVPASSEQ(nr_regions([1, 0, 0], 3), [(1, 2)])
WVPASSEQ(nr_regions([1, 0, 0, 0, 1], None), [(1, 3)])
WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 1), [(1, 1), (2, 1), (3, 1)])
WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 2), [(1, 2), (3, 1)])
WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 3), [(1, 3)])
WVPASSEQ(nr_regions([1, 0, 0, 0, 1], 4), [(1, 3)])
def test_item_read_write():
with no_lingering_errors():
x = vfs.Root(meta=13)
stream = BytesIO()
vfs.write_item(stream, x)
print('stream:', repr(stream.getvalue()), stream.tell(), file=sys.stderr)
stream.seek(0)
wvpasseq(x, vfs.read_item(stream))
def test_path_components():
with no_lingering_errors():
WVPASSEQ(path_components('/'), [('', '/')])
WVPASSEQ(path_components('/foo'), [('', '/'), ('foo', '/foo')])
WVPASSEQ(path_components('/foo/'), [('', '/'), ('foo', '/foo')])
WVPASSEQ(path_components('/foo/bar'),
[('', '/'), ('foo', '/foo'), ('bar', '/foo/bar')])
WVEXCEPT(Exception, path_components, 'foo')
def test_vint():
with no_lingering_errors():
values = (0, 1, 42, 64, 10**16)
for x in values:
WVPASSEQ(encode_and_decode_vint(x), x)
for x in [-x for x in values]:
WVPASSEQ(encode_and_decode_vint(x), x)
WVEXCEPT(EOFError, vint.read_vint, BytesIO())
def test_parse_num():
with no_lingering_errors():
pn = parse_num
WVPASSEQ(pn('1'), 1)
WVPASSEQ(pn('0'), 0)
WVPASSEQ(pn('1.5k'), 1536)
WVPASSEQ(pn('2 gb'), 2*1024*1024*1024)
WVPASSEQ(pn('1e+9 k'), 1000000000 * 1024)
WVPASSEQ(pn('-3e-3mb'), int(-0.003 * 1024 * 1024))
def test_fallback_next():
with no_lingering_errors():
global next
orig = next
next = helpers._fallback_next
try:
test_next()
finally:
next = orig
def test_readpipe():
with no_lingering_errors():
x = readpipe(['echo', '42'])
WVPASSEQ(x, '42\n')
try:
readpipe(['bash', '-c', 'exit 42'])
except Exception as ex:
WVPASSEQ(str(ex),
"subprocess 'bash -c exit 42' failed with status 42")
def test_bloom():
with no_lingering_errors():
with test_tempdir('bup-tbloom-') as tmpdir:
hashes = [os.urandom(20) for i in range(100)]
class Idx:
pass
ix = Idx()
ix.name = 'dummy.idx'
ix.shatable = ''.join(hashes)
for k in (4, 5):
b = bloom.create(tmpdir + '/pybuptest.bloom',
expected=100,
k=k)
b.add_idx(ix)
WVPASSLT(b.pfalse_positive(), .1)
b.close()
b = bloom.ShaBloom(tmpdir + '/pybuptest.bloom')
all_present = True
for h in hashes:
all_present &= b.exists(h)
WVPASS(all_present)
false_positives = 0
for h in [os.urandom(20) for i in range(1000)]:
if b.exists(h):
false_positives += 1
WVPASSLT(false_positives, 5)
os.unlink(tmpdir + '/pybuptest.bloom')
tf = tempfile.TemporaryFile(dir=tmpdir)
b = bloom.create('bup.bloom', f=tf, expected=100)
WVPASSEQ(b.rwfile, tf)
WVPASSEQ(b.k, 5)
# Test large (~1GiB) filter. This may fail on s390 (31-bit
# architecture), and anywhere else where the address space is
# sufficiently limited.
tf = tempfile.TemporaryFile(dir=tmpdir)
skip_test = False
try:
b = bloom.create('bup.bloom',
f=tf,
expected=2**28,
delaywrite=False)
except EnvironmentError as ex:
(ptr_width, linkage) = platform.architecture()
if ptr_width == '32bit' and ex.errno == errno.ENOMEM:
WVMSG(
'skipping large bloom filter test (mmap probably failed) '
+ str(ex))
skip_test = True
else:
raise
if not skip_test:
WVPASSEQ(b.k, 4)
def test_misc():
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
data_path = tmpdir + '/src'
os.mkdir(data_path)
with open(data_path + '/file', 'w+') as tmpfile:
tmpfile.write(b'canary\n')
symlink('file', data_path + '/symlink')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
data_path))
repo = LocalRepo()
wvstart('readlink')
ls_tree = exo(('git', 'ls-tree', 'test', 'symlink')).out
mode, typ, oidx, name = ls_tree.strip().split(None, 3)
assert name == 'symlink'
link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
wvpasseq('file', vfs.readlink(repo, link_item))
ls_tree = exo(('git', 'ls-tree', 'test', 'file')).out
mode, typ, oidx, name = ls_tree.strip().split(None, 3)
assert name == 'file'
file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
wvexcept(Exception, vfs.readlink, repo, file_item)
wvstart('item_size')
wvpasseq(4, vfs.item_size(repo, link_item))
wvpasseq(7, vfs.item_size(repo, file_item))
meta = metadata.from_path(__file__)
meta.size = 42
fake_item = file_item._replace(meta=meta)
wvpasseq(42, vfs.item_size(repo, fake_item))
wvstart('augment_item_meta')
run_augment_item_meta_tests(repo,
'/test/latest/file', 7,
'/test/latest/symlink', 'file')
wvstart('copy_item')
# FIXME: this caused StopIteration
#_, file_item = vfs.resolve(repo, '/file')[-1]
_, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
file_copy = vfs.copy_item(file_item)
wvpass(file_copy is not file_item)
wvpass(file_copy.meta is not file_item.meta)
wvpass(isinstance(file_copy, tuple))
wvpass(file_item.meta.user)
wvpass(file_copy.meta.user)
file_copy.meta.user = None
wvpass(file_item.meta.user)
def test_midx_close():
fddir = b'/proc/self/fd'
try:
os.listdir(fddir)
except Exception:
# not supported, not Linux, I guess
return
def openfiles():
for fd in os.listdir(fddir):
try:
yield os.readlink(os.path.join(fddir, fd))
except OSError:
pass
def force_midx(objdir):
args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
check_call(args)
with no_lingering_errors(), \
test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
# create a few dummy idxes
for i in range(10):
_create_idx(tmpdir, i)
git.auto_midx(tmpdir)
l = git.PackIdxList(tmpdir)
# this doesn't exist (yet)
WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
for i in range(10, 15):
_create_idx(tmpdir, i)
# delete the midx ...
# TODO: why do we need to? git.auto_midx() below doesn't?!
for fn in os.listdir(tmpdir):
if fn.endswith(b'.midx'):
os.unlink(os.path.join(tmpdir, fn))
# and make a new one
git.auto_midx(tmpdir)
# check it still doesn't exist - we haven't refreshed
WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
# check that we still have the midx open, this really
# just checks more for the kernel API ('deleted' string)
for fn in openfiles():
if not b'midx-' in fn:
continue
WVPASSEQ(True, b'deleted' in fn)
# refresh the PackIdxList
l.refresh()
# and check that an object in pack 10 exists now
WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
for fn in openfiles():
if not b'midx-' in fn:
continue
# check that we don't have it open anymore
WVPASSEQ(False, b'deleted' in fn)
def prep_and_test_repo(name, create_repo, test_repo):
with no_lingering_errors():
with test_tempdir(b'bup-t' + name) as tmpdir:
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
ex((bup_path, b'init'))
git.repodir = bup_dir
with create_repo(bup_dir) as repo:
test_repo(repo, tmpdir)
def test_parse_num():
with no_lingering_errors():
pn = parse_num
WVPASSEQ(pn(b'1'), 1)
WVPASSEQ(pn('1'), 1)
WVPASSEQ(pn('0'), 0)
WVPASSEQ(pn('1.5k'), 1536)
WVPASSEQ(pn('2 gb'), 2 * 1024 * 1024 * 1024)
WVPASSEQ(pn('1e+9 k'), 1000000000 * 1024)
WVPASSEQ(pn('-3e-3mb'), int(-0.003 * 1024 * 1024))
def test_readpipe():
with no_lingering_errors():
x = readpipe([b'echo', b'42'])
WVPASSEQ(x, b'42\n')
try:
readpipe([b'bash', b'-c', b'exit 42'])
except Exception as ex:
rx = '^subprocess b?"bash -c \'exit 42\'" failed with status 42$'
if not re.match(rx, str(ex)):
WVPASSEQ(str(ex), rx)
def test_misc():
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
data_path = tmpdir + '/src'
os.mkdir(data_path)
with open(data_path + '/file', 'w+') as tmpfile:
tmpfile.write(b'canary\n')
symlink('file', data_path + '/symlink')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
data_path))
repo = LocalRepo()
wvstart('readlink')
ls_tree = exo(('git', 'ls-tree', 'test', 'symlink'))
mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
assert name == 'symlink'
link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
wvpasseq('file', vfs.readlink(repo, link_item))
ls_tree = exo(('git', 'ls-tree', 'test', 'file'))
mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
assert name == 'file'
file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
wvexcept(Exception, vfs.readlink, repo, file_item)
wvstart('item_size')
wvpasseq(4, vfs.item_size(repo, link_item))
wvpasseq(7, vfs.item_size(repo, file_item))
meta = metadata.from_path(__file__)
meta.size = 42
fake_item = file_item._replace(meta=meta)
wvpasseq(42, vfs.item_size(repo, fake_item))
wvstart('augment_item_meta')
run_augment_item_meta_tests(repo, '/test/latest/file', 7,
'/test/latest/symlink', 'file')
wvstart('copy_item')
# FIXME: this caused StopIteration
#_, file_item = vfs.resolve(repo, '/file')[-1]
_, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
file_copy = vfs.copy_item(file_item)
wvpass(file_copy is not file_item)
wvpass(file_copy.meta is not file_item.meta)
wvpass(isinstance(file_copy, tuple))
wvpass(file_item.meta.user)
wvpass(file_copy.meta.user)
file_copy.meta.user = None
wvpass(file_item.meta.user)
def test_encrypted_container():
with no_lingering_errors(), create_test_config() as (tmpdir, store):
secret = libnacl.public.SecretKey()
p = encrypted.EncryptedContainer(store,
b'test.pack',
'w',
Kind.DATA,
compression=9,
key=secret.pk)
p.finish()
pfile = open(os.path.join(tmpdir, b'repo', b'test.pack'), 'rb')
# minimum file size with header and footer
wvpasseq(len(pfile.read()), 92)
p = encrypted.EncryptedContainer(store,
b'test2.pack',
'w',
Kind.DATA,
compression=9,
key=secret.pk)
offsets = {}
offsets[b'A'] = p.write(3, None, b'A' * 1000)
offsets[b'B'] = p.write(3, None, b'B' * 1000)
offsets[b'C'] = p.write(3, None, b'C' * 1000)
offsets[b'D'] = p.write(3, None, b'D' * 1000)
offsets[b'ABCD'] = p.write(3, None, b'ABCD' * 250)
sk = p.box.sk
p.finish()
pfile = open(os.path.join(tmpdir, b'repo', b'test2.pack'), 'rb')
pdata = pfile.read()
# the simple stuff above compresses well
wvpasseq(len(pdata), 265)
# check header
wvpasseq(struct.unpack('<4sBBH', pdata[:8]), (b'BUPe', 1, 0, 84))
# check secret header
eh = libnacl.sealed.SealedBox(secret).decrypt(pdata[8:84 + 8])
wvpasseq(struct.unpack('<BBBB', eh[:4]), (1, 1, 1, 1))
# ignore vuint_key here, it's random
wvpasseq(sk, eh[4:])
# read the objects and check if they're fine
p = encrypted.EncryptedContainer(store,
b'test2.pack',
'r',
Kind.DATA,
key=secret)
for k in sorted(offsets.keys()):
wvpasseq(p.read(offsets[k]), (3, k * (1000 // len(k))))
p.close()
# this does some extra checks - do it explicitly
store.close()
def prep_and_test_repo(name, create_repo, test_repo):
with no_lingering_errors():
with test_tempdir('bup-t' + name) as tmpdir:
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
environ['BUP_MAIN_EXE'] = bup_path
ex((bup_path, 'init'))
git.repodir = bup_dir
with create_repo(bup_dir) as repo:
test_repo(repo, tmpdir)
def test_item_read_write():
with no_lingering_errors():
x = vfs.Root(meta=13)
stream = BytesIO()
vfs.write_item(stream, x)
print('stream:',
repr(stream.getvalue()),
stream.tell(),
file=sys.stderr)
stream.seek(0)
wvpasseq(x, vfs.read_item(stream))
def index_basic():
with no_lingering_errors():
cd = os.path.realpath('../../../t')
WVPASS(cd)
sd = os.path.realpath(cd + '/sampledata')
WVPASSEQ(resolve_parent(cd + '/sampledata'), sd)
WVPASSEQ(os.path.realpath(cd + '/sampledata/x'), sd + '/x')
WVPASSEQ(os.path.realpath(cd + '/sampledata/var/abs-symlink'),
sd + '/var/abs-symlink-target')
WVPASSEQ(resolve_parent(cd + '/sampledata/var/abs-symlink'),
sd + '/var/abs-symlink')
def index_basic():
with no_lingering_errors():
cd = os.path.realpath('../../../t')
WVPASS(cd)
sd = os.path.realpath(cd + '/sampledata')
WVPASSEQ(resolve_parent(cd + '/sampledata'), sd)
WVPASSEQ(os.path.realpath(cd + '/sampledata/x'), sd + '/x')
WVPASSEQ(os.path.realpath(cd + '/sampledata/var/abs-symlink'),
sd + '/var/abs-symlink-target')
WVPASSEQ(resolve_parent(cd + '/sampledata/var/abs-symlink'),
sd + '/var/abs-symlink')
def test_abbreviate():
with no_lingering_errors():
l1 = [b"1234", b"1235", b"1236"]
WVPASSEQ(tree._tree_names_abbreviate(l1), l1)
l2 = [b"aaaa", b"bbbb", b"cccc"]
WVPASSEQ(tree._tree_names_abbreviate(l2), [b'a', b'b', b'c'])
l3 = [b".bupm"]
WVPASSEQ(tree._tree_names_abbreviate(l3), [b'.b'])
l4 = [b"..strange..name"]
WVPASSEQ(tree._tree_names_abbreviate(l4), [b'..s'])
l5 = [b"justone"]
WVPASSEQ(tree._tree_names_abbreviate(l5), [b'j'])
def test_fanout_behaviour():
with no_lingering_errors():
global hashbits
global fanout
levels = lambda data: [(len(b), l)
for b, l in hashsplit.hashsplit_iter(
[BytesIO(data)], True, None, fanout=fanout)]
def hslevels(data):
global hashbits
global fanout
return [(len(b), l)
for b, l in HashSplitter([BytesIO(data)],
bits=hashbits,
fanbits=int(math.log(fanout, 2)))]
# This is a tuple of max blob size (4 << 13 bytes) and expected level (0)
# Return tuple with split content and expected level (from table)
def sb(pfx, n):
global fanout
needed = hashbits + int(math.log(fanout, 2)) * n
# internal algorithm ignores one bit after the split bits,
# adjust for that (if n > 0):
if n:
needed += 1
return (b'\x00' * pfx + split_test_objs[needed], n)
def end(n):
return (b'\x00' * n, 0)
# check a given sequence is handled correctly
def check(objs):
# old API allows only hashbits == 13
if hashbits == 13:
WVPASSEQ(levels(b''.join([x[0] for x in objs])),
[(len(x[0]), x[1]) for x in objs])
WVPASSEQ(hslevels(b''.join([x[0] for x in objs])),
[(len(x[0]), x[1]) for x in objs])
for hashbits in (13, 14, 15):
max_blob = (b'\x00' * (4 << hashbits), 0)
for fanout in (2, 4):
# never split - just max blobs
check([max_blob] * 4)
check([sb(0, 0)])
check([max_blob, sb(1, 3), max_blob])
check([sb(13, 1)])
check([sb(13, 1), end(200)])
fanout = 2
check([sb(0, 1), sb(30, 2), sb(20, 0), sb(10, 5)])
check([sb(0, 1), sb(30, 2), sb(20, 0), sb(10, 5), end(10)])
def test_list_refs():
with no_lingering_errors():
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
src = tmpdir + b'/src'
mkdirp(src)
with open(src + b'/1', 'wb+') as f:
f.write(b'something\n')
with open(src + b'/2', 'wb+') as f:
f.write(b'something else\n')
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, b'index', src)
exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
src_hash = exo(b'git', b'--git-dir', bupdir, b'rev-parse',
b'src').strip().split(b'\n')
assert (len(src_hash) == 1)
src_hash = unhexlify(src_hash[0])
tree_hash = unhexlify(
exo(b'git', b'--git-dir', bupdir, b'rev-parse',
b'src:').strip().split(b'\n')[0])
blob_hash = unhexlify(
exo(b'git', b'--git-dir', bupdir, b'rev-parse',
b'src:1').strip().split(b'\n')[0])
WVPASSEQ(frozenset(git.list_refs()),
frozenset([(b'refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([(b'refs/heads/src', src_hash)]))
exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
WVPASSEQ(
frozenset(git.list_refs()),
frozenset([(b'refs/heads/src', src_hash),
(b'refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([(b'refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([(b'refs/heads/src', src_hash)]))
exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
os.unlink(bupdir + b'/refs/heads/src')
expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
(b'refs/tags/tree-tag', tree_hash),
(b'refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
expected_tags)
def test_list_refs():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
src_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src').strip().split('\n')
assert (len(src_hash) == 1)
src_hash = src_hash[0].decode('hex')
tree_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src:').strip().split('\n')[0].decode('hex')
blob_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src:1').strip().split('\n')[0].decode('hex')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'commit-tag', 'src')
WVPASSEQ(
frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash),
('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'tree-tag', 'src:')
exc('git', '--git-dir', bupdir, 'tag', 'blob-tag', 'src:1')
os.unlink(bupdir + '/refs/heads/src')
expected_tags = frozenset([('refs/tags/commit-tag', src_hash),
('refs/tags/tree-tag', tree_hash),
('refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
expected_tags)
def test_bvec():
with no_lingering_errors():
values = (b'', b'x', b'foo', b'\0', b'\0foo', b'foo\0bar\0')
for x in values:
WVPASSEQ(encode_and_decode_bvec(x), x)
WVEXCEPT(EOFError, vint.read_bvec, BytesIO())
outf = BytesIO()
for x in (b'foo', b'bar', b'baz', b'bax'):
vint.write_bvec(outf, x)
inf = BytesIO(outf.getvalue())
WVPASSEQ(vint.read_bvec(inf), b'foo')
WVPASSEQ(vint.read_bvec(inf), b'bar')
vint.skip_bvec(inf)
WVPASSEQ(vint.read_bvec(inf), b'bax')
def test_next():
with no_lingering_errors():
# Test whatever you end up with for next() after import '*'.
WVPASSEQ(next(iter([]), None), None)
x = iter([1])
WVPASSEQ(next(x, None), 1)
WVPASSEQ(next(x, None), None)
x = iter([1])
WVPASSEQ(next(x, 'x'), 1)
WVPASSEQ(next(x, 'x'), 'x')
WVEXCEPT(StopIteration, next, iter([]))
x = iter([1])
WVPASSEQ(next(x), 1)
WVEXCEPT(StopIteration, next, x)
def test_next():
with no_lingering_errors():
# Test whatever you end up with for next() after import '*'.
WVPASSEQ(next(iter([]), None), None)
x = iter([1])
WVPASSEQ(next(x, None), 1)
WVPASSEQ(next(x, None), None)
x = iter([1])
WVPASSEQ(next(x, 'x'), 1)
WVPASSEQ(next(x, 'x'), 'x')
WVEXCEPT(StopIteration, next, iter([]))
x = iter([1])
WVPASSEQ(next(x), 1)
WVEXCEPT(StopIteration, next, x)
def test_bvec():
with no_lingering_errors():
values = ('', 'x', 'foo', '\0', '\0foo', 'foo\0bar\0')
for x in values:
WVPASSEQ(encode_and_decode_bvec(x), x)
WVEXCEPT(EOFError, vint.read_bvec, BytesIO())
outf = BytesIO()
for x in ('foo', 'bar', 'baz', 'bax'):
vint.write_bvec(outf, x)
inf = BytesIO(outf.getvalue())
WVPASSEQ(vint.read_bvec(inf), 'foo')
WVPASSEQ(vint.read_bvec(inf), 'bar')
vint.skip_bvec(inf)
WVPASSEQ(vint.read_bvec(inf), 'bax')
def test_bloom():
with no_lingering_errors():
with test_tempdir('bup-tbloom-') as tmpdir:
hashes = [os.urandom(20) for i in range(100)]
class Idx:
pass
ix = Idx()
ix.name='dummy.idx'
ix.shatable = ''.join(hashes)
for k in (4, 5):
b = bloom.create(tmpdir + '/pybuptest.bloom', expected=100, k=k)
b.add_idx(ix)
WVPASSLT(b.pfalse_positive(), .1)
b.close()
b = bloom.ShaBloom(tmpdir + '/pybuptest.bloom')
all_present = True
for h in hashes:
all_present &= b.exists(h)
WVPASS(all_present)
false_positives = 0
for h in [os.urandom(20) for i in range(1000)]:
if b.exists(h):
false_positives += 1
WVPASSLT(false_positives, 5)
os.unlink(tmpdir + '/pybuptest.bloom')
tf = tempfile.TemporaryFile(dir=tmpdir)
b = bloom.create('bup.bloom', f=tf, expected=100)
WVPASSEQ(b.rwfile, tf)
WVPASSEQ(b.k, 5)
# Test large (~1GiB) filter. This may fail on s390 (31-bit
# architecture), and anywhere else where the address space is
# sufficiently limited.
tf = tempfile.TemporaryFile(dir=tmpdir)
skip_test = False
try:
b = bloom.create('bup.bloom', f=tf, expected=2**28,
delaywrite=False)
except EnvironmentError as ex:
(ptr_width, linkage) = platform.architecture()
if ptr_width == '32bit' and ex.errno == errno.ENOMEM:
WVMSG('skipping large bloom filter test (mmap probably failed) '
+ str(ex))
skip_test = True
else:
raise
if not skip_test:
WVPASSEQ(b.k, 4)
def test_bup_lutimes():
if not xstat._bup_lutimes:
return
with no_lingering_errors(), test_tempdir('bup-txstat-') as tmpdir:
path = tmpdir + '/foo'
open(path, 'w').close()
frac_ts = (0, 10**6 / 2)
xstat._bup_lutimes(path, (frac_ts, frac_ts))
st = _helpers.stat(path)
atime_ts = st[8]
mtime_ts = st[9]
WVPASSEQ(atime_ts[0], 0)
WVPASS(atime_ts[1] == 0 or atime_ts[1] == frac_ts[1] * 1000)
WVPASSEQ(mtime_ts[0], 0)
WVPASS(mtime_ts[1] == 0 or mtime_ts[1] == frac_ts[1] * 1000)
def testencode():
with no_lingering_errors():
s = 'hello world'
looseb = ''.join(git._encode_looseobj('blob', s))
looset = ''.join(git._encode_looseobj('tree', s))
loosec = ''.join(git._encode_looseobj('commit', s))
packb = ''.join(git._encode_packobj('blob', s))
packt = ''.join(git._encode_packobj('tree', s))
packc = ''.join(git._encode_packobj('commit', s))
WVPASSEQ(git._decode_looseobj(looseb), ('blob', s))
WVPASSEQ(git._decode_looseobj(looset), ('tree', s))
WVPASSEQ(git._decode_looseobj(loosec), ('commit', s))
WVPASSEQ(git._decode_packobj(packb), ('blob', s))
WVPASSEQ(git._decode_packobj(packt), ('tree', s))
WVPASSEQ(git._decode_packobj(packc), ('commit', s))
def test_pack_and_unpack():
with no_lingering_errors():
tests = [('', []), ('s', ['foo']), ('ss', ['foo', 'bar']),
('sV', ['foo', 0]), ('sv', ['foo', -1]), ('V', [0]),
('Vs', [0, 'foo']), ('VV', [0, 1]), ('Vv', [0,
-1]), ('v', [0]),
('vs', [0, 'foo']), ('vV', [0, 1]), ('vv', [0, -1])]
for test in tests:
(types, values) = test
WVPASSEQ(pack_and_unpack(types, *values), values)
WVEXCEPT(Exception, vint.pack, 's')
WVEXCEPT(Exception, vint.pack, 's', 'foo', 'bar')
WVEXCEPT(Exception, vint.pack, 'x', 1)
WVEXCEPT(Exception, vint.unpack, 's', '')
WVEXCEPT(Exception, vint.unpack, 'x', '')
def test_fstime():
with no_lingering_errors():
WVPASSEQ(xstat.timespec_to_nsecs((0, 0)), 0)
WVPASSEQ(xstat.timespec_to_nsecs((1, 0)), 10**9)
WVPASSEQ(xstat.timespec_to_nsecs((0, 10**9 / 2)), 500000000)
WVPASSEQ(xstat.timespec_to_nsecs((1, 10**9 / 2)), 1500000000)
WVPASSEQ(xstat.timespec_to_nsecs((-1, 0)), -10**9)
WVPASSEQ(xstat.timespec_to_nsecs((-1, 10**9 / 2)), -500000000)
WVPASSEQ(xstat.timespec_to_nsecs((-2, 10**9 / 2)), -1500000000)
WVPASSEQ(xstat.timespec_to_nsecs((0, -1)), -1)
WVPASSEQ(type(xstat.timespec_to_nsecs((2, 22222222))), type(0))
WVPASSEQ(type(xstat.timespec_to_nsecs((-2, 22222222))), type(0))
WVPASSEQ(xstat.nsecs_to_timespec(0), (0, 0))
WVPASSEQ(xstat.nsecs_to_timespec(10**9), (1, 0))
WVPASSEQ(xstat.nsecs_to_timespec(500000000), (0, 10**9 / 2))
WVPASSEQ(xstat.nsecs_to_timespec(1500000000), (1, 10**9 / 2))
WVPASSEQ(xstat.nsecs_to_timespec(-10**9), (-1, 0))
WVPASSEQ(xstat.nsecs_to_timespec(-500000000), (-1, 10**9 / 2))
WVPASSEQ(xstat.nsecs_to_timespec(-1500000000), (-2, 10**9 / 2))
x = xstat.nsecs_to_timespec(1977777778)
WVPASSEQ(type(x[0]), type(0))
WVPASSEQ(type(x[1]), type(0))
x = xstat.nsecs_to_timespec(-1977777778)
WVPASSEQ(type(x[0]), type(0))
WVPASSEQ(type(x[1]), type(0))
WVPASSEQ(xstat.nsecs_to_timeval(0), (0, 0))
WVPASSEQ(xstat.nsecs_to_timeval(10**9), (1, 0))
WVPASSEQ(xstat.nsecs_to_timeval(500000000), (0, (10**9 / 2) / 1000))
WVPASSEQ(xstat.nsecs_to_timeval(1500000000), (1, (10**9 / 2) / 1000))
WVPASSEQ(xstat.nsecs_to_timeval(-10**9), (-1, 0))
WVPASSEQ(xstat.nsecs_to_timeval(-500000000), (-1, (10**9 / 2) / 1000))
WVPASSEQ(xstat.nsecs_to_timeval(-1500000000), (-2, (10**9 / 2) / 1000))
x = xstat.nsecs_to_timeval(1977777778)
WVPASSEQ(type(x[0]), type(0))
WVPASSEQ(type(x[1]), type(0))
x = xstat.nsecs_to_timeval(-1977777778)
WVPASSEQ(type(x[0]), type(0))
WVPASSEQ(type(x[1]), type(0))
WVPASSEQ(xstat.fstime_floor_secs(0), 0)
WVPASSEQ(xstat.fstime_floor_secs(10**9 / 2), 0)
WVPASSEQ(xstat.fstime_floor_secs(10**9), 1)
WVPASSEQ(xstat.fstime_floor_secs(-10**9 / 2), -1)
WVPASSEQ(xstat.fstime_floor_secs(-10**9), -1)
WVPASSEQ(type(xstat.fstime_floor_secs(10**9 / 2)), type(0))
WVPASSEQ(type(xstat.fstime_floor_secs(-10**9 / 2)), type(0))
def test_fanout_behaviour():
# Drop in replacement for bupsplit, but splitting if the int value of a
# byte >= BUP_BLOBBITS
basebits = _helpers.blobbits()
def splitbuf(buf):
ofs = 0
for b in buf:
b = byte_int(b)
ofs += 1
if b >= basebits:
return ofs, b
return 0, 0
with no_lingering_errors():
old_splitbuf = _helpers.splitbuf
_helpers.splitbuf = splitbuf
old_BLOB_MAX = hashsplit.BLOB_MAX
hashsplit.BLOB_MAX = 4
old_BLOB_READ_SIZE = hashsplit.BLOB_READ_SIZE
hashsplit.BLOB_READ_SIZE = 10
old_fanout = hashsplit.fanout
hashsplit.fanout = 2
levels = lambda f: [
(len(b), l) for b, l in hashsplit.hashsplit_iter([f], True, None)
]
# Return a string of n null bytes
z = lambda n: b'\x00' * n
# Return a byte which will be split with a level of n
sb = lambda n: bytes_from_uint(basebits + n)
split_never = BytesIO(z(16))
split_first = BytesIO(z(1) + sb(3) + z(14))
split_end = BytesIO(z(13) + sb(1) + z(2))
split_many = BytesIO(
sb(1) + z(3) + sb(2) + z(4) + sb(0) + z(4) + sb(5) + z(1))
WVPASSEQ(levels(split_never), [(4, 0), (4, 0), (4, 0), (4, 0)])
WVPASSEQ(levels(split_first), [(2, 3), (4, 0), (4, 0), (4, 0), (2, 0)])
WVPASSEQ(levels(split_end), [(4, 0), (4, 0), (4, 0), (2, 1), (2, 0)])
WVPASSEQ(levels(split_many), [(1, 1), (4, 2), (4, 0), (1, 0), (4, 0),
(1, 5), (1, 0)])
_helpers.splitbuf = old_splitbuf
hashsplit.BLOB_MAX = old_BLOB_MAX
hashsplit.BLOB_READ_SIZE = old_BLOB_READ_SIZE
hashsplit.fanout = old_fanout
def test_server_split_with_indexes():
with no_lingering_errors():
with test_tempdir(b'bup-tclient-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
lw.new_blob(s1)
lw.close()
rw.new_blob(s2)
rw.breakpoint()
rw.new_blob(s1)
rw.close()
def test_bup_utimensat():
if not xstat._bup_utimensat:
return
with no_lingering_errors():
with test_tempdir('bup-txstat-') as tmpdir:
path = tmpdir + '/foo'
open(path, 'w').close()
frac_ts = (0, 10**9 / 2)
xstat._bup_utimensat(_helpers.AT_FDCWD, path, (frac_ts, frac_ts), 0)
st = _helpers.stat(path)
atime_ts = st[8]
mtime_ts = st[9]
WVPASSEQ(atime_ts[0], 0)
WVPASS(atime_ts[1] == 0 or atime_ts[1] == frac_ts[1])
WVPASSEQ(mtime_ts[0], 0)
WVPASS(mtime_ts[1] == 0 or mtime_ts[1] == frac_ts[1])
def test_from_path_error():
if is_superuser() or detect_fakeroot():
return
with no_lingering_errors(), test_tempdir('bup-tmetadata-') as tmpdir:
path = tmpdir + '/foo'
os.mkdir(path)
m = metadata.from_path(path, archive_path=path, save_symlinks=True)
WVPASSEQ(m.path, path)
os.chmod(path, 000)
metadata.from_path(path, archive_path=path, save_symlinks=True)
if metadata.get_linux_file_attr:
print >> sys.stderr, 'saved_errors:', helpers.saved_errors
WVPASS(len(helpers.saved_errors) == 1)
errmsg = _first_err()
WVPASS(errmsg.startswith('read Linux attr'))
clear_errors()
def test_read_and_seek():
# Write a set of randomly sized files containing random data whose
# names are their sizes, and then verify that what we get back
# from the vfs when seeking and reading with various block sizes
# matches the original content.
with no_lingering_errors():
with test_tempdir(b'bup-tvfs-read-') as tmpdir:
resolve = vfs.resolve
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
git.repodir = bup_dir
repo = LocalRepo()
data_path = tmpdir + b'/src'
os.mkdir(data_path)
seed = randint(-(1 << 31), (1 << 31) - 1)
rand = Random()
rand.seed(seed)
print('test_read seed:', seed, file=sys.stderr)
max_size = 2 * 1024 * 1024
sizes = set((rand.randint(1, max_size) for _ in range(5)))
sizes.add(1)
sizes.add(max_size)
for size in sizes:
write_sized_random_content(data_path, size, seed)
ex((bup_path, b'init'))
ex((bup_path, b'index', b'-v', data_path))
ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test',
b'--strip', data_path))
read_sizes = set((rand.randint(1, max_size) for _ in range(10)))
sizes.add(1)
sizes.add(max_size)
print('test_read src sizes:', sizes, file=sys.stderr)
print('test_read read sizes:', read_sizes, file=sys.stderr)
for size in sizes:
res = resolve(repo,
b'/test/latest/' + str(size).encode('ascii'))
_, item = res[-1]
wvpasseq(size, vfs.item_size(repo, res[-1][1]))
validate_vfs_streaming_read(repo, item,
b'%s/%d' % (data_path, size),
read_sizes)
validate_vfs_seeking_read(repo, item,
b'%s/%d' % (data_path, size),
read_sizes)
def test_risky_path():
with no_lingering_errors():
risky = metadata._risky_path
WVPASS(risky('/foo'))
WVPASS(risky('///foo'))
WVPASS(risky('/../foo'))
WVPASS(risky('../foo'))
WVPASS(risky('foo/..'))
WVPASS(risky('foo/../'))
WVPASS(risky('foo/../bar'))
WVFAIL(risky('foo'))
WVFAIL(risky('foo/'))
WVFAIL(risky('foo///'))
WVFAIL(risky('./foo'))
WVFAIL(risky('foo/.'))
WVFAIL(risky('./foo/.'))
WVFAIL(risky('foo/bar'))
WVFAIL(risky('foo/./bar'))
def index_writer():
with no_lingering_errors(), test_tempdir('bup-tindex-') as tmpdir:
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
ds = xstat.stat('.')
fs = xstat.stat(lib_t_dir + '/tindex.py')
ms = index.MetaStoreWriter('index.meta.tmp')
tmax = (time.time() - 1) * 10**9
w = index.Writer('index.tmp', ms, tmax)
w.add('/var/tmp/sporky', fs, 0)
w.add('/etc/passwd', fs, 0)
w.add('/etc/', ds, 0)
w.add('/', ds, 0)
ms.close()
w.close()
finally:
os.chdir(orig_cwd)
def test_stripped_path_components():
with no_lingering_errors():
WVPASSEQ(stripped_path_components('/', []), [('', '/')])
WVPASSEQ(stripped_path_components('/', ['']), [('', '/')])
WVPASSEQ(stripped_path_components('/', ['/']), [('', '/')])
WVPASSEQ(stripped_path_components('/foo', ['/']),
[('', '/'), ('foo', '/foo')])
WVPASSEQ(stripped_path_components('/', ['/foo']), [('', '/')])
WVPASSEQ(stripped_path_components('/foo', ['/bar']),
[('', '/'), ('foo', '/foo')])
WVPASSEQ(stripped_path_components('/foo', ['/foo']), [('', '/foo')])
WVPASSEQ(stripped_path_components('/foo/bar', ['/foo']),
[('', '/foo'), ('bar', '/foo/bar')])
WVPASSEQ(stripped_path_components('/foo/bar', ['/bar', '/foo', '/baz']),
[('', '/foo'), ('bar', '/foo/bar')])
WVPASSEQ(stripped_path_components('/foo/bar/baz', ['/foo/bar/baz']),
[('', '/foo/bar/baz')])
WVEXCEPT(Exception, stripped_path_components, 'foo', [])
def test_resolve_loop():
with no_lingering_errors():
with test_tempdir('bup-tvfs-resloop-') as tmpdir:
resolve = vfs.resolve
lresolve = vfs.lresolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
repo = LocalRepo()
data_path = tmpdir + '/src'
os.mkdir(data_path)
symlink('loop', data_path + '/loop')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
data_path))
wvexcept(vfs.Loop, resolve, repo, '/test/latest/loop')
def test_remote_parsing():
with no_lingering_errors():
tests = (
(':/bup', ('file', None, None, '/bup')),
('file:///bup', ('file', None, None, '/bup')),
('192.168.1.1:/bup', ('ssh', '192.168.1.1', None, '/bup')),
('ssh://192.168.1.1:2222/bup', ('ssh', '192.168.1.1', '2222', '/bup')),
('ssh://[ff:fe::1]:2222/bup', ('ssh', 'ff:fe::1', '2222', '/bup')),
('bup://foo.com:1950', ('bup', 'foo.com', '1950', None)),
('bup://foo.com:1950/bup', ('bup', 'foo.com', '1950', '/bup')),
('bup://[ff:fe::1]/bup', ('bup', 'ff:fe::1', None, '/bup')),)
for remote, values in tests:
WVPASSEQ(client.parse_remote(remote), values)
try:
client.parse_remote('http://asdf.com/bup')
WVFAIL()
except client.ClientError:
WVPASS()
def test_dumb_client_server():
with no_lingering_errors():
with test_tempdir(b'bup-tclient-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
open(git.repo(b'bup-dumb-server'), 'w').close()
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1)
rw.new_blob(s1)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1)
rw.new_blob(s2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 2)
