def test_multiple_suggestions():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tclient-')
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
lw = git.PackWriter()
lw.new_blob(s2)
lw.close()
WVPASSEQ(len(glob.glob(git.repo('objects/pack' + IDX_PAT))), 2)
c = client.Client(bupdir, create=True)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 0)
rw = c.new_packwriter()
s1sha = rw.new_blob(s1)
WVPASS(rw.exists(s1sha))
s2sha = rw.new_blob(s2)
# This is a little hacky, but ensures that we test the code under test
while (len(glob.glob(c.cachedir + IDX_PAT)) < 2
and not c.conn.has_input()):
pass
rw.new_blob(s2)
WVPASS(rw.objcache.exists(s1sha))
WVPASS(rw.objcache.exists(s2sha))
rw.new_blob(s3)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 3)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_multiple_suggestions():
with no_lingering_errors():
with test_tempdir(b'bup-tclient-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
lw = git.PackWriter()
lw.new_blob(s2)
lw.close()
WVPASSEQ(len(glob.glob(git.repo(b'objects/pack' + IDX_PAT))), 2)
c = client.Client(bupdir, create=True)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 0)
rw = c.new_packwriter()
s1sha = rw.new_blob(s1)
WVPASS(rw.exists(s1sha))
s2sha = rw.new_blob(s2)
# This is a little hacky, but ensures that we test the
# code under test
while (len(glob.glob(c.cachedir + IDX_PAT)) < 2
and not c.conn.has_input()):
pass
rw.new_blob(s2)
WVPASS(rw.objcache.exists(s1sha))
WVPASS(rw.objcache.exists(s2sha))
rw.new_blob(s3)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 3)
def test_multiple_suggestions(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
lw = git.PackWriter()
lw.new_blob(s2)
lw.close()
assert len(glob.glob(git.repo(b'objects/pack' + IDX_PAT,
repo_dir=bupdir))) == 2
c = client.Client(bupdir, create=True)
assert len(glob.glob(c.cachedir + IDX_PAT)) == 0
rw = c.new_packwriter()
s1sha = rw.new_blob(s1)
assert rw.exists(s1sha)
s2sha = rw.new_blob(s2)
# This is a little hacky, but ensures that we test the
# code under test. First, flush to ensure that we've
# actually sent all the command ('receive-objects-v2')
# and their data to the server. This may be needed if
# the output buffer size is bigger than the data (both
# command and objects) we're writing. To see the need
# for this, change the object sizes at the beginning
# of this file to be very small (e.g. 10 instead of 10k)
c.conn.outp.flush()
# Then, check if we've already received the idx files.
# This may happen if we're preempted just after writing
# the data, then the server runs and suggests, and only
# then we continue in PackWriter_Remote::_raw_write()
# and check the has_input(), in that case we'll receive
# the idx still in the rw.new_blob() calls above.
#
# In most cases though, that doesn't happen, and we'll
# get past the has_input() check before the server has
# a chance to respond - it has to actually hash the new
# object here, so it takes some time. So also break out
# of the loop if the server has sent something on the
# connection.
#
# Finally, abort this after a little while (about one
# second) just in case something's actually broken.
n = 0
while (len(glob.glob(c.cachedir + IDX_PAT)) < 2 and not c.conn.has_input()
and n < 10):
time.sleep(0.1)
n += 1
assert len(glob.glob(c.cachedir + IDX_PAT)) == 2 or c.conn.has_input()
rw.new_blob(s2)
assert rw.objcache.exists(s1sha)
assert rw.objcache.exists(s2sha)
rw.new_blob(s3)
assert len(glob.glob(c.cachedir + IDX_PAT)) == 2
rw.close()
assert len(glob.glob(c.cachedir + IDX_PAT)) == 3
def receive_objects_v2(conn, junk):
global suspended_w
_init_session()
suggested = set()
if suspended_w:
w = suspended_w
suspended_w = None
else:
if dumb_server_mode:
w = git.PackWriter(objcache_maker=None)
else:
w = git.PackWriter()
while 1:
ns = conn.read(4)
if not ns:
w.abort()
raise Exception('object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
conn.write('%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
suspended_w = w
conn.ok()
return
shar = conn.read(20)
crcr = struct.unpack('!I', conn.read(4))[0]
n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
_check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
if not dumb_server_mode:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
assert(oldpack.endswith('.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
% git.shorten_hash(name))
debug1("bup server: because of object %s\n"
% shar.encode('hex'))
conn.write('index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
_check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def test_new_commit():
with no_lingering_errors():
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = b'Author'
author_mail = b'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = b'Committer'
committer_mail = b'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
b'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
b'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
b'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
b'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
b'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
b'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(hexlify(commit), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, unhexlify(commit_items.tree))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
WVPASSEQ(tree, unhexlify(commit_items.tree))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def test_long_index():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
w = git.PackWriter()
obj_bin = struct.pack('!IIIII', 0x00112233, 0x44556677, 0x88990011,
0x22334455, 0x66778899)
obj2_bin = struct.pack('!IIIII', 0x11223344, 0x55667788,
0x99001122, 0x33445566, 0x77889900)
obj3_bin = struct.pack('!IIIII', 0x22334455, 0x66778899,
0x00112233, 0x44556677, 0x88990011)
pack_bin = struct.pack('!IIIII', 0x99887766, 0x55443322,
0x11009988, 0x77665544, 0x33221100)
idx = list(list() for i in xrange(256))
idx[0].append((obj_bin, 1, 0xfffffffff))
idx[0x11].append((obj2_bin, 2, 0xffffffffff))
idx[0x22].append((obj3_bin, 3, 0xff))
w.count = 3
name = tmpdir + '/tmp.idx'
r = w._write_pack_idx_v2(name, idx, pack_bin)
i = git.PackIdxV2(name, open(name, 'rb'))
WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
WVPASSEQ(i.find_offset(obj3_bin), 0xff)
def _ensure_packwriter(self):
if not self._packwriter:
self._packwriter = git.PackWriter(repo_dir=self.repo_dir,
compression_level=self.compression_level,
max_pack_size=self.max_pack_size,
max_pack_objects=self.max_pack_objects,
objcache_maker=self.objcache_maker)
def test_pack_name_lookup():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
packdir = git.repo('objects/pack')
idxnames = []
hashes = []
for start in range(0, 28, 2):
w = git.PackWriter()
for i in range(start, start + 2):
hashes.append(w.new_blob(str(i)))
log('\n')
idxnames.append(os.path.basename(w.close() + '.idx'))
r = git.PackIdxList(packdir)
WVPASSEQ(len(r.packs), 2)
for e, idxname in enumerate(idxnames):
for i in range(e * 2, (e + 1) * 2):
WVPASSEQ(r.exists(hashes[i], want_source=True), idxname)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_new_commit():
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = 'Author'
author_mail = 'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = 'Committer'
committer_mail = 'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(commit.encode('hex'), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp())
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def test_long_index():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
w = git.PackWriter()
obj_bin = struct.pack('!IIIII', 0x00112233, 0x44556677, 0x88990011,
0x22334455, 0x66778899)
obj2_bin = struct.pack('!IIIII', 0x11223344, 0x55667788, 0x99001122,
0x33445566, 0x77889900)
obj3_bin = struct.pack('!IIIII', 0x22334455, 0x66778899, 0x00112233,
0x44556677, 0x88990011)
pack_bin = struct.pack('!IIIII', 0x99887766, 0x55443322, 0x11009988,
0x77665544, 0x33221100)
idx = list(list() for i in xrange(256))
idx[0].append((obj_bin, 1, 0xfffffffff))
idx[0x11].append((obj2_bin, 2, 0xffffffffff))
idx[0x22].append((obj3_bin, 3, 0xff))
(fd, name) = tempfile.mkstemp(suffix='.idx', dir=git.repo('objects'))
os.close(fd)
w.count = 3
r = w._write_pack_idx_v2(name, idx, pack_bin)
i = git.PackIdxV2(name, open(name, 'rb'))
WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
WVPASSEQ(i.find_offset(obj3_bin), 0xff)
if wvfailure_count() == initial_failures:
os.remove(name)
subprocess.call(['rm', '-rf', tmpdir])
def new_packwriter(self,
compression_level=1,
max_pack_size=None,
max_pack_objects=None):
return git.PackWriter(repo_dir=self.repo_dir,
compression_level=compression_level,
max_pack_size=max_pack_size,
max_pack_objects=max_pack_objects)
def testpacks():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
w.new_blob(os.urandom(100))
w.new_blob(os.urandom(100))
w.abort()
w = git.PackWriter()
hashes = []
nobj = 1000
for i in range(nobj):
hashes.append(w.new_blob(str(i)))
log('\n')
nameprefix = w.close()
print repr(nameprefix)
WVPASS(os.path.exists(nameprefix + '.pack'))
WVPASS(os.path.exists(nameprefix + '.idx'))
r = git.open_idx(nameprefix + '.idx')
print repr(r.fanout)
for i in range(nobj):
WVPASS(r.find_offset(hashes[i]) > 0)
WVPASS(r.exists(hashes[99]))
WVFAIL(r.exists('\0' * 20))
pi = iter(r)
for h in sorted(hashes):
WVPASSEQ(str(pi.next()).encode('hex'), h.encode('hex'))
WVFAIL(r.find_offset('\0' * 20))
r = git.PackIdxList(bupdir + '/objects/pack')
WVPASS(r.exists(hashes[5]))
WVPASS(r.exists(hashes[6]))
WVFAIL(r.exists('\0' * 20))
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_server_split_with_indexes(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
with git.PackWriter() as lw:
lw.new_blob(s1)
with client.Client(bupdir, create=True) as c, \
c.new_packwriter() as rw:
rw.new_blob(s2)
rw.breakpoint()
rw.new_blob(s1)
def testpacks():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
w.new_blob(os.urandom(100))
w.new_blob(os.urandom(100))
w.abort()
w = git.PackWriter()
hashes = []
nobj = 1000
for i in range(nobj):
hashes.append(w.new_blob(str(i)))
log('\n')
nameprefix = w.close()
print repr(nameprefix)
WVPASS(os.path.exists(nameprefix + '.pack'))
WVPASS(os.path.exists(nameprefix + '.idx'))
r = git.open_idx(nameprefix + '.idx')
print repr(r.fanout)
for i in range(nobj):
WVPASS(r.find_offset(hashes[i]) > 0)
WVPASS(r.exists(hashes[99]))
WVFAIL(r.exists('\0' * 20))
pi = iter(r)
for h in sorted(hashes):
WVPASSEQ(str(pi.next()).encode('hex'), h.encode('hex'))
WVFAIL(r.find_offset('\0' * 20))
r = git.PackIdxList(bupdir + '/objects/pack')
WVPASS(r.exists(hashes[5]))
WVPASS(r.exists(hashes[6]))
WVFAIL(r.exists('\0' * 20))
def testpacks():
os.environ['BUP_MAIN_EXE'] = bupmain = '../../../bup'
os.environ['BUP_DIR'] = bupdir = 'pybuptest.tmp'
subprocess.call(['rm', '-rf', bupdir])
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
w.new_blob(os.urandom(100))
w.new_blob(os.urandom(100))
w.abort()
w = git.PackWriter()
hashes = []
nobj = 1000
for i in range(nobj):
hashes.append(w.new_blob(str(i)))
log('\n')
nameprefix = w.close()
print repr(nameprefix)
WVPASS(os.path.exists(nameprefix + '.pack'))
WVPASS(os.path.exists(nameprefix + '.idx'))
r = git.open_idx(nameprefix + '.idx')
print repr(r.fanout)
for i in range(nobj):
WVPASS(r.find_offset(hashes[i]) > 0)
WVPASS(r.exists(hashes[99]))
WVFAIL(r.exists('\0' * 20))
pi = iter(r)
for h in sorted(hashes):
WVPASSEQ(str(pi.next()).encode('hex'), h.encode('hex'))
WVFAIL(r.find_offset('\0' * 20))
r = git.PackIdxList('pybuptest.tmp/objects/pack')
WVPASS(r.exists(hashes[5]))
WVPASS(r.exists(hashes[6]))
WVFAIL(r.exists('\0' * 20))
def testpacks():
with no_lingering_errors():
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
w.new_blob(os.urandom(100))
w.new_blob(os.urandom(100))
w.abort()
w = git.PackWriter()
hashes = []
nobj = 1000
for i in range(nobj):
hashes.append(w.new_blob(b'%d' % i))
log('\n')
nameprefix = w.close()
print(repr(nameprefix))
WVPASS(os.path.exists(nameprefix + b'.pack'))
WVPASS(os.path.exists(nameprefix + b'.idx'))
r = git.open_idx(nameprefix + b'.idx')
print(repr(r.fanout))
for i in range(nobj):
WVPASS(r.find_offset(hashes[i]) > 0)
WVPASS(r.exists(hashes[99]))
WVFAIL(r.exists(b'\0'*20))
pi = iter(r)
for h in sorted(hashes):
WVPASSEQ(hexlify(next(pi)), hexlify(h))
WVFAIL(r.find_offset(b'\0'*20))
r = git.PackIdxList(bupdir + b'/objects/pack')
WVPASS(r.exists(hashes[5]))
WVPASS(r.exists(hashes[6]))
WVFAIL(r.exists(b'\0'*20))
def bup_rm(paths, compression=6, verbosity=None):
root = vfs.RefList(None)
dead_branches, dead_saves = dead_items(root, paths)
die_if_errors('not proceeding with any removals\n')
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
for branch, node in dead_branches.iteritems():
ref = 'refs/heads/' + branch
assert(not ref in updated_refs)
updated_refs[ref] = (node.hash, None)
if dead_saves:
writer = git.PackWriter(compression_level=compression)
try:
for branch, saves in dead_saves.iteritems():
assert(saves)
updated_refs['refs/heads/' + branch] = rm_saves(saves, writer)
except:
if writer:
writer.abort()
raise
else:
if writer:
# Must close before we can update the ref(s) below.
writer.close()
# Only update the refs here, at the very end, so that if something
# goes wrong above, the old refs will be undisturbed. Make an attempt
# to update each ref.
for ref_name, info in updated_refs.iteritems():
orig_ref, new_ref = info
try:
if not new_ref:
git.delete_ref(ref_name, orig_ref.encode('hex'))
else:
git.update_ref(ref_name, new_ref, orig_ref)
if verbosity:
new_hex = new_ref.encode('hex')
if orig_ref:
orig_hex = orig_ref.encode('hex')
log('updated %r (%s -> %s)\n'
% (ref_name, orig_hex, new_hex))
else:
log('updated %r (%s)\n' % (ref_name, new_hex))
except (git.GitError, ClientError) as ex:
if new_ref:
add_error('while trying to update %r (%s -> %s): %s'
% (ref_name, orig_ref, new_ref, ex))
else:
add_error('while trying to delete %r (%s): %s'
% (ref_name, orig_ref, ex))
def test_server_split_with_indexes(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
lw.new_blob(s1)
lw.close()
rw.new_blob(s2)
rw.breakpoint()
rw.new_blob(s1)
rw.close()
def test_server_split_with_indexes():
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = 'buptest_tclient.tmp'
subprocess.call(['rm', '-rf', bupdir])
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
lw.new_blob(s1)
lw.close()
rw.new_blob(s2)
rw.breakpoint()
rw.new_blob(s1)
def test_dumb_client_server(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
open(git.repo(b'bup-dumb-server'), 'w').close()
with git.PackWriter() as lw:
lw.new_blob(s1)
with client.Client(bupdir, create=True) as c, \
c.new_packwriter() as rw:
assert len(glob.glob(c.cachedir + IDX_PAT)) == 1
rw.new_blob(s1)
assert len(glob.glob(c.cachedir + IDX_PAT)) == 1
rw.new_blob(s2)
assert len(glob.glob(c.cachedir + IDX_PAT)) == 2
def bup_rm(repo, paths, compression=6, verbosity=None):
dead_branches, dead_saves = dead_items(repo, paths)
die_if_errors('not proceeding with any removals\n')
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
for branchname, branchitem in dead_branches.items():
ref = b'refs/heads/' + branchname
assert(not ref in updated_refs)
updated_refs[ref] = (branchitem.oid, None)
if dead_saves:
writer = git.PackWriter(compression_level=compression)
try:
for branch, saves in dead_saves.items():
assert(saves)
updated_refs[b'refs/heads/' + branch] = rm_saves(saves, writer)
except BaseException as ex:
with pending_raise(ex):
writer.abort()
finally:
writer.close()
# Only update the refs here, at the very end, so that if something
# goes wrong above, the old refs will be undisturbed. Make an attempt
# to update each ref.
for ref_name, info in updated_refs.items():
orig_ref, new_ref = info
try:
if not new_ref:
git.delete_ref(ref_name, hexlify(orig_ref))
else:
git.update_ref(ref_name, new_ref, orig_ref)
if verbosity:
log('updated %s (%s%s)\n'
% (path_msg(ref_name),
hexstr(orig_ref) + ' -> ' if orig_ref else '',
hexstr(new_ref)))
except (git.GitError, ClientError) as ex:
if new_ref:
add_error('while trying to update %s (%s%s): %s'
% (path_msg(ref_name),
hexstr(orig_ref) + ' -> ' if orig_ref else '',
hexstr(new_ref),
ex))
else:
add_error('while trying to delete %r (%s): %s'
% (ref_name, hexstr(orig_ref), ex))
def test_server_split_with_indexes():
with no_lingering_errors():
with test_tempdir(b'bup-tclient-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
lw.new_blob(s1)
lw.close()
rw.new_blob(s2)
rw.breakpoint()
rw.new_blob(s1)
rw.close()
def test_server_split_with_indexes():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tclient-')
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
lw.new_blob(s1)
lw.close()
rw.new_blob(s2)
rw.breakpoint()
rw.new_blob(s1)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_dumb_client_server():
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = 'buptest_tclient.tmp'
subprocess.call(['rm', '-rf', bupdir])
git.init_repo(bupdir)
open(git.repo('bup-dumb-server'), 'w').close()
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 1)
rw.new_blob(s1)
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 1)
rw.new_blob(s2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 2)
def test_dumb_client_server():
with no_lingering_errors():
with test_tempdir(b'bup-tclient-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
open(git.repo(b'bup-dumb-server'), 'w').close()
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1)
rw.new_blob(s1)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1)
rw.new_blob(s2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 2)
def test_pack_name_lookup(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
git.verbose = 1
packdir = git.repo(b'objects/pack')
idxnames = []
hashes = []
for start in range(0, 28, 2):
w = git.PackWriter()
for i in range(start, start + 2):
hashes.append(w.new_blob(b'%d' % i))
log('\n')
idxnames.append(os.path.basename(w.close() + b'.idx'))
r = git.PackIdxList(packdir)
WVPASSEQ(len(r.packs), 2)
for e, idxname in enumerate(idxnames):
for i in range(e * 2, (e + 1) * 2):
WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
def test_dumb_client_server(dumb_mode, tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
if dumb_mode == 'file':
open(git.repo(b'bup-dumb-server', repo_dir=bupdir), 'w').close()
elif dumb_mode == 'config':
git.git_config_write(b'bup.dumb-server', b'true', repo_dir=bupdir)
else:
assert False
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
assert len(glob.glob(c.cachedir + IDX_PAT)) == 1
rw.new_blob(s1)
assert len(glob.glob(c.cachedir + IDX_PAT)) == 1
rw.new_blob(s2)
rw.close()
assert len(glob.glob(c.cachedir + IDX_PAT)) == 2
def test_dumb_client_server():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tclient-')
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
open(git.repo('bup-dumb-server'), 'w').close()
lw = git.PackWriter()
lw.new_blob(s1)
lw.close()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1)
rw.new_blob(s1)
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 1)
rw.new_blob(s2)
rw.close()
WVPASSEQ(len(glob.glob(c.cachedir + IDX_PAT)), 2)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_long_index():
w = git.PackWriter()
obj_bin = struct.pack('!IIIII',
0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
obj2_bin = struct.pack('!IIIII',
0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
obj3_bin = struct.pack('!IIIII',
0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
pack_bin = struct.pack('!IIIII',
0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
idx = list(list() for i in xrange(256))
idx[0].append((obj_bin, 1, 0xfffffffff))
idx[0x11].append((obj2_bin, 2, 0xffffffffff))
idx[0x22].append((obj3_bin, 3, 0xff))
(fd,name) = tempfile.mkstemp(suffix='.idx', dir=git.repo('objects'))
os.close(fd)
w.count = 3
r = w._write_pack_idx_v2(name, idx, pack_bin)
i = git.PackIdxV2(name, open(name, 'rb'))
WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
WVPASSEQ(i.find_offset(obj3_bin), 0xff)
os.remove(name)
def test_pack_name_lookup():
os.environ['BUP_MAIN_EXE'] = bupmain = '../../../bup'
os.environ['BUP_DIR'] = bupdir = 'pybuptest.tmp'
subprocess.call(['rm','-rf', bupdir])
git.init_repo(bupdir)
git.verbose = 1
packdir = git.repo('objects/pack')
idxnames = []
hashes = []
for start in range(0,28,2):
w = git.PackWriter()
for i in range(start, start+2):
hashes.append(w.new_blob(str(i)))
log('\n')
idxnames.append(os.path.basename(w.close() + '.idx'))
r = git.PackIdxList(packdir)
WVPASSEQ(len(r.packs), 2)
for e,idxname in enumerate(idxnames):
for i in range(e*2, (e+1)*2):
WVPASSEQ(r.exists(hashes[i], want_source=True), idxname)