def _mksubs(self):
self._subs = {}
it = cp(self._repo_dir).get(self.hash.encode('hex'))
type = it.next()
if type == 'commit':
del it
it = cp(self._repo_dir).get(self.hash.encode('hex') + ':')
type = it.next()
assert(type == 'tree')
for (mode,mangled_name,sha) in git.tree_decode(''.join(it)):
if mangled_name == '.bupm':
bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL
self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha,
bupmode)
continue
name, bupmode = git.demangle_name(mangled_name, mode)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha, self._repo_dir)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode,
self._repo_dir)
else:
self._subs[name] = File(self, name, mode, sha, bupmode,
self._repo_dir)
def test_new_commit():
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = 'Author'
author_mail = 'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = 'Committer'
committer_mail = 'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(commit.encode('hex'), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp())
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def test_new_commit():
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = 'Author'
author_mail = 'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = 'Committer'
committer_mail = 'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(commit.encode('hex'), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp())
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def test_new_commit():
with no_lingering_errors():
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = b'Author'
author_mail = b'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = b'Committer'
committer_mail = b'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
b'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
b'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
b'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
b'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
b'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
b'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(hexlify(commit), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, unhexlify(commit_items.tree))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
WVPASSEQ(tree, unhexlify(commit_items.tree))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
def _mksubs(self):
self._subs = {}
it = cp(self._repo_dir).get(self.hash.encode('hex'))
type = it.next()
if type == 'commit':
del it
it = cp(self._repo_dir).get(self.hash.encode('hex') + ':')
type = it.next()
assert (type == 'tree')
for (mode, mangled_name, sha) in git.tree_decode(''.join(it)):
if mangled_name == '.bupm':
bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL
self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha,
bupmode)
continue
(name, bupmode) = git.demangle_name(mangled_name)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha, self._repo_dir)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode,
self._repo_dir)
else:
self._subs[name] = File(self, name, mode, sha, bupmode,
self._repo_dir)
def test_commit_parsing():
def showval(commit, val):
return readpipe(['git', 'show', '-s',
'--pretty=format:%s' % val, commit]).strip()
initial_failures = wvfailure_count()
orig_cwd = os.getcwd()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
workdir = tmpdir + "/work"
repodir = workdir + '/.git'
try:
readpipe(['git', 'init', workdir])
os.environ['GIT_DIR'] = os.environ['BUP_DIR'] = repodir
git.check_repo_or_die(repodir)
os.chdir(workdir)
with open('foo', 'w') as f:
print >> f, 'bar'
readpipe(['git', 'add', '.'])
readpipe(['git', 'commit', '-am', 'Do something',
'--author', 'Someone <someone@somewhere>',
'--date', 'Sat Oct 3 19:48:49 2009 -0400'])
commit = readpipe(['git', 'show-ref', '-s', 'master']).strip()
parents = showval(commit, '%P')
tree = showval(commit, '%T')
cname = showval(commit, '%cn')
cmail = showval(commit, '%ce')
cdate = showval(commit, '%ct')
coffs = showval(commit, '%ci')
coffs = coffs[-5:]
coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
if coffs[-5] == '-':
coff = - coff
commit_items = git.get_commit_items(commit, git.cp())
WVPASSEQ(commit_items.parents, [])
WVPASSEQ(commit_items.tree, tree)
WVPASSEQ(commit_items.author_name, 'Someone')
WVPASSEQ(commit_items.author_mail, 'someone@somewhere')
WVPASSEQ(commit_items.author_sec, 1254613729)
WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
WVPASSEQ(commit_items.committer_name, cname)
WVPASSEQ(commit_items.committer_mail, cmail)
WVPASSEQ(commit_items.committer_sec, int(cdate))
WVPASSEQ(commit_items.committer_offset, coff)
WVPASSEQ(commit_items.message, 'Do something\n')
with open('bar', 'w') as f:
print >> f, 'baz'
readpipe(['git', 'add', '.'])
readpipe(['git', 'commit', '-am', 'Do something else'])
child = readpipe(['git', 'show-ref', '-s', 'master']).strip()
parents = showval(child, '%P')
commit_items = git.get_commit_items(child, git.cp())
WVPASSEQ(commit_items.parents, [commit])
finally:
os.chdir(orig_cwd)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_commit_parsing():
def showval(commit, val):
return readpipe(['git', 'show', '-s',
'--pretty=format:%s' % val, commit]).strip()
initial_failures = wvfailure_count()
orig_cwd = os.getcwd()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
workdir = tmpdir + "/work"
repodir = workdir + '/.git'
try:
readpipe(['git', 'init', workdir])
os.environ['GIT_DIR'] = os.environ['BUP_DIR'] = repodir
git.check_repo_or_die(repodir)
os.chdir(workdir)
with open('foo', 'w') as f:
print >> f, 'bar'
readpipe(['git', 'add', '.'])
readpipe(['git', 'commit', '-am', 'Do something',
'--author', 'Someone <someone@somewhere>',
'--date', 'Sat Oct 3 19:48:49 2009 -0400'])
commit = readpipe(['git', 'show-ref', '-s', 'master']).strip()
parents = showval(commit, '%P')
tree = showval(commit, '%T')
cname = showval(commit, '%cn')
cmail = showval(commit, '%ce')
cdate = showval(commit, '%ct')
coffs = showval(commit, '%ci')
coffs = coffs[-5:]
coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
if coffs[-5] == '-':
coff = - coff
commit_items = git.get_commit_items(commit, git.cp())
WVPASSEQ(commit_items.parents, [])
WVPASSEQ(commit_items.tree, tree)
WVPASSEQ(commit_items.author_name, 'Someone')
WVPASSEQ(commit_items.author_mail, 'someone@somewhere')
WVPASSEQ(commit_items.author_sec, 1254613729)
WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
WVPASSEQ(commit_items.committer_name, cname)
WVPASSEQ(commit_items.committer_mail, cmail)
WVPASSEQ(commit_items.committer_sec, int(cdate))
WVPASSEQ(commit_items.committer_offset, coff)
WVPASSEQ(commit_items.message, 'Do something\n')
with open('bar', 'w') as f:
print >> f, 'baz'
readpipe(['git', 'add', '.'])
readpipe(['git', 'commit', '-am', 'Do something else'])
child = readpipe(['git', 'show-ref', '-s', 'master']).strip()
parents = showval(child, '%P')
commit_items = git.get_commit_items(child, git.cp())
WVPASSEQ(commit_items.parents, [commit])
finally:
os.chdir(orig_cwd)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def bup_gc(threshold=10, compression=1, verbosity=0):
cat_pipe = git.cp()
existing_count = count_objects(git.repo('objects/pack'), verbosity)
if verbosity:
log('found %d objects\n' % existing_count)
if not existing_count:
if verbosity:
log('nothing to collect\n')
else:
try:
live_objects = find_live_objects(existing_count, cat_pipe,
verbosity=verbosity)
except MissingObject as ex:
log('bup: missing object %r \n' % ex.id.encode('hex'))
sys.exit(1)
try:
# FIXME: just rename midxes and bloom, and restore them at the end if
# we didn't change any packs?
if verbosity: log('clearing midx files\n')
midx.clear_midxes()
if verbosity: log('clearing bloom filter\n')
bloom.clear_bloom(git.repo('objects/pack'))
if verbosity: log('clearing reflog\n')
expirelog_cmd = ['git', 'reflog', 'expire', '--all', '--expire=all']
expirelog = subprocess.Popen(expirelog_cmd, preexec_fn = git._gitenv())
git._git_wait(' '.join(expirelog_cmd), expirelog)
if verbosity: log('removing unreachable data\n')
sweep(live_objects, existing_count, cat_pipe,
threshold, compression,
verbosity)
finally:
live_objects.close()
def __init__(self, repo_dir=None):
self.closed = False
self.repo_dir = realpath(repo_dir or git.repo())
self._cp = git.cp(self.repo_dir)
self.update_ref = partial(git.update_ref, repo_dir=self.repo_dir)
self.rev_list = partial(git.rev_list, repo_dir=self.repo_dir)
self._id = _repo_id(self.repo_dir)
def bup_gc(threshold=10, compression=1, verbosity=0):
cat_pipe = git.cp()
existing_count = count_objects(git.repo(b'objects/pack'), verbosity)
if verbosity:
log('found %d objects\n' % existing_count)
if not existing_count:
if verbosity:
log('nothing to collect\n')
else:
try:
live_objects = find_live_objects(existing_count,
cat_pipe,
verbosity=verbosity)
except MissingObject as ex:
log('bup: missing object %r \n' % hexstr(ex.oid))
sys.exit(1)
try:
# FIXME: just rename midxes and bloom, and restore them at the end if
# we didn't change any packs?
packdir = git.repo(b'objects/pack')
if verbosity: log('clearing midx files\n')
midx.clear_midxes(packdir)
if verbosity: log('clearing bloom filter\n')
bloom.clear_bloom(packdir)
if verbosity: log('clearing reflog\n')
expirelog_cmd = [
b'git', b'reflog', b'expire', b'--all', b'--expire=all'
]
expirelog = subprocess.Popen(expirelog_cmd, env=git._gitenv())
git._git_wait(b' '.join(expirelog_cmd), expirelog)
if verbosity: log('removing unreachable data\n')
sweep(live_objects, existing_count, cat_pipe, threshold,
compression, verbosity)
finally:
live_objects.close()
def __init__(self, hash, isdir, startofs, repo_dir=None):
if isdir:
self.it = _chunkiter(hash, startofs, repo_dir)
self.blob = None
else:
self.it = None
self.blob = ''.join(cp(repo_dir).join(hash.encode('hex')))[startofs:]
self.ofs = startofs
def filter_branch(tip_commit_hex, exclude, writer):
# May return None if everything is excluded.
commits = [unhexlify(x) for x in git.rev_list(tip_commit_hex)]
commits.reverse()
last_c, tree = None, None
# Rather than assert that we always find an exclusion here, we'll
# just let the StopIteration signal the error.
first_exclusion = next(i for i, c in enumerate(commits) if exclude(c))
if first_exclusion != 0:
last_c = commits[first_exclusion - 1]
tree = unhexlify(get_commit_items(hexlify(last_c), git.cp()).tree)
commits = commits[first_exclusion:]
for c in commits:
if exclude(c):
continue
last_c, tree = append_commit(hexlify(c), last_c, git.cp(), writer)
return last_c
def filter_branch(tip_commit_hex, exclude, writer):
# May return None if everything is excluded.
commits = [c for _, c in git.rev_list(tip_commit_hex)]
commits.reverse()
last_c, tree = None, None
# Rather than assert that we always find an exclusion here, we'll
# just let the StopIteration signal the error.
first_exclusion = next(i for i, c in enumerate(commits) if exclude(c))
if first_exclusion != 0:
last_c = commits[first_exclusion - 1]
tree = get_commit_items(last_c.encode('hex'),
git.cp()).tree.decode('hex')
commits = commits[first_exclusion:]
for c in commits:
if exclude(c):
continue
last_c, tree = append_commit(c.encode('hex'), last_c, git.cp(), writer)
return last_c
def join(conn, id):
_init_session()
try:
for blob in git.cp().join(id):
conn.write(struct.pack('!I', len(blob)))
conn.write(blob)
except KeyError as e:
log('server: error: %s\n' % e)
conn.write('\0\0\0\0')
conn.error(e)
else:
conn.write('\0\0\0\0')
conn.ok()
def join(conn, id):
_init_session()
try:
for blob in git.cp().join(id):
conn.write(struct.pack('!I', len(blob)))
conn.write(blob)
except KeyError as e:
log('server: error: %s\n' % e)
conn.write('\0\0\0\0')
conn.error(e)
else:
conn.write('\0\0\0\0')
conn.ok()
def _mksubs(self):
self._subs = {}
it = cp(self._repo_dir).get(self.hash.encode("hex"))
type = it.next()
if type == "commit":
del it
it = cp(self._repo_dir).get(self.hash.encode("hex") + ":")
type = it.next()
assert type == "tree"
for (mode, mangled_name, sha) in git.tree_decode("".join(it)):
if mangled_name == ".bupm":
bupmode = stat.S_ISDIR(mode) and BUP_CHUNKED or BUP_NORMAL
self._bupm = File(self, mangled_name, GIT_MODE_FILE, sha, bupmode)
continue
(name, bupmode) = git.demangle_name(mangled_name)
if bupmode == git.BUP_CHUNKED:
mode = GIT_MODE_FILE
if stat.S_ISDIR(mode):
self._subs[name] = Dir(self, name, mode, sha, self._repo_dir)
elif stat.S_ISLNK(mode):
self._subs[name] = Symlink(self, name, sha, bupmode, self._repo_dir)
else:
self._subs[name] = File(self, name, mode, sha, bupmode, self._repo_dir)
def cat_batch(conn, dummy):
_init_session()
cat_pipe = git.cp()
# For now, avoid potential deadlock by just reading them all
for ref in tuple(lines_until_sentinel(conn, '\n', Exception)):
ref = ref[:-1]
it = cat_pipe.get(ref)
info = next(it)
if not info[0]:
conn.write('missing\n')
continue
conn.write('%s %s %d\n' % info)
for buf in it:
conn.write(buf)
conn.ok()
def __init__(self, repo_dir=None, compression_level=None,
max_pack_size=None, max_pack_objects=None,
objcache_maker=None):
self.repo_dir = realpath(git.guess_repo(repo_dir))
self.config = partial(git.git_config_get, repo_dir=self.repo_dir)
# init the superclass only afterwards so it can access self.config()
super(LocalRepo, self).__init__(self.repo_dir,
compression_level=compression_level,
max_pack_size=max_pack_size,
max_pack_objects=max_pack_objects)
self._cp = git.cp(self.repo_dir)
self.rev_list = partial(git.rev_list, repo_dir=self.repo_dir)
self._dumb_server_mode = None
self._packwriter = None
self.objcache_maker = objcache_maker
def cat_batch(conn, dummy):
_init_session()
cat_pipe = git.cp()
# For now, avoid potential deadlock by just reading them all
for ref in tuple(lines_until_sentinel(conn, '\n', Exception)):
ref = ref[:-1]
it = cat_pipe.get(ref)
info = next(it)
if not info[0]:
conn.write('missing\n')
continue
conn.write('%s %s %d\n' % info)
for buf in it:
conn.write(buf)
conn.ok()
def test_cat_pipe(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
src = tmpdir + b'/src'
mkdirp(src)
with open(src + b'/1', 'wb+') as f:
f.write(b'something\n')
with open(src + b'/2', 'wb+') as f:
f.write(b'something else\n')
git.init_repo(bupdir)
exc(bup_exe, b'index', src)
oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip', src).strip()
typ = exo(b'git', b'--git-dir', bupdir, b'cat-file', b'-t', b'src').strip()
size = int(exo(b'git', b'--git-dir', bupdir, b'cat-file', b'-s', b'src'))
it = git.cp().get(b'src')
get_info = next(it)
for buf in next(it):
pass
WVPASSEQ((oidx, typ, size), get_info)
def _chunkiter(hash, startofs, repo_dir=None):
assert(startofs >= 0)
tree = _tree_decode(hash, repo_dir)
# skip elements before startofs
for i in xrange(len(tree)):
if i+1 >= len(tree) or tree[i+1][0] > startofs:
break
first = i
# iterate through what's left
for i in xrange(first, len(tree)):
(ofs,isdir,sha) = tree[i]
skipmore = startofs-ofs
if skipmore < 0:
skipmore = 0
if isdir:
for b in _chunkiter(sha, skipmore, repo_dir):
yield b
else:
yield ''.join(cp(repo_dir).join(sha.encode('hex')))[skipmore:]
def _chunkiter(hash, startofs, repo_dir=None):
assert (startofs >= 0)
tree = _tree_decode(hash, repo_dir)
# skip elements before startofs
for i in xrange(len(tree)):
if i + 1 >= len(tree) or tree[i + 1][0] > startofs:
break
first = i
# iterate through what's left
for i in xrange(first, len(tree)):
(ofs, isdir, sha) = tree[i]
skipmore = startofs - ofs
if skipmore < 0:
skipmore = 0
if isdir:
for b in _chunkiter(sha, skipmore, repo_dir):
yield b
else:
yield ''.join(cp(repo_dir).join(sha.encode('hex')))[skipmore:]
def test_cat_pipe():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print('something', file=f)
with open(src + '/2', 'w+') as f:
print('something else', file=f)
git.init_repo(bupdir)
exc(bup_exe, 'index', src)
oidx = exo(bup_exe, 'save', '-cn', 'src', '--strip', src).strip()
typ = exo('git', '--git-dir', bupdir, 'cat-file', '-t',
'src').strip()
size = int(exo('git', '--git-dir', bupdir, 'cat-file', '-s',
'src'))
it = git.cp().get('src')
get_info = it.next()
for buf in it.next():
pass
WVPASSEQ((oidx, typ, size), get_info)
def bup_gc(repo, threshold=10, compression=1, verbosity=0):
# Yes - this is a hack. We should use repo.cat() instead of cat_pipe.get(),
# but the repo abstraction right now can't properly deal with the fact that
# we modify the repository underneath.
repodir = os.path.join(repo.packdir(), b'..', b'..')
cat_pipe = git.cp(repodir)
existing_count = count_objects(repo.packdir(), verbosity)
if verbosity:
log('found %d objects\n' % existing_count)
if not existing_count:
if verbosity:
log('nothing to collect\n')
else:
try:
live_objects = find_live_objects(repo, existing_count, cat_pipe,
verbosity=verbosity)
except MissingObject as ex:
log('bup: missing object %r \n' % hexstr(ex.oid))
sys.exit(1)
try:
# FIXME: just rename midxes and bloom, and restore them at the end if
# we didn't change any packs?
packdir = repo.packdir()
if verbosity: log('clearing midx files\n')
midx.clear_midxes(packdir)
if verbosity: log('clearing bloom filter\n')
bloom.clear_bloom(packdir)
if verbosity: log('clearing reflog\n')
expirelog_cmd = [b'git', b'reflog', b'expire', b'--all', b'--expire=all']
expirelog = subprocess.Popen(expirelog_cmd,
env=git._gitenv(repo_dir=repodir))
git._git_wait(b' '.join(expirelog_cmd), expirelog)
if verbosity: log('removing unreachable data\n')
sweep(repo, live_objects, existing_count, cat_pipe,
threshold, compression,
verbosity)
finally:
live_objects.close()
def test_cat_pipe():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
git_type = exo('git', '--git-dir', bupdir, 'cat-file', '-t',
'src').strip()
git_size = int(
exo('git', '--git-dir', bupdir, 'cat-file', '-s', 'src'))
it = git.cp().get('src', size=True)
get_type, get_size = it.next()
for buf in it.next():
pass
WVPASSEQ(get_type, git_type)
WVPASSEQ(get_size, git_size)
def _treeget(hash):
it = cp().get(hash.encode('hex'))
type = it.next()
assert (type == 'tree')
return git.tree_decode(''.join(it))
def _chunk_len(hash):
return sum(len(b) for b in cp().join(hash.encode('hex')))
def test_commit_parsing():
def restore_env_var(name, val):
if val is None:
del environ[name]
else:
environ[name] = val
def showval(commit, val):
return readpipe([b'git', b'show', b'-s',
b'--pretty=format:%s' % val, commit]).strip()
with no_lingering_errors():
with test_tempdir(b'bup-tgit-') as tmpdir:
orig_cwd = os.getcwd()
workdir = tmpdir + b'/work'
repodir = workdir + b'/.git'
orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
environ[b'GIT_AUTHOR_NAME'] = b'bup test'
environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
try:
readpipe([b'git', b'init', workdir])
environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
git.check_repo_or_die(repodir)
os.chdir(workdir)
with open('foo', 'w') as f:
print('bar', file=f)
readpipe([b'git', b'add', b'.'])
readpipe([b'git', b'commit', b'-am', b'Do something',
b'--author', b'Someone <someone@somewhere>',
b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
parents = showval(commit, b'%P')
tree = showval(commit, b'%T')
cname = showval(commit, b'%cn')
cmail = showval(commit, b'%ce')
cdate = showval(commit, b'%ct')
coffs = showval(commit, b'%ci')
coffs = coffs[-5:]
coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
if bytes_from_byte(coffs[-5]) == b'-':
coff = - coff
commit_items = git.get_commit_items(commit, git.cp())
WVPASSEQ(commit_items.parents, [])
WVPASSEQ(commit_items.tree, tree)
WVPASSEQ(commit_items.author_name, b'Someone')
WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
WVPASSEQ(commit_items.author_sec, 1254613729)
WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
WVPASSEQ(commit_items.committer_name, cname)
WVPASSEQ(commit_items.committer_mail, cmail)
WVPASSEQ(commit_items.committer_sec, int(cdate))
WVPASSEQ(commit_items.committer_offset, coff)
WVPASSEQ(commit_items.message, b'Do something\n')
with open(b'bar', 'wb') as f:
f.write(b'baz\n')
readpipe([b'git', b'add', '.'])
readpipe([b'git', b'commit', b'-am', b'Do something else'])
child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
parents = showval(child, b'%P')
commit_items = git.get_commit_items(child, git.cp())
WVPASSEQ(commit_items.parents, [commit])
finally:
os.chdir(orig_cwd)
restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
def _treeget(hash):
it = cp().get(hash.encode('hex'))
type = it.next()
assert(type == 'tree')
return git.tree_decode(''.join(it))
def readlink(self):
"""Get the path that this link points at."""
return ''.join(cp(self._repo_dir).join(self.hash.encode('hex')))
def _treeget(hash, repo_dir=None):
it = cp(repo_dir).get(hash.encode("hex"))
type = it.next()
assert type == "tree"
return git.tree_decode("".join(it))
def _treeget(hash, repo_dir=None):
it = cp(repo_dir).get(hash.encode('hex'))
type = it.next()
assert (type == 'tree')
return git.tree_decode(''.join(it))
def readlink(self):
"""Get the path that this link points at."""
return ''.join(cp(self._repo_dir).join(self.hash.encode('hex')))
def __init__(self, repo_dir=None):
self.repo_dir = realpath(repo_dir or git.repo())
self._cp = git.cp(self.repo_dir)
self.rev_list = partial(git.rev_list, repo_dir=self.repo_dir)
self._id = _repo_id(self.repo_dir)
def test_new_commit():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = 'Author'
author_mail = 'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = 'Committer'
committer_mail = 'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(commit.encode('hex'), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp())
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_commit_parsing():
def restore_env_var(name, val):
if val is None:
del os.environ[name]
else:
os.environ[name] = val
def showval(commit, val):
return readpipe(['git', 'show', '-s',
'--pretty=format:%s' % val, commit]).strip()
with no_lingering_errors(), test_tempdir('bup-tgit-') as tmpdir:
orig_cwd = os.getcwd()
workdir = tmpdir + "/work"
repodir = workdir + '/.git'
orig_author_name = os.environ.get('GIT_AUTHOR_NAME')
orig_author_email = os.environ.get('GIT_AUTHOR_EMAIL')
orig_committer_name = os.environ.get('GIT_COMMITTER_NAME')
orig_committer_email = os.environ.get('GIT_COMMITTER_EMAIL')
os.environ['GIT_AUTHOR_NAME'] = 'bup test'
os.environ['GIT_COMMITTER_NAME'] = os.environ['GIT_AUTHOR_NAME']
os.environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
os.environ['GIT_COMMITTER_EMAIL'] = os.environ['GIT_AUTHOR_EMAIL']
try:
readpipe(['git', 'init', workdir])
os.environ['GIT_DIR'] = os.environ['BUP_DIR'] = repodir
git.check_repo_or_die(repodir)
os.chdir(workdir)
with open('foo', 'w') as f:
print >> f, 'bar'
readpipe(['git', 'add', '.'])
readpipe(['git', 'commit', '-am', 'Do something',
'--author', 'Someone <someone@somewhere>',
'--date', 'Sat Oct 3 19:48:49 2009 -0400'])
commit = readpipe(['git', 'show-ref', '-s', 'master']).strip()
parents = showval(commit, '%P')
tree = showval(commit, '%T')
cname = showval(commit, '%cn')
cmail = showval(commit, '%ce')
cdate = showval(commit, '%ct')
coffs = showval(commit, '%ci')
coffs = coffs[-5:]
coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
if coffs[-5] == '-':
coff = - coff
commit_items = git.get_commit_items(commit, git.cp())
WVPASSEQ(commit_items.parents, [])
WVPASSEQ(commit_items.tree, tree)
WVPASSEQ(commit_items.author_name, 'Someone')
WVPASSEQ(commit_items.author_mail, 'someone@somewhere')
WVPASSEQ(commit_items.author_sec, 1254613729)
WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
WVPASSEQ(commit_items.committer_name, cname)
WVPASSEQ(commit_items.committer_mail, cmail)
WVPASSEQ(commit_items.committer_sec, int(cdate))
WVPASSEQ(commit_items.committer_offset, coff)
WVPASSEQ(commit_items.message, 'Do something\n')
with open('bar', 'w') as f:
print >> f, 'baz'
readpipe(['git', 'add', '.'])
readpipe(['git', 'commit', '-am', 'Do something else'])
child = readpipe(['git', 'show-ref', '-s', 'master']).strip()
parents = showval(child, '%P')
commit_items = git.get_commit_items(child, git.cp())
WVPASSEQ(commit_items.parents, [commit])
finally:
os.chdir(orig_cwd)
restore_env_var('GIT_AUTHOR_NAME', orig_author_name)
restore_env_var('GIT_AUTHOR_EMAIL', orig_author_email)
restore_env_var('GIT_COMMITTER_NAME', orig_committer_name)
restore_env_var('GIT_COMMITTER_EMAIL', orig_committer_email)
def __init__(self, repo_dir=None):
self.repo_dir = repo_dir or git.repo()
self._cp = git.cp(repo_dir)
self.rev_list = partial(git.rev_list, repo_dir=repo_dir)
def _chunk_len(hash):
return sum(len(b) for b in cp().join(hash.encode('hex')))
def test_new_commit():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
git.init_repo(bupdir)
git.verbose = 1
w = git.PackWriter()
tree = os.urandom(20)
parent = os.urandom(20)
author_name = 'Author'
author_mail = 'author@somewhere'
adate_sec = 1439657836
cdate_sec = adate_sec + 1
committer_name = 'Committer'
committer_mail = 'committer@somewhere'
adate_tz_sec = cdate_tz_sec = None
commit = w.new_commit(tree, parent, '%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
adate_tz_sec = -60 * 60
cdate_tz_sec = 120 * 60
commit_off = w.new_commit(tree, parent,
'%s <%s>' % (author_name, author_mail),
adate_sec, adate_tz_sec,
'%s <%s>' % (committer_name, committer_mail),
cdate_sec, cdate_tz_sec,
'There is a small mailbox here')
w.close()
commit_items = git.get_commit_items(commit.encode('hex'), git.cp())
local_author_offset = localtime(adate_sec).tm_gmtoff
local_committer_offset = localtime(cdate_sec).tm_gmtoff
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(local_author_offset, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(local_committer_offset, commit_items.committer_offset)
commit_items = git.get_commit_items(commit_off.encode('hex'), git.cp())
WVPASSEQ(tree, commit_items.tree.decode('hex'))
WVPASSEQ(1, len(commit_items.parents))
WVPASSEQ(parent, commit_items.parents[0].decode('hex'))
WVPASSEQ(author_name, commit_items.author_name)
WVPASSEQ(author_mail, commit_items.author_mail)
WVPASSEQ(adate_sec, commit_items.author_sec)
WVPASSEQ(adate_tz_sec, commit_items.author_offset)
WVPASSEQ(committer_name, commit_items.committer_name)
WVPASSEQ(committer_mail, commit_items.committer_mail)
WVPASSEQ(cdate_sec, commit_items.committer_sec)
WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def _chunk_len(hash, repo_dir=None):
return sum(len(b) for b in cp(repo_dir).join(hash.encode('hex')))
def _treeget(hash, repo_dir=None):
it = cp(repo_dir).get(hash.encode('hex'))
type = it.next()
assert(type == 'tree')
return git.tree_decode(''.join(it))
def _chunk_len(hash, repo_dir=None):
return sum(len(b) for b in cp(repo_dir).join(hash.encode('hex')))
