def sync_indexes(self):
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
debug1('%s\n' % f)
if f.endswith('.idx'):
extra.add(f)
needed = set()
conn.write('list-indexes\n')
for line in linereader(conn):
if not line:
break
assert(line.find('/') < 0)
parts = line.split(' ')
idx = parts[0]
if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# Any idx that the server has heard of is proven not extra
extra.discard(idx)
self.check_ok()
debug1('client: removing extra indexes: %s\n' % extra)
for idx in extra:
os.unlink(os.path.join(self.cachedir, idx))
debug1('client: server requested load of: %s\n' % needed)
for idx in needed:
self.sync_index(idx)
git.auto_midx(self.cachedir)
def sync_indexes(self):
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
debug1(path_msg(f) + '\n')
if f.endswith(b'.idx'):
extra.add(f)
needed = set()
for idx, load in self._list_indexes():
if load:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# Any idx that the server has heard of is proven not extra
extra.discard(idx)
debug1('client: removing extra indexes: %s\n' % extra)
for idx in extra:
os.unlink(os.path.join(self.cachedir, idx))
debug1('client: server requested load of: %s\n' % needed)
for idx in needed:
self.sync_index(idx)
git.auto_midx(self.cachedir)
def sync_indexes(self):
self._require_command(b'list-indexes')
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
debug1(path_msg(f) + '\n')
if f.endswith(b'.idx'):
extra.add(f)
needed = set()
conn.write(b'list-indexes\n')
for line in linereader(conn):
if not line:
break
assert(line.find(b'/') < 0)
parts = line.split(b' ')
idx = parts[0]
if len(parts) == 2 and parts[1] == b'load' and idx not in extra:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# Any idx that the server has heard of is proven not extra
extra.discard(idx)
self.check_ok()
debug1('client: removing extra indexes: %s\n' % extra)
for idx in extra:
os.unlink(os.path.join(self.cachedir, idx))
debug1('client: server requested load of: %s\n' % needed)
for idx in needed:
self.sync_index(idx)
git.auto_midx(self.cachedir)
def __init__(self, cfg_file, create=False):
super(EncryptedRepo, self).__init__(cfg_file, create)
# init everything for __del__ in case we get an exception here
self.storage = None
self.data_writer = None
self.meta_writer = None
self.cfg_file = cfg_file
self.ec_cache = {}
if libnacl is None:
raise Exception("Encrypted repositories require libnacl")
if self.max_pack_size is None:
self.max_pack_size = 1000 * 1000 * 1000
self.cachedir = self.config(b'bup.cachedir', opttype='path')
if self.cachedir is None:
raise Exception("encrypted repositories need a 'cachedir'")
if create:
mkdirp(self.cachedir)
if not os.path.isdir(self.cachedir):
raise Exception(
"cachedir doesn't exist or isn't a directory - may have to init the repo?"
)
self.storage = get_storage(self, create=create)
self.readkey = None
self.repokey = None
self.writekey = None
self.refsname = self.config(b'bup.refsname')
if self.refsname is None:
self.refsname = b'refs'
readkey = self.config(b'bup.readkey')
if readkey is not None:
self.readkey = libnacl.public.SecretKey(unhexlify(readkey))
repokey = self.config(b'bup.repokey')
if repokey is not None:
self.repokey = unhexlify(repokey)
writekey = self.config(b'bup.writekey')
if writekey is not None:
self.writekey = unhexlify(writekey)
if self.readkey is not None:
assert self.writekey == self.readkey.pk
else:
assert self.readkey is not None, "at least one of 'readkey' or 'writekey' is required"
self.writekey = self.readkey.pk
self.compression = self.compression_level
if self.compression is None:
self.compression = -1
self.separatemeta = self.config(b'bup.separatemeta', opttype='bool')
self.data_written_objs = set()
if self.separatemeta:
self.meta_written_objs = set()
else:
self.meta_written_objs = self.data_written_objs
self._synchronize_idxes()
self.idxlist = git.PackIdxList(self.cachedir)
def __init__(self, repo, create=False):
self.openset = set()
self.path = repo.config(b'bup.path', opttype='path')
if create:
mkdirp(self.path)
if not os.path.isdir(self.path):
raise Exception(
"FileStorage: %s doesn't exist or isn't a directory, need to init?"
% path_msg(self.path))
def sync_index(self, name):
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
msg = ("won't request existing .idx, try `bup bloom --check %s`"
% path_msg(fn))
raise ClientError(msg)
with atomically_replaced_file(fn, 'wb') as f:
self.send_index(name, f, lambda size: None)
def _set_up_path(meta, create_symlinks=True):
# Allow directories to exist as a special case -- might have
# been created by an earlier longer path.
if meta.isdir():
mkdirp(meta.path)
else:
parent = os.path.dirname(meta.path)
if parent:
mkdirp(parent)
meta.create_path(meta.path, create_symlinks=create_symlinks)
def _set_up_path(meta, create_symlinks=True):
# Allow directories to exist as a special case -- might have
# been created by an earlier longer path.
if meta.isdir():
mkdirp(meta.path)
else:
parent = os.path.dirname(meta.path)
if parent:
mkdirp(parent)
meta.create_path(meta.path, create_symlinks=create_symlinks)
def test_list_refs():
with no_lingering_errors():
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
src = tmpdir + b'/src'
mkdirp(src)
with open(src + b'/1', 'wb+') as f:
f.write(b'something\n')
with open(src + b'/2', 'wb+') as f:
f.write(b'something else\n')
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, b'index', src)
exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
src_hash = exo(b'git', b'--git-dir', bupdir, b'rev-parse',
b'src').strip().split(b'\n')
assert (len(src_hash) == 1)
src_hash = unhexlify(src_hash[0])
tree_hash = unhexlify(
exo(b'git', b'--git-dir', bupdir, b'rev-parse',
b'src:').strip().split(b'\n')[0])
blob_hash = unhexlify(
exo(b'git', b'--git-dir', bupdir, b'rev-parse',
b'src:1').strip().split(b'\n')[0])
WVPASSEQ(frozenset(git.list_refs()),
frozenset([(b'refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([(b'refs/heads/src', src_hash)]))
exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
WVPASSEQ(
frozenset(git.list_refs()),
frozenset([(b'refs/heads/src', src_hash),
(b'refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([(b'refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([(b'refs/heads/src', src_hash)]))
exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
os.unlink(bupdir + b'/refs/heads/src')
expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
(b'refs/tags/tree-tag', tree_hash),
(b'refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
expected_tags)
def create_path(n, fullname, meta):
if meta:
meta.create_path(fullname)
else:
# These fallbacks are important -- meta could be null if, for
# example, save created a "fake" item, i.e. a new strip/graft
# path element, etc. You can find cases like that by
# searching for "Metadata()".
unlink(fullname)
if stat.S_ISDIR(n.mode):
mkdirp(fullname)
elif stat.S_ISLNK(n.mode):
os.symlink(n.readlink(), fullname)
def test_list_refs():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
src_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src').strip().split('\n')
assert (len(src_hash) == 1)
src_hash = src_hash[0].decode('hex')
tree_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src:').strip().split('\n')[0].decode('hex')
blob_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src:1').strip().split('\n')[0].decode('hex')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'commit-tag', 'src')
WVPASSEQ(
frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash),
('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'tree-tag', 'src:')
exc('git', '--git-dir', bupdir, 'tag', 'blob-tag', 'src:1')
os.unlink(bupdir + '/refs/heads/src')
expected_tags = frozenset([('refs/tags/commit-tag', src_hash),
('refs/tags/tree-tag', tree_hash),
('refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_list_refs():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
src_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src').strip().split('\n')
assert (len(src_hash) == 1)
src_hash = src_hash[0].decode('hex')
tree_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src:').strip().split('\n')[0].decode('hex')
blob_hash = exo('git', '--git-dir', bupdir, 'rev-parse',
'src:1').strip().split('\n')[0].decode('hex')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'commit-tag', 'src')
WVPASSEQ(
frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash),
('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'tree-tag', 'src:')
exc('git', '--git-dir', bupdir, 'tag', 'blob-tag', 'src:1')
os.unlink(bupdir + '/refs/heads/src')
expected_tags = frozenset([('refs/tags/commit-tag', src_hash),
('refs/tags/tree-tag', tree_hash),
('refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
expected_tags)
def test_list_refs():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tgit-')
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
src_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src').strip().split('\n')
assert(len(src_hash) == 1)
src_hash = src_hash[0].decode('hex')
tree_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:').strip().split('\n')[0].decode('hex')
blob_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:1').strip().split('\n')[0].decode('hex')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'commit-tag', 'src')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash),
('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'tree-tag', 'src:')
exc('git', '--git-dir', bupdir, 'tag', 'blob-tag', 'src:1')
os.unlink(bupdir + '/refs/heads/src')
expected_tags = frozenset([('refs/tags/commit-tag', src_hash),
('refs/tags/tree-tag', tree_hash),
('refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def test_list_refs():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
emptyset = frozenset()
WVPASSEQ(frozenset(git.list_refs()), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
src_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src').strip().split('\n')
assert(len(src_hash) == 1)
src_hash = src_hash[0].decode('hex')
tree_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:').strip().split('\n')[0].decode('hex')
blob_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:1').strip().split('\n')[0].decode('hex')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'commit-tag', 'src')
WVPASSEQ(frozenset(git.list_refs()),
frozenset([('refs/heads/src', src_hash),
('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
frozenset([('refs/tags/commit-tag', src_hash)]))
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
frozenset([('refs/heads/src', src_hash)]))
exc('git', '--git-dir', bupdir, 'tag', 'tree-tag', 'src:')
exc('git', '--git-dir', bupdir, 'tag', 'blob-tag', 'src:1')
os.unlink(bupdir + '/refs/heads/src')
expected_tags = frozenset([('refs/tags/commit-tag', src_hash),
('refs/tags/tree-tag', tree_hash),
('refs/tags/blob-tag', blob_hash)])
WVPASSEQ(frozenset(git.list_refs()), expected_tags)
WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
msg = "won't request existing .idx, try `bup bloom --check %s`" % fn
raise ClientError(msg)
self.conn.write('send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert(n)
with atomically_replaced_file(fn, 'w') as f:
count = 0
for b in chunkyreader(self.conn, n):
f.write(b)
count += len(b)
qprogress('Receiving index from server: %d/%d\r' % (count, n))
self.check_ok()
def test_cat_pipe(tmpdir):
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
src = tmpdir + b'/src'
mkdirp(src)
with open(src + b'/1', 'wb+') as f:
f.write(b'something\n')
with open(src + b'/2', 'wb+') as f:
f.write(b'something else\n')
git.init_repo(bupdir)
exc(bup_exe, b'index', src)
oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip', src).strip()
typ = exo(b'git', b'--git-dir', bupdir, b'cat-file', b'-t', b'src').strip()
size = int(exo(b'git', b'--git-dir', bupdir, b'cat-file', b'-s', b'src'))
it = git.cp().get(b'src')
get_info = next(it)
for buf in next(it):
pass
WVPASSEQ((oidx, typ, size), get_info)
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
msg = "won't request existing .idx, try `bup bloom --check %s`" % fn
raise ClientError(msg)
self.conn.write('send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert (n)
with atomically_replaced_file(fn, 'w') as f:
count = 0
progress('Receiving index from server: %d/%d\r' % (count, n))
for b in chunkyreader(self.conn, n):
f.write(b)
count += len(b)
qprogress('Receiving index from server: %d/%d\r' % (count, n))
progress('Receiving index from server: %d/%d, done.\n' %
(count, n))
self.check_ok()
def test_cat_pipe():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print('something', file=f)
with open(src + '/2', 'w+') as f:
print('something else', file=f)
git.init_repo(bupdir)
exc(bup_exe, 'index', src)
oidx = exo(bup_exe, 'save', '-cn', 'src', '--strip', src).strip()
typ = exo('git', '--git-dir', bupdir, 'cat-file', '-t',
'src').strip()
size = int(exo('git', '--git-dir', bupdir, 'cat-file', '-s',
'src'))
it = git.cp().get('src')
get_info = it.next()
for buf in it.next():
pass
WVPASSEQ((oidx, typ, size), get_info)
def test_cat_pipe():
with no_lingering_errors():
with test_tempdir('bup-tgit-') as tmpdir:
os.environ['BUP_MAIN_EXE'] = bup_exe
os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
git.init_repo(bupdir)
exc(bup_exe, 'index', src)
exc(bup_exe, 'save', '-n', 'src', '--strip', src)
git_type = exo('git', '--git-dir', bupdir, 'cat-file', '-t',
'src').strip()
git_size = int(
exo('git', '--git-dir', bupdir, 'cat-file', '-s', 'src'))
it = git.cp().get('src', size=True)
get_type, get_size = it.next()
for buf in it.next():
pass
WVPASSEQ(get_type, git_type)
WVPASSEQ(get_size, git_size)
import sys, os, stat, time, random, subprocess, glob, tempfile
from bup import client, git
from bup.helpers import mkdirp
from wvtest import *
bup_tmp = os.path.realpath('../../../t/tmp')
mkdirp(bup_tmp)
def randbytes(sz):
s = ''
for i in xrange(sz):
s += chr(random.randrange(0, 256))
return s
s1 = randbytes(10000)
s2 = randbytes(10000)
s3 = randbytes(10000)
IDX_PAT = '/*.idx'
@wvtest
def test_server_split_with_indexes():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tclient-')
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
import sys, os, stat, time, random, subprocess, glob, tempfile
from bup import client, git
from bup.helpers import mkdirp
from wvtest import *
bup_tmp = os.path.realpath('../../../t/tmp')
mkdirp(bup_tmp)
def randbytes(sz):
s = ''
for i in xrange(sz):
s += chr(random.randrange(0,256))
return s
s1 = randbytes(10000)
s2 = randbytes(10000)
s3 = randbytes(10000)
IDX_PAT = '/*.idx'
@wvtest
def test_server_split_with_indexes():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tclient-')
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
msg = 'saved_errors ' + repr(helpers.saved_errors)
print('! %-70s %s' %
('%s:%-4d %s' %
(basename(src_file), src_line, msg), 'FAILED'))
sys.stdout.flush()
fail_if_errors()
helpers.clear_errors()
yield
fail_if_errors()
helpers.clear_errors()
# Assumes (of course) this file is at the top-level of the source tree
_bup_tmp = realpath(dirname(__file__) + '/t/tmp')
helpers.mkdirp(_bup_tmp)
@contextmanager
def test_tempdir(prefix):
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=_bup_tmp, prefix=prefix)
yield tmpdir
if wvfailure_count() == initial_failures:
subprocess.call(['chmod', '-R', 'u+rwX', tmpdir])
subprocess.call(['rm', '-rf', tmpdir])
def logcmd(cmd):
if isinstance(cmd, basestring):
print(cmd, file=sys.stderr)
def main():
o = options.Options(optspec)
opt, flags, extra = o.parse(sys.argv[1:])
verbosity = opt.verbose if not opt.quiet else -1
git.check_repo_or_die()
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
top = os.getcwd()
hardlinks = {}
for path in extra:
if not valid_restore_path(path):
add_error("path %r doesn't include a branch and revision" % path)
continue
try:
resolved = vfs2.lresolve(repo, path, want_meta=True)
except vfs2.IOError as e:
add_error(e)
continue
path_parent, path_name = os.path.split(path)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
add_error('error: cannot access %r in %r' %
('/'.join(name for name, item in resolved), path))
continue
if not path_name or path_name == '.':
# Source is /foo/what/ever/ or /foo/what/ever/. -- extract
# what/ever/* to the current directory, and if name == '.'
# (i.e. /foo/what/ever/.), then also restore what/ever's
# metadata to the current directory.
treeish = vfs2.item_mode(leaf_item)
if not treeish:
add_error('%r cannot be restored as a directory' % path)
else:
items = vfs2.contents(repo, leaf_item, want_meta=True)
dot, leaf_item = next(items, None)
assert (dot == '.')
for sub_name, sub_item in items:
restore(repo, '', sub_name, sub_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if path_name == '.':
leaf_item = vfs2.augment_item_meta(repo,
leaf_item,
include_size=True)
apply_metadata(leaf_item.meta, '.', opt.numeric_ids,
owner_map)
else:
restore(repo, '', leaf_name, leaf_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if verbosity >= 0:
progress('Restoring: %d, done.\n' % total_restored)
die_if_errors()
(opt, flags, extra) = o.parse(sys.argv[1:])
git.check_repo_or_die()
top = vfs.RefList(None)
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
ret = 0
for d in extra:
if not valid_restore_path(d):
add_error("ERROR: path %r doesn't include a branch and revision" % d)
continue
path,name = os.path.split(d)
try:
n = top.lresolve(d)
except vfs.NodeError as e:
add_error(e)
continue
isdir = stat.S_ISDIR(n.mode)
if not name or name == '.':
def main():
o = options.Options(optspec)
opt, flags, extra = o.parse(sys.argv[1:])
verbosity = opt.verbose if not opt.quiet else -1
git.check_repo_or_die()
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
top = os.getcwd()
hardlinks = {}
for path in extra:
if not valid_restore_path(path):
add_error("path %r doesn't include a branch and revision" % path)
continue
try:
resolved = vfs.resolve(repo, path, want_meta=True, follow=False)
except vfs.IOError as e:
add_error(e)
continue
if len(resolved) == 3 and resolved[2][0] == 'latest':
# Follow latest symlink to the actual save
try:
resolved = vfs.resolve(repo, 'latest', parent=resolved[:-1],
want_meta=True)
except vfs.IOError as e:
add_error(e)
continue
# Rename it back to 'latest'
resolved = tuple(elt if i != 2 else ('latest',) + elt[1:]
for i, elt in enumerate(resolved))
path_parent, path_name = os.path.split(path)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
add_error('error: cannot access %r in %r'
% ('/'.join(name for name, item in resolved),
path))
continue
if not path_name or path_name == '.':
# Source is /foo/what/ever/ or /foo/what/ever/. -- extract
# what/ever/* to the current directory, and if name == '.'
# (i.e. /foo/what/ever/.), then also restore what/ever's
# metadata to the current directory.
treeish = vfs.item_mode(leaf_item)
if not treeish:
add_error('%r cannot be restored as a directory' % path)
else:
items = vfs.contents(repo, leaf_item, want_meta=True)
dot, leaf_item = next(items, None)
assert(dot == '.')
for sub_name, sub_item in items:
restore(repo, '', sub_name, sub_item, top,
opt.sparse, opt.numeric_ids, owner_map,
exclude_rxs, verbosity, hardlinks)
if path_name == '.':
leaf_item = vfs.augment_item_meta(repo, leaf_item,
include_size=True)
apply_metadata(leaf_item.meta, '.',
opt.numeric_ids, owner_map)
else:
restore(repo, '', leaf_name, leaf_item, top,
opt.sparse, opt.numeric_ids, owner_map,
exclude_rxs, verbosity, hardlinks)
if verbosity >= 0:
progress('Restoring: %d, done.\n' % total_restored)
die_if_errors()
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
verbosity = (opt.verbose or 0) if not opt.quiet else -1
if opt.remote:
opt.remote = argv_bytes(opt.remote)
if opt.outdir:
opt.outdir = argv_bytes(opt.outdir)
git.check_repo_or_die()
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
top = fsencode(os.getcwd())
hardlinks = {}
for path in [argv_bytes(x) for x in extra]:
if not valid_restore_path(path):
add_error("path %r doesn't include a branch and revision" % path)
continue
try:
resolved = vfs.resolve(repo, path, want_meta=True, follow=False)
except vfs.IOError as e:
add_error(e)
continue
if len(resolved) == 3 and resolved[2][0] == b'latest':
# Follow latest symlink to the actual save
try:
resolved = vfs.resolve(repo,
b'latest',
parent=resolved[:-1],
want_meta=True)
except vfs.IOError as e:
add_error(e)
continue
# Rename it back to 'latest'
resolved = tuple(elt if i != 2 else (b'latest', ) + elt[1:]
for i, elt in enumerate(resolved))
path_parent, path_name = os.path.split(path)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
add_error('error: cannot access %r in %r' %
(b'/'.join(name for name, item in resolved), path))
continue
if not path_name or path_name == b'.':
# Source is /foo/what/ever/ or /foo/what/ever/. -- extract
# what/ever/* to the current directory, and if name == '.'
# (i.e. /foo/what/ever/.), then also restore what/ever's
# metadata to the current directory.
treeish = vfs.item_mode(leaf_item)
if not treeish:
add_error('%r cannot be restored as a directory' % path)
else:
items = vfs.contents(repo, leaf_item, want_meta=True)
dot, leaf_item = next(items, None)
assert dot == b'.'
for sub_name, sub_item in items:
restore(repo, b'', sub_name, sub_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if path_name == b'.':
leaf_item = vfs.augment_item_meta(repo,
leaf_item,
include_size=True)
apply_metadata(leaf_item.meta, b'.', opt.numeric_ids,
owner_map)
else:
restore(repo, b'', leaf_name, leaf_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if verbosity >= 0:
progress('Restoring: %d, done.\n' % total_restored)
die_if_errors()
def test_path_info():
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tclient-')
os.environ['BUP_MAIN_EXE'] = '../../../bup'
os.environ['BUP_DIR'] = bupdir = tmpdir
src = tmpdir + '/src'
mkdirp(src)
with open(src + '/1', 'w+') as f:
print f, 'something'
with open(src + '/2', 'w+') as f:
print f, 'something else'
os.mkdir(src + '/dir')
git.init_repo(bupdir)
c = client.Client(bupdir, create=True)
info = c.path_info(['/'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
name, id, type = info[0]
WVPASSEQ(type, 'root')
info = c.path_info(['/not-there/'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0] is None)
data = exo(bup_exe, 'random', '128k')
with open(src + '/chunky', 'wb+') as f:
f.write(data)
ex(bup_exe, 'index', '-vv', src)
ex(bup_exe, 'save', '-n', 'src', '--strip', src)
ex(bup_exe, 'tag', 'src-latest-tag', 'src')
src_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src').strip().split('\n')
assert(len(src_hash) == 1)
src_hash = src_hash[0].decode('hex')
tree_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:dir').strip().split('\n')[0].decode('hex')
file_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:1').strip().split('\n')[0].decode('hex')
chunky_hash = exo('git', '--git-dir', bupdir,
'rev-parse', 'src:chunky.bup').strip().split('\n')[0].decode('hex')
info = c.path_info(['/src'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/src', src_hash, 'branch'])
info = c.path_info(['/src/latest'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/src/latest', src_hash, 'save'])
info = c.path_info(['/src/latest/dir'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/src/latest/dir', tree_hash, 'dir'])
info = c.path_info(['/src/latest/1'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/src/latest/1', file_hash, 'file'])
info = c.path_info(['/src/latest/chunky'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/src/latest/chunky', chunky_hash, 'chunked-file'])
info = c.path_info(['/.tag/src-latest-tag'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/.tag/src-latest-tag', src_hash, 'commit'])
info = c.path_info(['.tag////src-latest-tag'])
WVPASS(info)
WVPASS(len(info) == 1)
WVPASS(info[0])
WVPASSEQ(info[0], ['/.tag/src-latest-tag', src_hash, 'commit'])
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def fail_if_errors():
if helpers.saved_errors:
bt = extract_stack()
src_file, src_line, src_func, src_txt = bt[-4]
msg = 'saved_errors ' + repr(helpers.saved_errors)
print '! %-70s %s' % ('%s:%-4d %s' % (basename(src_file),
src_line,
msg),
'FAILED')
sys.stdout.flush()
fail_if_errors()
helpers.clear_errors()
yield
fail_if_errors()
helpers.clear_errors()
# Assumes (of course) this file is at the top-level of the source tree
_bup_tmp = realpath(dirname(__file__) + '/t/tmp')
helpers.mkdirp(_bup_tmp)
@contextmanager
def test_tempdir(prefix):
initial_failures = wvfailure_count()
tmpdir = tempfile.mkdtemp(dir=_bup_tmp, prefix=prefix)
yield tmpdir
if wvfailure_count() == initial_failures:
subprocess.call(['chmod', '-R', 'u+rwX', tmpdir])
subprocess.call(['rm', '-rf', tmpdir])
