def _init_session(reinit_with_new_repopath=None):
global repo
if reinit_with_new_repopath is None and git.repodir:
if not repo:
repo = LocalRepo()
return
git.check_repo_or_die(reinit_with_new_repopath)
if repo:
repo.close()
repo = LocalRepo()
# OK. we now know the path is a proper repository. Record this path in the
# environment so that subprocesses inherit it and know where to operate.
os.environ['BUP_DIR'] = git.repodir
debug1('bup server: bupdir is %r\n' % git.repodir)
_set_mode()
def test_contents_with_mismatched_bupm_git_ordering():
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
data_path = tmpdir + '/src'
os.mkdir(data_path)
os.mkdir(data_path + '/foo')
with open(data_path + '/foo.', 'w+') as tmpfile:
tmpfile.write(b'canary\n')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-tvvn', 'test', '--strip', data_path))
repo = LocalRepo()
tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
tip_oidx = tip_sref.strip().split()[0]
tip_tree_oidx = exo(
('git', 'log', '--pretty=%T', '-n1', tip_oidx))[0].strip()
tip_tree_oid = tip_tree_oidx.decode('hex')
tip_tree = tree_dict(repo, tip_tree_oid)
name, item = vfs.resolve(repo, '/test/latest')[2]
wvpasseq('latest', name)
expected = frozenset(
(x.name, vfs.Item(oid=x.oid, meta=x.meta))
for x in (tip_tree[name] for name in ('.', 'foo', 'foo.')))
contents = tuple(vfs.contents(repo, item))
wvpasseq(expected, frozenset(contents))
# Spot check, in case tree_dict shares too much code with the vfs
name, item = next(((n, i) for n, i in contents if n == 'foo'))
wvpass(S_ISDIR(item.meta))
name, item = next(((n, i) for n, i in contents if n == 'foo.'))
wvpass(S_ISREG(item.meta.mode))
def test_duplicate_save_dates():
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
environ['TZ'] = 'UTC'
git.repodir = bup_dir
data_path = tmpdir + '/src'
os.mkdir(data_path)
with open(data_path + '/file', 'w+') as tmpfile:
tmpfile.write(b'canary\n')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
for i in range(11):
ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
repo = LocalRepo()
res = vfs.resolve(repo, '/test')
wvpasseq(2, len(res))
name, revlist = res[-1]
wvpasseq('test', name)
wvpasseq(('.', '1970-01-02-034640-00', '1970-01-02-034640-01',
'1970-01-02-034640-02', '1970-01-02-034640-03',
'1970-01-02-034640-04', '1970-01-02-034640-05',
'1970-01-02-034640-06', '1970-01-02-034640-07',
'1970-01-02-034640-08', '1970-01-02-034640-09',
'1970-01-02-034640-10', 'latest'),
tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
def test_resolve_loop():
with no_lingering_errors():
with test_tempdir('bup-tvfs-resloop-') as tmpdir:
resolve = vfs.resolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
repo = LocalRepo()
data_path = tmpdir + '/src'
os.mkdir(data_path)
symlink('loop', data_path + '/loop')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
save_utc = 100000
ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test',
'--strip', data_path))
save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
try:
wvpasseq('this call should never return',
resolve(repo, '/test/%s/loop' % save_name))
except vfs.IOError as res_ex:
wvpasseq(ELOOP, res_ex.errno)
wvpasseq(['', 'test', save_name, 'loop'],
[name for name, item in res_ex.terminus])
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no positional parameters expected')
if not opt.check and opt.k and opt.k not in (4, 5):
o.fatal('only k values of 4 and 5 are supported')
if opt.check:
opt.check = argv_bytes(opt.check)
output = argv_bytes(opt.output) if opt.output else None
if opt.dir:
path = argv_bytes(opt.dir)
else:
path = LocalRepo().packdir()
debug1('bloom: scanning %s\n' % path_msg(path))
outfilename = output or os.path.join(path, b'bup.bloom')
if opt.check:
check_bloom(path, outfilename, opt.check)
elif opt.ruin:
ruin_bloom(outfilename)
else:
do_bloom(path, outfilename, opt.k, opt.force)
if saved_errors:
log('WARNING: %d errors encountered during bloom.\n' %
len(saved_errors))
sys.exit(1)
elif opt.check:
log('All tests passed.\n')
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if opt.remote:
opt.remote = argv_bytes(opt.remote)
git.check_repo_or_die()
stdin = byte_stream(sys.stdin)
if not extra:
extra = linereader(stdin)
ret = 0
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
if opt.o:
outfile = open(opt.o, 'wb')
else:
sys.stdout.flush()
outfile = byte_stream(sys.stdout)
for ref in [argv_bytes(x) for x in extra]:
try:
for blob in repo.join(ref):
outfile.write(blob)
except KeyError as e:
outfile.flush()
log('error: %s\n' % e)
ret = 1
sys.exit(ret)
def test_duplicate_save_dates(tmpdir):
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
environ[b'TZ'] = b'UTC'
tzset()
git.repodir = bup_dir
data_path = tmpdir + b'/src'
os.mkdir(data_path)
with open(data_path + b'/file', 'wb+') as tmpfile:
tmpfile.write(b'canary\n')
ex((b'env', ))
ex((bup_path, b'init'))
ex((bup_path, b'index', b'-v', data_path))
for i in range(11):
ex((bup_path, b'save', b'-d', b'100000', b'-n', b'test', data_path))
with LocalRepo() as repo:
res = vfs.resolve(repo, b'/test')
wvpasseq(2, len(res))
name, revlist = res[-1]
wvpasseq(b'test', name)
wvpasseq((b'.', b'1970-01-02-034640-00', b'1970-01-02-034640-01',
b'1970-01-02-034640-02', b'1970-01-02-034640-03',
b'1970-01-02-034640-04', b'1970-01-02-034640-05',
b'1970-01-02-034640-06', b'1970-01-02-034640-07',
b'1970-01-02-034640-08', b'1970-01-02-034640-09',
b'1970-01-02-034640-10', b'latest'),
tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if not opt.verbose:
opt.verbose = 0
# Set stderr to be line buffered, even if it's not connected to the console
# so that we'll be able to see diagnostics in a timely fashion.
errfd = sys.stderr.fileno()
sys.stderr.flush()
sys.stderr = os.fdopen(errfd, 'w', 1)
if len(extra) != 1:
o.fatal('only one mount point argument expected')
repo = LocalRepo()
f = BupFs(repo=repo, verbose=opt.verbose, fake_metadata=(not opt.meta))
# This is likely wrong, but the fuse module doesn't currently accept bytes
f.fuse_args.mountpoint = extra[0]
if opt.debug:
f.fuse_args.add('debug')
if opt.foreground:
f.fuse_args.setmod('foreground')
f.multithreaded = False
if opt.allow_other:
f.fuse_args.add('allow_other')
f.main()
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if not opt.unsafe:
o.fatal(
'refusing to run dangerous, experimental command without --unsafe')
if extra:
o.fatal('no positional parameters expected')
if opt.threshold:
try:
opt.threshold = int(opt.threshold)
except ValueError:
o.fatal('threshold must be an integer percentage value')
if opt.threshold < 0 or opt.threshold > 100:
o.fatal('threshold must be an integer percentage value')
bup_gc(LocalRepo(),
threshold=opt.threshold,
compression=opt.compress,
verbosity=opt.verbose)
die_if_errors()
def test_metadata_method(tmpdir):
bup_dir = tmpdir + b'/bup'
data_path = tmpdir + b'/foo'
os.mkdir(data_path)
ex(b'touch', data_path + b'/file')
ex(b'ln', b'-s', b'file', data_path + b'/symlink')
test_time1 = 13 * 1000000000
test_time2 = 42 * 1000000000
utime(data_path + b'/file', (0, test_time1))
lutime(data_path + b'/symlink', (0, 0))
utime(data_path, (0, test_time2))
ex(bup_path, b'-d', bup_dir, b'init')
ex(bup_path, b'-d', bup_dir, b'index', b'-v', data_path)
ex(bup_path, b'-d', bup_dir, b'save', b'-tvvn', b'test', data_path)
repo = LocalRepo(bup_dir)
resolved = vfs.resolve(repo,
b'/test/latest' + resolve_parent(data_path),
follow=False)
leaf_name, leaf_item = resolved[-1]
m = leaf_item.meta
WVPASS(m.mtime == test_time2)
WVPASS(leaf_name == b'foo')
contents = tuple(vfs.contents(repo, leaf_item))
WVPASS(len(contents) == 3)
WVPASSEQ(frozenset(name for name, item in contents),
frozenset((b'.', b'file', b'symlink')))
for name, item in contents:
if name == b'file':
m = item.meta
WVPASS(m.mtime == test_time1)
elif name == b'symlink':
m = item.meta
WVPASSEQ(m.symlink_target, b'file')
WVPASSEQ(m.size, 4)
WVPASSEQ(m.mtime, 0)
def main(argv):
global dry_run
log('\nbup: import-duplicity is EXPERIMENTAL (proceed with caution)\n\n')
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
dry_run = opt.dry_run
if len(extra) < 1 or not extra[0]:
o.fatal('duplicity source URL required')
if len(extra) < 2 or not extra[1]:
o.fatal('bup destination save name required')
if len(extra) > 2:
o.fatal('too many arguments')
source_url, save_name = extra
source_url = argv_bytes(source_url)
save_name = argv_bytes(save_name)
bup_path = bup.path.exe()
LocalRepo()
tmpdir = tempfile.mkdtemp(prefix=b'bup-import-dup-')
try:
dup = [b'duplicity', b'--archive-dir', tmpdir + b'/dup-cache']
restoredir = tmpdir + b'/restore'
tmpidx = tmpdir + b'/index'
collection_status = \
exo(dup + [b'collection-status', b'--log-fd=3', source_url],
close_fds=False, preexec_fn=redirect_dup_output) # i.e. 3>&1 1>&2
# Duplicity output lines of interest look like this (one leading space):
# full 20150222T073111Z 1 noenc
# inc 20150222T073233Z 1 noenc
dup_timestamps = []
for line in collection_status.splitlines():
if line.startswith(b' inc '):
assert (len(line) >= len(b' inc 20150222T073233Z'))
dup_timestamps.append(line[5:21])
elif line.startswith(b' full '):
assert (len(line) >= len(b' full 20150222T073233Z'))
dup_timestamps.append(line[6:22])
for i, dup_ts in enumerate(dup_timestamps):
tm = strptime(dup_ts.decode('ascii'), '%Y%m%dT%H%M%SZ')
exc([b'rm', b'-rf', restoredir])
exc(dup + [b'restore', b'-t', dup_ts, source_url, restoredir])
exc([bup_path, b'index', b'-uxf', tmpidx, restoredir])
exc([
bup_path, b'save', b'--strip', b'--date',
b'%d' % timegm(tm), b'-f', tmpidx, b'-n', save_name, restoredir
])
sys.stderr.flush()
finally:
exc([b'rm', b'-rf', tmpdir])
if saved_errors:
log('warning: %d errors encountered\n' % len(saved_errors))
sys.exit(1)
def test_misc():
with no_lingering_errors():
with test_tempdir(b'bup-tvfs-') as tmpdir:
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
git.repodir = bup_dir
data_path = tmpdir + b'/src'
os.mkdir(data_path)
with open(data_path + b'/file', 'wb+') as tmpfile:
tmpfile.write(b'canary\n')
symlink(b'file', data_path + b'/symlink')
ex((bup_path, b'init'))
ex((bup_path, b'index', b'-v', data_path))
ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test',
b'--strip', data_path))
repo = LocalRepo()
wvstart('readlink')
ls_tree = exo((b'git', b'ls-tree', b'test', b'symlink')).out
mode, typ, oidx, name = ls_tree.strip().split(None, 3)
assert name == b'symlink'
link_item = vfs.Item(oid=unhexlify(oidx), meta=int(mode, 8))
wvpasseq(b'file', vfs.readlink(repo, link_item))
ls_tree = exo((b'git', b'ls-tree', b'test', b'file')).out
mode, typ, oidx, name = ls_tree.strip().split(None, 3)
assert name == b'file'
file_item = vfs.Item(oid=unhexlify(oidx), meta=int(mode, 8))
wvexcept(Exception, vfs.readlink, repo, file_item)
wvstart('item_size')
wvpasseq(4, vfs.item_size(repo, link_item))
wvpasseq(7, vfs.item_size(repo, file_item))
meta = metadata.from_path(fsencode(__file__))
meta.size = 42
fake_item = file_item._replace(meta=meta)
wvpasseq(42, vfs.item_size(repo, fake_item))
_, fakelink_item = vfs.resolve(repo, b'/test/latest',
follow=False)[-1]
wvpasseq(17, vfs.item_size(repo, fakelink_item))
wvstart('augment_item_meta')
run_augment_item_meta_tests(repo, b'/test/latest/file', 7,
b'/test/latest/symlink', b'file')
wvstart('copy_item')
# FIXME: this caused StopIteration
#_, file_item = vfs.resolve(repo, '/file')[-1]
_, file_item = vfs.resolve(repo, b'/test/latest/file')[-1]
file_copy = vfs.copy_item(file_item)
wvpass(file_copy is not file_item)
wvpass(file_copy.meta is not file_item.meta)
wvpass(isinstance(file_copy, tuple))
wvpass(file_item.meta.user)
wvpass(file_copy.meta.user)
file_copy.meta.user = None
wvpass(file_item.meta.user)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
opt.output = argv_bytes(opt.output) if opt.output else None
if extra and (opt.auto or opt.force):
o.fatal("you can't use -f/-a and also provide filenames")
if opt.check and (not extra and not opt.auto):
o.fatal("if using --check, you must provide filenames or -a")
if opt.max_files < 0:
opt.max_files = max_files()
assert (opt.max_files >= 5)
if opt.dir:
path = argv_bytes(opt.dir)
else:
path = LocalRepo().packdir()
extra = [argv_bytes(x) for x in extra]
if opt.check:
# check existing midx files
if extra:
midxes = extra
else:
debug1('midx: scanning %s\n' % path)
midxes = glob.glob(os.path.join(path, b'*.midx'))
for name in midxes:
check_midx(name)
if not saved_errors:
log('All tests passed.\n')
else:
if extra:
sys.stdout.flush()
do_midx(path,
opt.output,
extra,
b'',
byte_stream(sys.stdout),
auto=opt.auto,
force=opt.force,
print_names=opt.print)
elif opt.auto or opt.force:
sys.stdout.flush()
debug1('midx: scanning %s\n' % path_msg(path))
do_midx_dir(path,
opt.output,
byte_stream(sys.stdout),
auto=opt.auto,
force=opt.force,
max_files=opt.max_files)
else:
o.fatal("you must use -f or -a or provide input filenames")
if saved_errors:
log('WARNING: %d errors encountered.\n' % len(saved_errors))
sys.exit(1)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal("no arguments expected")
mi = git.PackIdxList(LocalRepo().packdir(), ignore_midx=opt.ignore_midx)
def do_predict(ix, out):
total = len(ix)
maxdiff = 0
for count, i in enumerate(ix):
prefix = struct.unpack('!Q', i[:8])[0]
expected = prefix * total // (1 << 64)
diff = count - expected
maxdiff = max(maxdiff, abs(diff))
out.write(b'%d of %d (%.3f%%) ' %
(maxdiff, len(ix), maxdiff * 100.0 / len(ix)))
out.flush()
assert (count + 1 == len(ix))
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.predict:
if opt.ignore_midx:
for pack in mi.packs:
do_predict(pack, out)
else:
do_predict(mi, out)
else:
# default mode: find longest matching prefix
last = b'\0' * 20
longmatch = 0
for i in mi:
if i == last:
continue
#assert(str(i) >= last)
pm = _helpers.bitmatch(last, i)
longmatch = max(longmatch, pm)
last = i
out.write(b'%d\n' % longmatch)
log('%d matching prefix bits\n' % longmatch)
doublings = math.log(len(mi), 2)
bpd = longmatch / doublings
log('%.2f bits per doubling\n' % bpd)
remain = 160 - longmatch
rdoublings = remain / bpd
log('%d bits (%.2f doublings) remaining\n' % (remain, rdoublings))
larger = 2**rdoublings
log('%g times larger is possible\n' % larger)
perperson = larger / POPULATION_OF_EARTH
log('\nEveryone on earth could have %d data sets like yours, all in one\n'
'repository, and we would expect 1 object collision.\n' %
int(perperson))
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
git.check_repo_or_die()
if not extra:
o.fatal('must specify a target')
if len(extra) > 1:
o.fatal('only one target file allowed')
if opt.bupm and opt.meta:
o.fatal('--meta and --bupm are incompatible')
target = argv_bytes(extra[0])
if not re.match(br'/*[^/]+/[^/]+', target):
o.fatal("path %r doesn't include a branch and revision" % target)
with LocalRepo() as repo:
resolved = vfs.resolve(repo, target, follow=False)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
log('error: cannot access %r in %r\n' %
(b'/'.join(name for name, item in resolved), target))
sys.exit(1)
mode = vfs.item_mode(leaf_item)
sys.stdout.flush()
out = byte_stream(sys.stdout)
if opt.bupm:
if not stat.S_ISDIR(mode):
o.fatal('%r is not a directory' % target)
_, bupm_oid = vfs.tree_data_and_bupm(repo, leaf_item.oid)
if bupm_oid:
with vfs.tree_data_reader(repo, bupm_oid) as meta_stream:
out.write(meta_stream.read())
elif opt.meta:
augmented = vfs.augment_item_meta(repo,
leaf_item,
include_size=True)
out.write(augmented.meta.encode())
else:
if stat.S_ISREG(mode):
with vfs.fopen(repo, leaf_item) as f:
for b in chunkyreader(f):
out.write(b)
else:
o.fatal('%r is not a plain file' % target)
if saved_errors:
log('warning: %d errors encountered\n' % len(saved_errors))
sys.exit(1)
def via_cmdline(args, onabort=None):
"""Output a listing of a file or directory in the bup repository.
When a long listing is not requested and stdout is attached to a
tty, the output is formatted in columns. When not attached to tty
(for example when the output is piped to another command), one
file is listed per line.
"""
opt = opts_from_cmdline(args, onabort=onabort)
return within_repo(
RemoteRepo(opt.remote) if opt.remote else LocalRepo(), opt)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if extra:
o.fatal('no arguments expected')
m = git.PackIdxList(LocalRepo().packdir(), ignore_midx=opt.ignore_midx)
sys.stdout.flush()
out = byte_stream(sys.stdout)
report(-1, out)
_helpers.random_sha()
report(0, out)
if opt.existing:
def foreverit(mi):
while 1:
for e in mi:
yield e
objit = iter(foreverit(m))
for c in range(opt.cycles):
for n in range(opt.number):
if opt.existing:
bin = next(objit)
assert(m.exists(bin))
else:
bin = _helpers.random_sha()
# technically, a randomly generated object id might exist.
# but the likelihood of that is the likelihood of finding
# a collision in sha-1 by accident, which is so unlikely that
# we don't care.
assert(not m.exists(bin))
report((c+1)*opt.number, out)
if bloom._total_searches:
out.write(b'bloom: %d objects searched in %d steps: avg %.3f steps/object\n'
% (bloom._total_searches, bloom._total_steps,
bloom._total_steps*1.0/bloom._total_searches))
if midx._total_searches:
out.write(b'midx: %d objects searched in %d steps: avg %.3f steps/object\n'
% (midx._total_searches, midx._total_steps,
midx._total_steps*1.0/midx._total_searches))
if git._total_searches:
out.write(b'idx: %d objects searched in %d steps: avg %.3f steps/object\n'
% (git._total_searches, git._total_steps,
git._total_steps*1.0/git._total_searches))
out.write(b'Total time: %.3fs\n' % (time.time() - start))
def via_cmdline(args, out=None, onabort=None):
"""Write a listing of a file or directory in the bup repository to out.
When a long listing is not requested and stdout is attached to a
tty, the output is formatted in columns. When not attached to tty
(for example when the output is piped to another command), one
file is listed per line.
"""
assert out
opt = opts_from_cmdline(args, onabort=onabort)
with RemoteRepo(argv_bytes(opt.remote)) if opt.remote \
else LocalRepo() as repo:
return within_repo(repo, opt, out)
def main(argv):
o = Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
if not opt.unsafe:
o.fatal('refusing to run dangerous, experimental command without --unsafe')
if len(extra) < 1:
o.fatal('no paths specified')
check_repo_or_die()
repo = LocalRepo()
bup_rm(repo, [argv_bytes(x) for x in extra],
compression=opt.compress, verbosity=opt.verbose)
die_if_errors()
def main(argv):
global repo
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
git.check_repo_or_die()
sys.stdout.flush()
out = byte_stream(sys.stdout)
stdin = byte_stream(sys.stdin)
with LocalRepo() as repo:
present_interface(stdin, out, extra, repo)
if saved_errors:
log('warning: %d errors encountered\n' % len(saved_errors))
sys.exit(1)
def test_read_and_seek():
# Write a set of randomly sized files containing random data whose
# names are their sizes, and then verify that what we get back
# from the vfs when seeking and reading with various block sizes
# matches the original content.
with no_lingering_errors():
with test_tempdir(b'bup-tvfs-read-') as tmpdir:
resolve = vfs.resolve
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
git.repodir = bup_dir
repo = LocalRepo()
data_path = tmpdir + b'/src'
os.mkdir(data_path)
seed = randint(-(1 << 31), (1 << 31) - 1)
rand = Random()
rand.seed(seed)
print('test_read seed:', seed, file=sys.stderr)
max_size = 2 * 1024 * 1024
sizes = set((rand.randint(1, max_size) for _ in range(5)))
sizes.add(1)
sizes.add(max_size)
for size in sizes:
write_sized_random_content(data_path, size, seed)
ex((bup_path, b'init'))
ex((bup_path, b'index', b'-v', data_path))
ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test',
b'--strip', data_path))
read_sizes = set((rand.randint(1, max_size) for _ in range(10)))
sizes.add(1)
sizes.add(max_size)
print('test_read src sizes:', sizes, file=sys.stderr)
print('test_read read sizes:', read_sizes, file=sys.stderr)
for size in sizes:
res = resolve(repo,
b'/test/latest/' + str(size).encode('ascii'))
_, item = res[-1]
wvpasseq(size, vfs.item_size(repo, res[-1][1]))
validate_vfs_streaming_read(repo, item,
b'%s/%d' % (data_path, size),
read_sizes)
validate_vfs_seeking_read(repo, item,
b'%s/%d' % (data_path, size),
read_sizes)
def test_resolve_loop():
with no_lingering_errors():
with test_tempdir('bup-tvfs-resloop-') as tmpdir:
resolve = vfs.resolve
lresolve = vfs.lresolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
repo = LocalRepo()
data_path = tmpdir + '/src'
os.mkdir(data_path)
symlink('loop', data_path + '/loop')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
data_path))
wvexcept(vfs.Loop, resolve, repo, '/test/latest/loop')
def _init_session(reinit_with_new_repopath=None):
global repo
if reinit_with_new_repopath is None and git.repodir:
if not repo:
repo = LocalRepo()
return
git.check_repo_or_die(reinit_with_new_repopath)
if repo:
repo.close()
repo = LocalRepo()
# OK. we now know the path is a proper repository. Record this path in the
# environment so that subprocesses inherit it and know where to operate.
environ[b'BUP_DIR'] = git.repodir
debug1('bup server: bupdir is %s\n' % path_msg(git.repodir))
_set_mode()
def test_contents_with_mismatched_bupm_git_ordering():
with no_lingering_errors():
with test_tempdir(b'bup-tvfs-') as tmpdir:
bup_dir = tmpdir + b'/bup'
environ[b'GIT_DIR'] = bup_dir
environ[b'BUP_DIR'] = bup_dir
git.repodir = bup_dir
data_path = tmpdir + b'/src'
os.mkdir(data_path)
os.mkdir(data_path + b'/foo')
with open(data_path + b'/foo.', 'wb+') as tmpfile:
tmpfile.write(b'canary\n')
ex((bup_path, b'init'))
ex((bup_path, b'index', b'-v', data_path))
save_utc = 100000
save_name = strftime('%Y-%m-%d-%H%M%S',
localtime(save_utc)).encode('ascii')
ex((bup_path, b'save', b'-tvvn', b'test', b'-d', b'%d' % save_utc,
b'--strip', data_path))
repo = LocalRepo()
tip_sref = exo((b'git', b'show-ref', b'refs/heads/test')).out
tip_oidx = tip_sref.strip().split()[0]
tip_tree_oidx = exo((b'git', b'log', b'--pretty=%T', b'-n1',
tip_oidx)).out.strip()
tip_tree_oid = unhexlify(tip_tree_oidx)
tip_tree = tree_dict(repo, tip_tree_oid)
name, item = vfs.resolve(repo, b'/test/latest')[2]
wvpasseq(save_name, name)
expected = frozenset(
(x.name, vfs.Item(oid=x.oid, meta=x.meta))
for x in (tip_tree[name] for name in (b'.', b'foo', b'foo.')))
contents = tuple(vfs.contents(repo, item))
wvpasseq(expected, frozenset(contents))
# Spot check, in case tree_dict shares too much code with the vfs
name, item = next(((n, i) for n, i in contents if n == b'foo'))
wvpass(S_ISDIR(item.meta))
name, item = next(((n, i) for n, i in contents if n == b'foo.'))
wvpass(S_ISREG(item.meta.mode))
def test_metadata_method():
with no_lingering_errors():
with test_tempdir('bup-tmetadata-') as tmpdir:
bup_dir = tmpdir + '/bup'
data_path = tmpdir + '/foo'
os.mkdir(data_path)
ex('touch', data_path + '/file')
ex('ln', '-s', 'file', data_path + '/symlink')
test_time1 = 13 * 1000000000
test_time2 = 42 * 1000000000
utime(data_path + '/file', (0, test_time1))
lutime(data_path + '/symlink', (0, 0))
utime(data_path, (0, test_time2))
ex(bup_path, '-d', bup_dir, 'init')
ex(bup_path, '-d', bup_dir, 'index', '-v', data_path)
ex(bup_path, '-d', bup_dir, 'save', '-tvvn', 'test', data_path)
git.check_repo_or_die(bup_dir)
repo = LocalRepo()
resolved = vfs.resolve(repo,
'/test/latest' + resolve_parent(data_path),
follow=False)
leaf_name, leaf_item = resolved[-1]
m = leaf_item.meta
WVPASS(m.mtime == test_time2)
WVPASS(leaf_name == 'foo')
contents = tuple(vfs.contents(repo, leaf_item))
WVPASS(len(contents) == 3)
WVPASSEQ(frozenset(name for name, item in contents),
frozenset(('.', 'file', 'symlink')))
for name, item in contents:
if name == 'file':
m = item.meta
WVPASS(m.mtime == test_time1)
elif name == 'symlink':
m = item.meta
WVPASSEQ(m.symlink_target, 'file')
WVPASSEQ(m.size, 4)
WVPASSEQ(m.mtime, 0)
def test_resolve_loop():
with no_lingering_errors():
with test_tempdir('bup-tvfs-resloop-') as tmpdir:
resolve = vfs.resolve
lresolve = vfs.lresolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
git.repodir = bup_dir
repo = LocalRepo()
data_path = tmpdir + '/src'
os.mkdir(data_path)
symlink('loop', data_path + '/loop')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
data_path))
try:
resolve(repo, '/test/latest/loop')
except vfs.IOError as res_ex:
wvpasseq(ELOOP, res_ex.errno)
wvpasseq(['', 'test', 'latest', 'loop'],
[name for name, item in res_ex.terminus])
def main():
handle_ctrl_c()
is_reverse = environ.get(b'BUP_SERVER_REVERSE')
opt = parse_args(compat.argv)
git.check_repo_or_die()
if opt.source:
opt.source = argv_bytes(opt.source)
if opt.bwlimit:
client.bwlimit = parse_num(opt.bwlimit)
if is_reverse and opt.remote:
misuse("don't use -r in reverse mode; it's automatic")
if opt.remote:
opt.remote = argv_bytes(opt.remote)
if opt.remote or is_reverse:
dest_repo = RemoteRepo(opt.remote)
else:
dest_repo = LocalRepo()
with dest_repo as dest_repo:
with LocalRepo(repo_dir=opt.source) as src_repo:
with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
# Resolve and validate all sources and destinations,
# implicit or explicit, and do it up-front, so we can
# fail before we start writing (for any obviously
# broken cases).
target_items = resolve_targets(opt.target_specs,
src_repo, dest_repo)
updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
no_ref_info = (None, None)
handlers = {'ff': handle_ff,
'append': handle_append,
'force-pick': handle_pick,
'pick': handle_pick,
'new-tag': handle_new_tag,
'replace': handle_replace,
'unnamed': handle_unnamed}
for item in target_items:
debug1('get-spec: %r\n' % (item.spec,))
debug1('get-src: %s\n' % loc_desc(item.src))
debug1('get-dest: %s\n' % loc_desc(item.dest))
dest_path = item.dest and item.dest.path
if dest_path:
if dest_path.startswith(b'/.tag/'):
dest_ref = b'refs/tags/%s' % dest_path[6:]
else:
dest_ref = b'refs/heads/%s' % dest_path[1:]
else:
dest_ref = None
dest_hash = item.dest and item.dest.hash
orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
orig_ref = orig_ref or dest_hash
cur_ref = cur_ref or dest_hash
handler = handlers[item.spec.method]
item_result = handler(item, src_repo, writer, opt)
if len(item_result) > 1:
new_id, tree = item_result
else:
new_id = item_result[0]
if not dest_ref:
log_item(item.spec.src, item.src.type, opt)
else:
updated_refs[dest_ref] = (orig_ref, new_id)
if dest_ref.startswith(b'refs/tags/'):
log_item(item.spec.src, item.src.type, opt, tag=new_id)
else:
log_item(item.spec.src, item.src.type, opt,
tree=tree, commit=new_id)
# Only update the refs at the very end, once the writer is
# closed, so that if something goes wrong above, the old refs
# will be undisturbed.
for ref_name, info in items(updated_refs):
orig_ref, new_ref = info
try:
dest_repo.update_ref(ref_name, new_ref, orig_ref)
if opt.verbose:
new_hex = hexlify(new_ref)
if orig_ref:
orig_hex = hexlify(orig_ref)
log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
else:
log('updated %r (%s)\n' % (ref_name, new_hex))
except (git.GitError, client.ClientError) as ex:
add_error('unable to update ref %r: %s' % (ref_name, ex))
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
sys.exit(1)
def main(argv):
o = options.Options(optspec)
opt, flags, extra = o.parse_bytes(argv[1:])
verbosity = (opt.verbose or 0) if not opt.quiet else -1
if opt.remote:
opt.remote = argv_bytes(opt.remote)
if opt.outdir:
opt.outdir = argv_bytes(opt.outdir)
git.check_repo_or_die()
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
top = fsencode(os.getcwd())
hardlinks = {}
for path in [argv_bytes(x) for x in extra]:
if not valid_restore_path(path):
add_error("path %r doesn't include a branch and revision" % path)
continue
try:
resolved = vfs.resolve(repo, path, want_meta=True, follow=False)
except vfs.IOError as e:
add_error(e)
continue
if len(resolved) == 3 and resolved[2][0] == b'latest':
# Follow latest symlink to the actual save
try:
resolved = vfs.resolve(repo,
b'latest',
parent=resolved[:-1],
want_meta=True)
except vfs.IOError as e:
add_error(e)
continue
# Rename it back to 'latest'
resolved = tuple(elt if i != 2 else (b'latest', ) + elt[1:]
for i, elt in enumerate(resolved))
path_parent, path_name = os.path.split(path)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
add_error('error: cannot access %r in %r' %
(b'/'.join(name for name, item in resolved), path))
continue
if not path_name or path_name == b'.':
# Source is /foo/what/ever/ or /foo/what/ever/. -- extract
# what/ever/* to the current directory, and if name == '.'
# (i.e. /foo/what/ever/.), then also restore what/ever's
# metadata to the current directory.
treeish = vfs.item_mode(leaf_item)
if not treeish:
add_error('%r cannot be restored as a directory' % path)
else:
items = vfs.contents(repo, leaf_item, want_meta=True)
dot, leaf_item = next(items, None)
assert dot == b'.'
for sub_name, sub_item in items:
restore(repo, b'', sub_name, sub_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if path_name == b'.':
leaf_item = vfs.augment_item_meta(repo,
leaf_item,
include_size=True)
apply_metadata(leaf_item.meta, b'.', opt.numeric_ids,
owner_map)
else:
restore(repo, b'', leaf_name, leaf_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if verbosity >= 0:
progress('Restoring: %d, done.\n' % total_restored)
die_if_errors()
log('Error printing traceback: %s\n' % e2)
log('\nError in completion: %s\n' % e)
optspec = """
bup ftp [commands...]
"""
o = options.Options(optspec)
opt, flags, extra = o.parse(compat.argv[1:])
git.check_repo_or_die()
sys.stdout.flush()
out = byte_stream(sys.stdout)
stdin = byte_stream(sys.stdin)
repo = LocalRepo()
pwd = vfs.resolve(repo, b'/')
rv = 0
if extra:
lines = (argv_bytes(arg) for arg in extra)
else:
if hasattr(_helpers, 'readline'):
_helpers.set_completer_word_break_characters(b' \t\n\r/')
_helpers.set_attempted_completion_function(attempt_completion)
_helpers.set_completion_entry_function(enter_completion)
if sys.platform.startswith('darwin'):
# MacOS uses a slightly incompatible clone of libreadline
_helpers.parse_and_bind(b'bind ^I rl_complete')
_helpers.parse_and_bind(b'tab: complete')
lines = inputiter()
def main():
o = options.Options(optspec)
opt, flags, extra = o.parse(sys.argv[1:])
verbosity = opt.verbose if not opt.quiet else -1
git.check_repo_or_die()
if not extra:
o.fatal('must specify at least one filename to restore')
exclude_rxs = parse_rx_excludes(flags, o.fatal)
owner_map = {}
for map_type in ('user', 'group', 'uid', 'gid'):
owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
if opt.outdir:
mkdirp(opt.outdir)
os.chdir(opt.outdir)
repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
top = os.getcwd()
hardlinks = {}
for path in extra:
if not valid_restore_path(path):
add_error("path %r doesn't include a branch and revision" % path)
continue
try:
resolved = vfs2.lresolve(repo, path, want_meta=True)
except vfs2.IOError as e:
add_error(e)
continue
path_parent, path_name = os.path.split(path)
leaf_name, leaf_item = resolved[-1]
if not leaf_item:
add_error('error: cannot access %r in %r' %
('/'.join(name for name, item in resolved), path))
continue
if not path_name or path_name == '.':
# Source is /foo/what/ever/ or /foo/what/ever/. -- extract
# what/ever/* to the current directory, and if name == '.'
# (i.e. /foo/what/ever/.), then also restore what/ever's
# metadata to the current directory.
treeish = vfs2.item_mode(leaf_item)
if not treeish:
add_error('%r cannot be restored as a directory' % path)
else:
items = vfs2.contents(repo, leaf_item, want_meta=True)
dot, leaf_item = next(items, None)
assert (dot == '.')
for sub_name, sub_item in items:
restore(repo, '', sub_name, sub_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if path_name == '.':
leaf_item = vfs2.augment_item_meta(repo,
leaf_item,
include_size=True)
apply_metadata(leaf_item.meta, '.', opt.numeric_ids,
owner_map)
else:
restore(repo, '', leaf_name, leaf_item, top, opt.sparse,
opt.numeric_ids, owner_map, exclude_rxs, verbosity,
hardlinks)
if verbosity >= 0:
progress('Restoring: %d, done.\n' % total_restored)
die_if_errors()
def test_local_resolve_loop():
prep_and_test_repo(b'local-vfs-resolve-loop',
lambda x: LocalRepo(x), test_resolve_loop)
