def save_index(tmax):
print('saving index')
_hlinks.prepare_save()
if _ri.exists():
_ri.save()
_wi.flush()
if _wi.count:
wr = _wi.new_reader()
mi = index.Writer(indexfile, _msw, tmax)
for e in index.merge(_ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
_ri.close()
mi.close()
wr.close()
_wi.abort()
else:
_wi.close()
_msw.close()
_hlinks.commit_save()
def index_dirty():
unlink('index.meta.tmp')
unlink('index2.meta.tmp')
unlink('index3.meta.tmp')
default_meta = metadata.Metadata()
ms1 = index.MetaStoreWriter('index.meta.tmp')
ms2 = index.MetaStoreWriter('index2.meta.tmp')
ms3 = index.MetaStoreWriter('index3.meta.tmp')
meta_ofs1 = ms1.store(default_meta)
meta_ofs2 = ms2.store(default_meta)
meta_ofs3 = ms3.store(default_meta)
unlink('index.tmp')
unlink('index2.tmp')
unlink('index3.tmp')
ds = xstat.stat('.')
fs = xstat.stat('tindex.py')
tmax = (time.time() - 1) * 10**9
w1 = index.Writer('index.tmp', ms1, tmax)
w1.add('/a/b/x', fs, meta_ofs1)
w1.add('/a/b/c', fs, meta_ofs1)
w1.add('/a/b/', ds, meta_ofs1)
w1.add('/a/', ds, meta_ofs1)
#w1.close()
WVPASS()
w2 = index.Writer('index2.tmp', ms2, tmax)
w2.add('/a/b/n/2', fs, meta_ofs2)
#w2.close()
WVPASS()
w3 = index.Writer('index3.tmp', ms3, tmax)
w3.add('/a/c/n/3', fs, meta_ofs3)
#w3.close()
WVPASS()
r1 = w1.new_reader()
r2 = w2.new_reader()
r3 = w3.new_reader()
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all,
['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all,
['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all,
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c',
'/a/b/', '/a/', '/'])
fake_validate(r1)
dump(r1)
print [hex(e.flags) for e in r1]
def index_dirty():
initial_failures = wvfailure_count()
orig_cwd = os.getcwd()
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-tindex-')
try:
os.chdir(tmpdir)
default_meta = metadata.Metadata()
ms1 = index.MetaStoreWriter('index.meta.tmp')
ms2 = index.MetaStoreWriter('index2.meta.tmp')
ms3 = index.MetaStoreWriter('index3.meta.tmp')
meta_ofs1 = ms1.store(default_meta)
meta_ofs2 = ms2.store(default_meta)
meta_ofs3 = ms3.store(default_meta)
ds = xstat.stat(lib_t_dir)
fs = xstat.stat(lib_t_dir + '/tindex.py')
tmax = (time.time() - 1) * 10**9
w1 = index.Writer('index.tmp', ms1, tmax)
w1.add('/a/b/x', fs, meta_ofs1)
w1.add('/a/b/c', fs, meta_ofs1)
w1.add('/a/b/', ds, meta_ofs1)
w1.add('/a/', ds, meta_ofs1)
#w1.close()
WVPASS()
w2 = index.Writer('index2.tmp', ms2, tmax)
w2.add('/a/b/n/2', fs, meta_ofs2)
#w2.close()
WVPASS()
w3 = index.Writer('index3.tmp', ms3, tmax)
w3.add('/a/c/n/3', fs, meta_ofs3)
#w3.close()
WVPASS()
r1 = w1.new_reader()
r2 = w2.new_reader()
r3 = w3.new_reader()
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all,
['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all,
['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all,
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c',
'/a/b/', '/a/', '/'])
fake_validate(r1)
dump(r1)
print [hex(e.flags) for e in r1]
def index_dirty():
unlink('index.tmp')
unlink('index2.tmp')
ds = xstat.stat('.')
fs = xstat.stat('tindex.py')
tmax = time.time() - 1
w1 = index.Writer('index.tmp', tmax)
w1.add('/a/b/x', fs)
w1.add('/a/b/c', fs)
w1.add('/a/b/', ds)
w1.add('/a/', ds)
#w1.close()
WVPASS()
w2 = index.Writer('index2.tmp', tmax)
w2.add('/a/b/n/2', fs)
#w2.close()
WVPASS()
w3 = index.Writer('index3.tmp', tmax)
w3.add('/a/c/n/3', fs)
#w3.close()
WVPASS()
r1 = w1.new_reader()
r2 = w2.new_reader()
r3 = w3.new_reader()
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all,
['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all,
['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all,
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c',
'/a/b/', '/a/', '/'])
fake_validate(r1)
dump(r1)
print [hex(e.flags) for e in r1]
def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions):
# tmax and start must be epoch nanoseconds.
tmax = (time.time() - 1) * 10**9
ri = index.Reader(indexfile)
msw = index.MetaStoreWriter(indexfile + '.meta')
wi = index.Writer(indexfile, msw, tmax)
rig = IterHelper(ri.iter(name=top))
tstart = int(time.time()) * 10**9
hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
fake_hash = None
if opt.fake_valid:
def fake_hash(name):
return (GIT_MODE_FILE, index.FAKE_SHA)
total = 0
bup_dir = os.path.abspath(git.repo())
index_start = time.time()
for path, pst in recursive_dirlist([top],
xdev=opt.xdev,
bup_dir=bup_dir,
excluded_paths=excluded_paths,
exclude_rxs=exclude_rxs,
xdev_exceptions=xdev_exceptions):
if opt.verbose >= 2 or (opt.verbose == 1
and stat.S_ISDIR(pst.st_mode)):
sys.stdout.write('%s\n' % path)
sys.stdout.flush()
elapsed = time.time() - index_start
paths_per_sec = total / elapsed if elapsed else 0
qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
elif not (total % 128):
elapsed = time.time() - index_start
paths_per_sec = total / elapsed if elapsed else 0
qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
total += 1
while rig.cur and rig.cur.name > path: # deleted paths
if rig.cur.exists():
rig.cur.set_deleted()
rig.cur.repack()
if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
hlinks.del_path(rig.cur.name)
rig.next()
if rig.cur and rig.cur.name == path: # paths that already existed
need_repack = False
if (rig.cur.stale(pst, tstart, check_device=opt.check_device)):
try:
meta = metadata.from_path(path, statinfo=pst)
except (OSError, IOError) as e:
add_error(e)
rig.next()
continue
if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
hlinks.del_path(rig.cur.name)
if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
hlinks.add_path(path, pst.st_dev, pst.st_ino)
# Clear these so they don't bloat the store -- they're
# already in the index (since they vary a lot and they're
# fixed length). If you've noticed "tmax", you might
# wonder why it's OK to do this, since that code may
# adjust (mangle) the index mtime and ctime -- producing
# fake values which must not end up in a .bupm. However,
# it looks like that shouldn't be possible: (1) When
# "save" validates the index entry, it always reads the
# metadata from the filesytem. (2) Metadata is only
# read/used from the index if hashvalid is true. (3)
# "faked" entries will be stale(), and so we'll invalidate
# them below.
meta.ctime = meta.mtime = meta.atime = 0
meta_ofs = msw.store(meta)
rig.cur.update_from_stat(pst, meta_ofs)
rig.cur.invalidate()
need_repack = True
if not (rig.cur.flags & index.IX_HASHVALID):
if fake_hash:
rig.cur.gitmode, rig.cur.sha = fake_hash(path)
rig.cur.flags |= index.IX_HASHVALID
need_repack = True
if opt.fake_invalid:
rig.cur.invalidate()
need_repack = True
if need_repack:
rig.cur.repack()
rig.next()
else: # new paths
try:
meta = metadata.from_path(path, statinfo=pst)
except (OSError, IOError) as e:
add_error(e)
continue
# See same assignment to 0, above, for rationale.
meta.atime = meta.mtime = meta.ctime = 0
meta_ofs = msw.store(meta)
wi.add(path, pst, meta_ofs, hashgen=fake_hash)
if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
hlinks.add_path(path, pst.st_dev, pst.st_ino)
elapsed = time.time() - index_start
paths_per_sec = total / elapsed if elapsed else 0
progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
hlinks.prepare_save()
if ri.exists():
ri.save()
wi.flush()
if wi.count:
wr = wi.new_reader()
if opt.check:
log('check: before merging: oldfile\n')
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
mi = index.Writer(indexfile, msw, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
ri.close()
mi.close()
wr.close()
wi.abort()
else:
wi.close()
msw.close()
hlinks.commit_save()
hlinks.prepare_save()
if ri.exists():
ri.save()
wi.flush()
if wi.count:
wr = wi.new_reader()
if opt.check:
log('check: before merging: oldfile\n')
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
mi = index.Writer(indexfile, msw, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
ri.close()
mi.close()
wr.close()
wi.abort()
else:
wi.close()
msw.close()
hlinks.commit_save()
optspec = """
def update_index(top, excluded_paths):
# tmax and start must be epoch nanoseconds.
tmax = (time.time() - 1) * 10**9
ri = index.Reader(indexfile)
msw = index.MetaStoreWriter(indexfile + '.meta')
wi = index.Writer(indexfile, msw, tmax)
rig = IterHelper(ri.iter(name=top))
tstart = int(time.time()) * 10**9
hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
hashgen = None
if opt.fake_valid:
def hashgen(name):
return (GIT_MODE_FILE, index.FAKE_SHA)
total = 0
bup_dir = os.path.abspath(git.repo())
for (path,pst) in drecurse.recursive_dirlist([top], xdev=opt.xdev,
bup_dir=bup_dir,
excluded_paths=excluded_paths):
if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
sys.stdout.write('%s\n' % path)
sys.stdout.flush()
qprogress('Indexing: %d\r' % total)
elif not (total % 128):
qprogress('Indexing: %d\r' % total)
total += 1
while rig.cur and rig.cur.name > path: # deleted paths
if rig.cur.exists():
rig.cur.set_deleted()
rig.cur.repack()
if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
hlinks.del_path(rig.cur.name)
rig.next()
if rig.cur and rig.cur.name == path: # paths that already existed
if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
hlinks.del_path(rig.cur.name)
if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
hlinks.add_path(path, pst.st_dev, pst.st_ino)
meta = metadata.from_path(path, statinfo=pst)
# Clear these so they don't bloat the store -- they're
# already in the index (since they vary a lot and they're
# fixed length). If you've noticed "tmax", you might
# wonder why it's OK to do this, since that code may
# adjust (mangle) the index mtime and ctime -- producing
# fake values which must not end up in a .bupm. However,
# it looks like that shouldn't be possible: (1) When
# "save" validates the index entry, it always reads the
# metadata from the filesytem. (2) Metadata is only
# read/used from the index if hashvalid is true. (3) index
# always invalidates "faked" entries, because "old != new"
# in from_stat().
meta.ctime = meta.mtime = meta.atime = 0
meta_ofs = msw.store(meta)
rig.cur.from_stat(pst, meta_ofs, tstart)
if not (rig.cur.flags & index.IX_HASHVALID):
if hashgen:
(rig.cur.gitmode, rig.cur.sha) = hashgen(path)
rig.cur.flags |= index.IX_HASHVALID
if opt.fake_invalid:
rig.cur.invalidate()
rig.cur.repack()
rig.next()
else: # new paths
meta = metadata.from_path(path, statinfo=pst)
# See same assignment to 0, above, for rationale.
meta.atime = meta.mtime = meta.ctime = 0
meta_ofs = msw.store(meta)
wi.add(path, pst, meta_ofs, hashgen = hashgen)
if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
hlinks.add_path(path, pst.st_dev, pst.st_ino)
progress('Indexing: %d, done.\n' % total)
hlinks.prepare_save()
if ri.exists():
ri.save()
wi.flush()
if wi.count:
wr = wi.new_reader()
if opt.check:
log('check: before merging: oldfile\n')
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
mi = index.Writer(indexfile, msw, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
ri.close()
mi.close()
wr.close()
wi.abort()
else:
wi.close()
msw.close()
hlinks.commit_save()
def index_dirty():
unlink('index.tmp')
unlink('index2.tmp')
ds = xstat.stat('.')
fs = xstat.stat('tindex.py')
tmax = time.time() - 1
w1 = index.Writer('index.tmp', tmax)
w1.add('/a/b/x', fs)
w1.add('/a/b/c', fs)
w1.add('/a/b/', ds)
w1.add('/a/', ds)
#w1.close()
WVPASS()
w2 = index.Writer('index2.tmp', tmax)
w2.add('/a/b/n/2', fs)
#w2.close()
WVPASS()
w3 = index.Writer('index3.tmp', tmax)
w3.add('/a/c/n/3', fs)
#w3.close()
WVPASS()
r1 = w1.new_reader()
r2 = w2.new_reader()
r3 = w3.new_reader()
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all,
['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all,
['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all,
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c',
'/a/b/', '/a/', '/'])
fake_validate(r1)
dump(r1)
print [hex(e.flags) for e in r1]
WVPASSEQ([e.name for e in r1 if e.is_valid()], r1all)
WVPASSEQ([e.name for e in r1 if not e.is_valid()], [])
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()],
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
expect_invalid = ['/'] + r2all + r3all
expect_real = (set(r1all) - set(r2all) - set(r3all)) \
| set(['/a/b/n/2', '/a/c/n/3'])
dump(index.merge(r2, r1, r3))
for e in index.merge(r2, r1, r3):
print e.name, hex(e.flags), e.ctime
eiv = e.name in expect_invalid
er = e.name in expect_real
WVPASSEQ(eiv, not e.is_valid())
WVPASSEQ(er, e.is_real())
fake_validate(r2, r3)
dump(index.merge(r2, r1, r3))
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()], [])
e = eget(index.merge(r2, r1, r3), '/a/b/c')
e.invalidate()
e.repack()
dump(index.merge(r2, r1, r3))
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()],
['/a/b/c', '/a/b/', '/a/', '/'])
def index_dirty():
with no_lingering_errors():
with test_tempdir('bup-tindex-') as tmpdir:
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
default_meta = metadata.Metadata()
ms1 = index.MetaStoreWriter('index.meta.tmp')
ms2 = index.MetaStoreWriter('index2.meta.tmp')
ms3 = index.MetaStoreWriter('index3.meta.tmp')
meta_ofs1 = ms1.store(default_meta)
meta_ofs2 = ms2.store(default_meta)
meta_ofs3 = ms3.store(default_meta)
ds = xstat.stat(lib_t_dir)
fs = xstat.stat(lib_t_dir + '/tindex.py')
tmax = (time.time() - 1) * 10**9
w1 = index.Writer('index.tmp', ms1, tmax)
w1.add('/a/b/x', fs, meta_ofs1)
w1.add('/a/b/c', fs, meta_ofs1)
w1.add('/a/b/', ds, meta_ofs1)
w1.add('/a/', ds, meta_ofs1)
#w1.close()
WVPASS()
w2 = index.Writer('index2.tmp', ms2, tmax)
w2.add('/a/b/n/2', fs, meta_ofs2)
#w2.close()
WVPASS()
w3 = index.Writer('index3.tmp', ms3, tmax)
w3.add('/a/c/n/3', fs, meta_ofs3)
#w3.close()
WVPASS()
r1 = w1.new_reader()
r2 = w2.new_reader()
r3 = w3.new_reader()
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all, ['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all, ['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all, ['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all, [
'/a/c/n/3', '/a/c/n/', '/a/c/', '/a/b/x', '/a/b/n/2',
'/a/b/n/', '/a/b/c', '/a/b/', '/a/', '/'
])
fake_validate(r1)
dump(r1)
print[hex(e.flags) for e in r1]
WVPASSEQ([e.name for e in r1 if e.is_valid()], r1all)
WVPASSEQ([e.name for e in r1 if not e.is_valid()], [])
WVPASSEQ([
e.name
for e in index.merge(r2, r1, r3) if not e.is_valid()
], [
'/a/c/n/3', '/a/c/n/', '/a/c/', '/a/b/n/2', '/a/b/n/',
'/a/b/', '/a/', '/'
])
expect_invalid = ['/'] + r2all + r3all
expect_real = (set(r1all) - set(r2all) - set(r3all)) \
| set(['/a/b/n/2', '/a/c/n/3'])
dump(index.merge(r2, r1, r3))
for e in index.merge(r2, r1, r3):
print e.name, hex(e.flags), e.ctime
eiv = e.name in expect_invalid
er = e.name in expect_real
WVPASSEQ(eiv, not e.is_valid())
WVPASSEQ(er, e.is_real())
fake_validate(r2, r3)
dump(index.merge(r2, r1, r3))
WVPASSEQ([
e.name
for e in index.merge(r2, r1, r3) if not e.is_valid()
], [])
e = eget(index.merge(r2, r1, r3), '/a/b/c')
e.invalidate()
e.repack()
dump(index.merge(r2, r1, r3))
WVPASSEQ([
e.name
for e in index.merge(r2, r1, r3) if not e.is_valid()
], ['/a/b/c', '/a/b/', '/a/', '/'])
w1.close()
w2.close()
w3.close()
finally:
os.chdir(orig_cwd)
['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all,
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c',
'/a/b/', '/a/', '/'])
fake_validate(r1)
dump(r1)
print [hex(e.flags) for e in r1]
WVPASSEQ([e.name for e in r1 if e.is_valid()], r1all)
WVPASSEQ([e.name for e in r1 if not e.is_valid()], [])
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()],
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
expect_invalid = ['/'] + r2all + r3all
expect_real = (set(r1all) - set(r2all) - set(r3all)) \
| set(['/a/b/n/2', '/a/c/n/3'])
dump(index.merge(r2, r1, r3))
for e in index.merge(r2, r1, r3):
print e.name, hex(e.flags), e.ctime
eiv = e.name in expect_invalid
er = e.name in expect_real
WVPASSEQ(eiv, not e.is_valid())
WVPASSEQ(er, e.is_real())
fake_validate(r2, r3)
dump(index.merge(r2, r1, r3))
def update_index(top, excluded_paths):
tmax = time.time() - 1
ri = index.Reader(indexfile)
wi = index.Writer(indexfile, tmax)
rig = IterHelper(ri.iter(name=top))
tstart = int(time.time())
hashgen = None
if opt.fake_valid:
def hashgen(name):
return (GIT_MODE_FILE, index.FAKE_SHA)
total = 0
bup_dir = os.path.abspath(git.repo())
for (path,
pst) in drecurse.recursive_dirlist([top],
xdev=opt.xdev,
bup_dir=bup_dir,
excluded_paths=excluded_paths):
if opt.verbose >= 2 or (opt.verbose == 1
and stat.S_ISDIR(pst.st_mode)):
sys.stdout.write('%s\n' % path)
sys.stdout.flush()
qprogress('Indexing: %d\r' % total)
elif not (total % 128):
qprogress('Indexing: %d\r' % total)
total += 1
while rig.cur and rig.cur.name > path: # deleted paths
if rig.cur.exists():
rig.cur.set_deleted()
rig.cur.repack()
rig.next()
if rig.cur and rig.cur.name == path: # paths that already existed
if pst:
rig.cur.from_stat(pst, tstart)
if not (rig.cur.flags & index.IX_HASHVALID):
if hashgen:
(rig.cur.gitmode, rig.cur.sha) = hashgen(path)
rig.cur.flags |= index.IX_HASHVALID
if opt.fake_invalid:
rig.cur.invalidate()
rig.cur.repack()
rig.next()
else: # new paths
wi.add(path, pst, hashgen=hashgen)
progress('Indexing: %d, done.\n' % total)
if ri.exists():
ri.save()
wi.flush()
if wi.count:
wr = wi.new_reader()
if opt.check:
log('check: before merging: oldfile\n')
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
mi = index.Writer(indexfile, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
ri.close()
mi.close()
wr.close()
wi.abort()
else:
wi.close()
def update_index(top, excluded_paths):
tmax = time.time() - 1
ri = index.Reader(indexfile)
wi = index.Writer(indexfile, tmax)
rig = IterHelper(ri.iter(name=top))
tstart = int(time.time())
hashgen = None
if opt.fake_valid:
def hashgen(name):
return (GIT_MODE_FILE, index.FAKE_SHA)
total = 0
bup_dir = os.path.abspath(git.repo())
for (path,pst) in drecurse.recursive_dirlist([top], xdev=opt.xdev,
bup_dir=bup_dir,
excluded_paths=excluded_paths):
if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
sys.stdout.write('%s\n' % path)
sys.stdout.flush()
qprogress('Indexing: %d\r' % total)
elif not (total % 128):
qprogress('Indexing: %d\r' % total)
total += 1
while rig.cur and rig.cur.name > path: # deleted paths
if rig.cur.exists():
rig.cur.set_deleted()
rig.cur.repack()
rig.next()
if rig.cur and rig.cur.name == path: # paths that already existed
if pst:
rig.cur.from_stat(pst, tstart)
if not (rig.cur.flags & index.IX_HASHVALID):
if hashgen:
(rig.cur.gitmode, rig.cur.sha) = hashgen(path)
rig.cur.flags |= index.IX_HASHVALID
if opt.fake_invalid:
rig.cur.invalidate()
rig.cur.repack()
rig.next()
else: # new paths
wi.add(path, pst, hashgen = hashgen)
progress('Indexing: %d, done.\n' % total)
if ri.exists():
ri.save()
wi.flush()
if wi.count:
wr = wi.new_reader()
if opt.check:
log('check: before merging: oldfile\n')
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
mi = index.Writer(indexfile, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
ri.close()
mi.close()
wr.close()
wi.abort()
else:
wi.close()
def index_dirty():
with no_lingering_errors():
with test_tempdir('bup-tindex-') as tmpdir:
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
default_meta = metadata.Metadata()
ms1 = index.MetaStoreWriter('index.meta.tmp')
ms2 = index.MetaStoreWriter('index2.meta.tmp')
ms3 = index.MetaStoreWriter('index3.meta.tmp')
meta_ofs1 = ms1.store(default_meta)
meta_ofs2 = ms2.store(default_meta)
meta_ofs3 = ms3.store(default_meta)
ds = xstat.stat(lib_t_dir)
fs = xstat.stat(lib_t_dir + '/tindex.py')
tmax = (time.time() - 1) * 10**9
w1 = index.Writer('index.tmp', ms1, tmax)
w1.add('/a/b/x', fs, meta_ofs1)
w1.add('/a/b/c', fs, meta_ofs1)
w1.add('/a/b/', ds, meta_ofs1)
w1.add('/a/', ds, meta_ofs1)
#w1.close()
WVPASS()
w2 = index.Writer('index2.tmp', ms2, tmax)
w2.add('/a/b/n/2', fs, meta_ofs2)
#w2.close()
WVPASS()
w3 = index.Writer('index3.tmp', ms3, tmax)
w3.add('/a/c/n/3', fs, meta_ofs3)
#w3.close()
WVPASS()
r1 = w1.new_reader()
r2 = w2.new_reader()
r3 = w3.new_reader()
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all,
['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all,
['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all,
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c',
'/a/b/', '/a/', '/'])
fake_validate(r1)
dump(r1)
print [hex(e.flags) for e in r1]
WVPASSEQ([e.name for e in r1 if e.is_valid()], r1all)
WVPASSEQ([e.name for e in r1 if not e.is_valid()], [])
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()],
['/a/c/n/3', '/a/c/n/', '/a/c/',
'/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/'])
expect_invalid = ['/'] + r2all + r3all
expect_real = (set(r1all) - set(r2all) - set(r3all)) \
| set(['/a/b/n/2', '/a/c/n/3'])
dump(index.merge(r2, r1, r3))
for e in index.merge(r2, r1, r3):
print e.name, hex(e.flags), e.ctime
eiv = e.name in expect_invalid
er = e.name in expect_real
WVPASSEQ(eiv, not e.is_valid())
WVPASSEQ(er, e.is_real())
fake_validate(r2, r3)
dump(index.merge(r2, r1, r3))
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()], [])
e = eget(index.merge(r2, r1, r3), '/a/b/c')
e.invalidate()
e.repack()
dump(index.merge(r2, r1, r3))
WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()],
['/a/b/c', '/a/b/', '/a/', '/'])
w1.close()
w2.close()
w3.close()
finally:
hlinks.prepare_save()
if ri.exists():
ri.save()
wi.flush()
if wi.count:
wr = wi.new_reader()
if opt.check:
log('check: before merging: oldfile\n')
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
mi = index.Writer(indexfile, msw, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
mi.add_ixentry(e)
ri.close()
mi.close()
wr.close()
wi.abort()
else:
wi.close()
msw.close()
hlinks.commit_save()
optspec = """
def test_index_dirty(tmpdir):
orig_cwd = os.getcwd()
try:
os.chdir(tmpdir)
default_meta = metadata.Metadata()
with index.MetaStoreWriter(b'index.meta.tmp') as ms1, \
index.MetaStoreWriter(b'index2.meta.tmp') as ms2, \
index.MetaStoreWriter(b'index3.meta.tmp') as ms3:
meta_ofs1 = ms1.store(default_meta)
meta_ofs2 = ms2.store(default_meta)
meta_ofs3 = ms3.store(default_meta)
ds = xstat.stat(lib_t_dir)
fs = xstat.stat(lib_t_dir + b'/test_index.py')
tmax = (time.time() - 1) * 10**9
with index.Writer(b'index.tmp', ms1, tmax) as w1, \
index.Writer(b'index2.tmp', ms2, tmax) as w2, \
index.Writer(b'index3.tmp', ms3, tmax) as w3:
w1.add(b'/a/b/x', fs, meta_ofs1)
w1.add(b'/a/b/c', fs, meta_ofs1)
w1.add(b'/a/b/', ds, meta_ofs1)
w1.add(b'/a/', ds, meta_ofs1)
#w1.close()
WVPASS()
w2.add(b'/a/b/n/2', fs, meta_ofs2)
#w2.close()
WVPASS()
w3.add(b'/a/c/n/3', fs, meta_ofs3)
#w3.close()
WVPASS()
with w1.new_reader() as r1, \
w2.new_reader() as r2, \
w3.new_reader() as r3:
WVPASS()
r1all = [e.name for e in r1]
WVPASSEQ(r1all,
[b'/a/b/x', b'/a/b/c', b'/a/b/', b'/a/', b'/'])
r2all = [e.name for e in r2]
WVPASSEQ(r2all,
[b'/a/b/n/2', b'/a/b/n/', b'/a/b/', b'/a/', b'/'])
r3all = [e.name for e in r3]
WVPASSEQ(r3all,
[b'/a/c/n/3', b'/a/c/n/', b'/a/c/', b'/a/', b'/'])
all = [e.name for e in index.merge(r2, r1, r3)]
WVPASSEQ(all, [
b'/a/c/n/3', b'/a/c/n/', b'/a/c/', b'/a/b/x',
b'/a/b/n/2', b'/a/b/n/', b'/a/b/c', b'/a/b/', b'/a/',
b'/'
])
fake_validate(r1)
dump(r1)
print([hex(e.flags) for e in r1])
WVPASSEQ([e.name for e in r1 if e.is_valid()], r1all)
WVPASSEQ([e.name for e in r1 if not e.is_valid()], [])
WVPASSEQ([
e.name
for e in index.merge(r2, r1, r3) if not e.is_valid()
], [
b'/a/c/n/3', b'/a/c/n/', b'/a/c/', b'/a/b/n/2',
b'/a/b/n/', b'/a/b/', b'/a/', b'/'
])
expect_invalid = [b'/'] + r2all + r3all
expect_real = (set(r1all) - set(r2all) - set(r3all)) \
| set([b'/a/b/n/2', b'/a/c/n/3'])
dump(index.merge(r2, r1, r3))
for e in index.merge(r2, r1, r3):
print(e.name, hex(e.flags), e.ctime)
eiv = e.name in expect_invalid
er = e.name in expect_real
WVPASSEQ(eiv, not e.is_valid())
WVPASSEQ(er, e.is_real())
fake_validate(r2, r3)
dump(index.merge(r2, r1, r3))
WVPASSEQ([
e.name
for e in index.merge(r2, r1, r3) if not e.is_valid()
], [])
e = eget(index.merge(r2, r1, r3), b'/a/b/c')
e.invalidate()
e.repack()
dump(index.merge(r2, r1, r3))
WVPASSEQ([
e.name
for e in index.merge(r2, r1, r3) if not e.is_valid()
], [b'/a/b/c', b'/a/b/', b'/a/', b'/'])
finally:
os.chdir(orig_cwd)
