def sync_indexes(self):
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
debug1('%s\n' % f)
if f.endswith('.idx'):
extra.add(f)
needed = set()
conn.write('list-indexes\n')
for line in linereader(conn):
if not line:
break
assert(line.find('/') < 0)
parts = line.split(' ')
idx = parts[0]
if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# Any idx that the server has heard of is proven not extra
extra.discard(idx)
self.check_ok()
debug1('client: removing extra indexes: %s\n' % extra)
for idx in extra:
os.unlink(os.path.join(self.cachedir, idx))
debug1('client: server requested load of: %s\n' % needed)
for idx in needed:
self.sync_index(idx)
git.auto_midx(self.cachedir)
def _suggest_packs(self):
ob = self._busy
if ob:
assert(ob == 'receive-objects-v2')
self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
debug2('%s\n' % line)
if line.startswith('index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
% git.shorten_hash(idx))
suggested.append(idx)
else:
assert(line.endswith('.idx'))
debug1('client: completed writing pack, idx: %s\n'
% git.shorten_hash(line))
suggested.append(line)
self.check_ok()
if ob:
self._busy = None
idx = None
for idx in suggested:
self.sync_index(idx)
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
self.conn.write('%s\n' % ob)
return idx
def _suggest_packs(self):
ob = self._busy
if ob:
assert (ob == 'receive-objects-v2')
self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
debug2('%s\n' % line)
if line.startswith('index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n' %
git.shorten_hash(idx))
suggested.append(idx)
else:
assert (line.endswith('.idx'))
debug1('client: completed writing pack, idx: %s\n' %
git.shorten_hash(line))
suggested.append(line)
self.check_ok()
if ob:
self._busy = None
idx = None
for idx in suggested:
self.sync_index(idx)
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
self.conn.write('%s\n' % ob)
return idx
def sync_indexes(self):
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
debug1('%s\n' % f)
if f.endswith('.idx'):
extra.add(f)
needed = set()
conn.write('list-indexes\n')
for line in linereader(conn):
if not line:
break
assert (line.find('/') < 0)
parts = line.split(' ')
idx = parts[0]
if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# Any idx that the server has heard of is proven not extra
extra.discard(idx)
self.check_ok()
debug1('client: removing extra indexes: %s\n' % extra)
for idx in extra:
os.unlink(os.path.join(self.cachedir, idx))
debug1('client: server requested load of: %s\n' % needed)
for idx in needed:
self.sync_index(idx)
git.auto_midx(self.cachedir)
def sync_indexes(self):
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
debug1(path_msg(f) + '\n')
if f.endswith(b'.idx'):
extra.add(f)
needed = set()
for idx, load in self._list_indexes():
if load:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# Any idx that the server has heard of is proven not extra
extra.discard(idx)
debug1('client: removing extra indexes: %s\n' % extra)
for idx in extra:
os.unlink(os.path.join(self.cachedir, idx))
debug1('client: server requested load of: %s\n' % needed)
for idx in needed:
self.sync_index(idx)
git.auto_midx(self.cachedir)
def test_midx_close():
fddir = b'/proc/self/fd'
try:
os.listdir(fddir)
except Exception:
# not supported, not Linux, I guess
return
def openfiles():
for fd in os.listdir(fddir):
try:
yield os.readlink(os.path.join(fddir, fd))
except OSError:
pass
def force_midx(objdir):
args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
check_call(args)
with no_lingering_errors(), \
test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
# create a few dummy idxes
for i in range(10):
_create_idx(tmpdir, i)
git.auto_midx(tmpdir)
l = git.PackIdxList(tmpdir)
# this doesn't exist (yet)
WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
for i in range(10, 15):
_create_idx(tmpdir, i)
# delete the midx ...
# TODO: why do we need to? git.auto_midx() below doesn't?!
for fn in os.listdir(tmpdir):
if fn.endswith(b'.midx'):
os.unlink(os.path.join(tmpdir, fn))
# and make a new one
git.auto_midx(tmpdir)
# check it still doesn't exist - we haven't refreshed
WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
# check that we still have the midx open, this really
# just checks more for the kernel API ('deleted' string)
for fn in openfiles():
if not b'midx-' in fn:
continue
WVPASSEQ(True, b'deleted' in fn)
# refresh the PackIdxList
l.refresh()
# and check that an object in pack 10 exists now
WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
for fn in openfiles():
if not b'midx-' in fn:
continue
# check that we don't have it open anymore
WVPASSEQ(False, b'deleted' in fn)
def finish_writing(self, run_midx=True):
if self.meta_writer != self.data_writer and self.meta_writer is not None:
self._finish(self.meta_writer, self.meta_fakesha, meta=True)
self.meta_writer = None
if self.data_writer is not None:
self._finish(self.data_writer, self.data_fakesha)
if self.meta_writer == self.data_writer:
self.meta_writer = None
self.data_writer = None
if run_midx:
git.auto_midx(self.cachedir)
def _synchronize_idxes(self):
changes = False
local_idxes = set(fnmatch.filter(os.listdir(self.cachedir), b'*.idx'))
for remote_idx in self.storage.list(b'*.encidx'):
local_idx = remote_idx.replace(b'.encidx', b'.idx')
if local_idx in local_idxes:
local_idxes.remove(local_idx)
else:
ec = self._open_read(remote_idx, Kind.IDX)
with open(os.path.join(self.cachedir, local_idx), 'wb') as f:
f.write(ec.read()[1])
changes = True
for local_idx in local_idxes:
changes = True
os.unlink(os.path.join(self.cachedir, local_idx))
if changes:
git.auto_midx(self.cachedir)
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
self.conn.write('send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert(n)
fn = os.path.join(self.cachedir, name)
f = open(fn + '.tmp', 'w')
count = 0
progress('Receiving index from server: %d/%d\r' % (count, n))
for b in chunkyreader(self.conn, n):
f.write(b)
count += len(b)
progress('Receiving index from server: %d/%d\r' % (count, n))
progress('Receiving index from server: %d/%d, done.\n' % (count, n))
self.check_ok()
f.close()
os.rename(fn + '.tmp', fn)
git.auto_midx(self.cachedir)