def check_midx(name):
nicename = git.repo_rel(name)
log('Checking %s.\n' % path_msg(nicename))
try:
ix = git.open_idx(name)
except git.GitError as e:
add_error('%s: %s' % (path_msg(name), e))
return
for count,subname in enumerate(ix.idxnames):
sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
for ecount,e in enumerate(sub):
if not (ecount % 1234):
qprogress(' %d/%d: %s %d/%d\r'
% (count, len(ix.idxnames),
git.shorten_hash(subname).decode('ascii'),
ecount, len(sub)))
if not sub.exists(e):
add_error("%s: %s: %s missing from idx"
% (path_msg(nicename),
git.shorten_hash(subname).decode('ascii'),
hexstr(e)))
if not ix.exists(e):
add_error("%s: %s: %s missing from midx"
% (path_msg(nicename),
git.shorten_hash(subname).decode('ascii'),
hexstr(e)))
prev = None
for ecount,e in enumerate(ix):
if not (ecount % 1234):
qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
if e and prev and not e >= prev:
add_error('%s: ordering error: %s < %s'
% (nicename, hexstr(e), hexstr(prev)))
prev = e
def _suggest_packs(self):
ob = self._busy
if ob:
assert(ob == 'receive-objects-v2')
self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
debug2('%s\n' % line)
if line.startswith('index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
% git.shorten_hash(idx))
suggested.append(idx)
else:
assert(line.endswith('.idx'))
debug1('client: completed writing pack, idx: %s\n'
% git.shorten_hash(line))
suggested.append(line)
self.check_ok()
if ob:
self._busy = None
idx = None
for idx in suggested:
self.sync_index(idx)
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
self.conn.write('%s\n' % ob)
return idx
def _suggest_packs(self):
ob = self._busy
if ob:
assert(ob == b'receive-objects-v2')
self.conn.write(b'\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
debug2('%r\n' % line)
if line.startswith(b'index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
% git.shorten_hash(idx).decode('ascii'))
suggested.append(idx)
else:
assert(line.endswith(b'.idx'))
debug1('client: completed writing pack, idx: %s\n'
% git.shorten_hash(line).decode('ascii'))
suggested.append(line)
self.check_ok()
if ob:
self._busy = None
idx = None
for idx in suggested:
self.sync_index(idx)
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
self.conn.write(b'%s\n' % ob)
return idx
def check_midx(name):
nicename = git.repo_rel(name)
log('Checking %s.\n' % nicename)
try:
ix = git.open_idx(name)
except git.GitError as e:
add_error('%s: %s' % (name, e))
return
for count,subname in enumerate(ix.idxnames):
sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
for ecount,e in enumerate(sub):
if not (ecount % 1234):
qprogress(' %d/%d: %s %d/%d\r'
% (count, len(ix.idxnames),
git.shorten_hash(subname), ecount, len(sub)))
if not sub.exists(e):
add_error("%s: %s: %s missing from idx"
% (nicename, git.shorten_hash(subname),
str(e).encode('hex')))
if not ix.exists(e):
add_error("%s: %s: %s missing from midx"
% (nicename, git.shorten_hash(subname),
str(e).encode('hex')))
prev = None
for ecount,e in enumerate(ix):
if not (ecount % 1234):
qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
if not e >= prev:
add_error('%s: ordering error: %s < %s'
% (nicename,
str(e).encode('hex'), str(prev).encode('hex')))
prev = e
def receive_objects_v2(conn, junk):
global suspended_w
_init_session()
suggested = set()
if suspended_w:
w = suspended_w
suspended_w = None
else:
if dumb_server_mode:
w = git.PackWriter(objcache_maker=None)
else:
w = git.PackWriter()
while 1:
ns = conn.read(4)
if not ns:
w.abort()
raise Exception('object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
conn.write('%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
suspended_w = w
conn.ok()
return
shar = conn.read(20)
crcr = struct.unpack('!I', conn.read(4))[0]
n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
_check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
if not dumb_server_mode:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
assert(oldpack.endswith('.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
% git.shorten_hash(name))
debug1("bup server: because of object %s\n"
% shar.encode('hex'))
conn.write('index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
_check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def receive_objects_v2(conn, junk):
global suspended_w
git.check_repo_or_die()
suggested = set()
if suspended_w:
w = suspended_w
suspended_w = None
else:
if dumb_server_mode:
w = git.PackWriter(objcache_maker=None)
else:
w = git.PackWriter()
while 1:
ns = conn.read(4)
if not ns:
w.abort()
raise Exception('object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
conn.write('%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
suspended_w = w
conn.ok()
return
shar = conn.read(20)
crcr = struct.unpack('!I', conn.read(4))[0]
n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
_check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
if not dumb_server_mode:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
assert(oldpack.endswith('.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
% git.shorten_hash(name))
debug1("bup server: because of object %s\n"
% shar.encode('hex'))
conn.write('index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
_check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
def check_midx(name):
nicename = git.repo_rel(name)
log('Checking %s.\n' % nicename)
try:
ix = git.open_idx(name)
except git.GitError, e:
add_error('%s: %s' % (name, e))
return
for count, subname in enumerate(ix.idxnames):
sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
for ecount, e in enumerate(sub):
if not (ecount % 1234):
qprogress(' %d/%d: %s %d/%d\r' % (count, len(
ix.idxnames), git.shorten_hash(subname), ecount, len(sub)))
if not sub.exists(e):
add_error("%s: %s: %s missing from idx" %
(nicename, git.shorten_hash(subname),
str(e).encode('hex')))
if not ix.exists(e):
add_error("%s: %s: %s missing from midx" %
(nicename, git.shorten_hash(subname),
str(e).encode('hex')))
prev = None
for ecount, e in enumerate(ix):
if not (ecount % 1234):
qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
if not e >= prev:
add_error(
'%s: ordering error: %s < %s' %
def check_midx(name):
nicename = git.repo_rel(name)
log('Checking %s.\n' % nicename)
try:
ix = git.open_idx(name)
except git.GitError, e:
add_error('%s: %s' % (name, e))
return
for count,subname in enumerate(ix.idxnames):
sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
for ecount,e in enumerate(sub):
if not (ecount % 1234):
qprogress(' %d/%d: %s %d/%d\r'
% (count, len(ix.idxnames),
git.shorten_hash(subname), ecount, len(sub)))
if not sub.exists(e):
add_error("%s: %s: %s missing from idx"
% (nicename, git.shorten_hash(subname),
str(e).encode('hex')))
if not ix.exists(e):
add_error("%s: %s: %s missing from midx"
% (nicename, git.shorten_hash(subname),
str(e).encode('hex')))
prev = None
for ecount,e in enumerate(ix):
if not (ecount % 1234):
qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
if not e >= prev:
add_error('%s: ordering error: %s < %s'
% (nicename,
def check_midx(name):
nicename = git.repo_rel(name)
log("Checking %s.\n" % nicename)
try:
ix = git.open_idx(name)
except git.GitError, e:
add_error("%s: %s" % (name, e))
return
for count, subname in enumerate(ix.idxnames):
sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
for ecount, e in enumerate(sub):
if not (ecount % 1234):
qprogress(
" %d/%d: %s %d/%d\r" % (count, len(ix.idxnames), git.shorten_hash(subname), ecount, len(sub))
)
if not sub.exists(e):
add_error("%s: %s: %s missing from idx" % (nicename, git.shorten_hash(subname), str(e).encode("hex")))
if not ix.exists(e):
add_error("%s: %s: %s missing from midx" % (nicename, git.shorten_hash(subname), str(e).encode("hex")))
prev = None
for ecount, e in enumerate(ix):
if not (ecount % 1234):
qprogress(" Ordering: %d/%d\r" % (ecount, len(ix)))
if not e >= prev:
add_error("%s: ordering error: %s < %s" % (nicename, str(e).encode("hex"), str(prev).encode("hex")))
prev = e
_first = None
def receive_objects_v2(self, junk):
self.init_session()
suggested = set()
if self.suspended:
self.suspended = False
else:
if self.dumb_server_mode:
objcache_maker = lambda: None
else:
objcache_maker = None
# FIXME: this goes together with the direct accesses below
self.repo._ensure_packwriter()
while 1:
ns = self.conn.read(4)
if not ns:
self.repo.abort_writing()
raise Exception(
'object read: expected length header, got EOF\n')
n = struct.unpack('!I', ns)[0]
#debug2('expecting %d bytes\n' % n)
if not n:
# FIXME: don't be lazy and count ourselves, or something, at least
# don't access self.repo internals
debug1('bup server: received %d object%s.\n' %
(self.repo._packwriter.count,
self.repo._packwriter.count != 1 and "s" or ''))
fullpath = self.repo.finish_writing(
run_midx=not self.dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
self.conn.write(b'%s.idx\n' % name)
self.conn.ok()
return
elif n == 0xffffffff:
debug2('bup server: receive-objects suspended.\n')
self.suspended = True
self.conn.ok()
return
shar = self.conn.read(20)
crcr = struct.unpack('!I', self.conn.read(4))[0]
n -= 20 + 4
buf = self.conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
self._check(n, len(buf),
'object read: expected %d bytes, got %d\n')
if not self.dumb_server_mode:
result = self.repo.exists(shar, want_source=True)
if result:
oldpack = result.pack
assert (oldpack.endswith(b'.idx'))
(dir, name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n" %
git.shorten_hash(name).decode('ascii'))
debug1("bup server: because of object %s\n" %
hexstr(shar))
self.conn.write(b'index %s\n' % name)
suggested.add(name)
continue
# FIXME: figure out the right abstraction for this; or better yet,
# make the protocol aware of the object type
nw, crc = self.repo._packwriter._raw_write((buf, ), sha=shar)
self._check(crcr, crc, 'object read: expected crc %d, got %d\n')
def check_midx(name):
nicename = git.repo_rel(name)
log('Checking %s.\n' % nicename)
try:
ix = git.open_idx(name)
except git.GitError, e:
add_error('%s: %s' % (name, e))
return
for count,subname in enumerate(ix.idxnames):
sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
for ecount,e in enumerate(sub):
if not (ecount % 1234):
qprogress(' %d/%d: %s %d/%d\r'
% (count, len(ix.idxnames),
git.shorten_hash(subname), ecount, len(sub)))
if not sub.exists(e):
add_error("%s: %s: %s missing from idx"
% (nicename, git.shorten_hash(subname),
str(e).encode('hex')))
if not ix.exists(e):
add_error("%s: %s: %s missing from midx"
% (nicename, git.shorten_hash(subname),
str(e).encode('hex')))
prev = None
for ecount,e in enumerate(ix):
if not (ecount % 1234):
qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
if not e >= prev:
add_error('%s: ordering error: %s < %s'
% (nicename,