def test_atomically_replaced_file():
tmpdir = tempfile.mkdtemp(dir=bup_tmp, prefix='bup-thelper-')
target_file = os.path.join(tmpdir, 'test-atomic-write')
initial_failures = wvfailure_count()
with atomically_replaced_file(target_file, mode='w') as f:
f.write('asdf')
WVPASSEQ(f.mode, 'w')
f = open(target_file, 'r')
WVPASSEQ(f.read(), 'asdf')
try:
with atomically_replaced_file(target_file, mode='w') as f:
f.write('wxyz')
raise Exception()
except:
pass
with open(target_file) as f:
WVPASSEQ(f.read(), 'asdf')
with atomically_replaced_file(target_file, mode='wb') as f:
f.write(os.urandom(20))
WVPASSEQ(f.mode, 'wb')
if wvfailure_count() == initial_failures:
subprocess.call(['rm', '-rf', tmpdir])
def sync_index(self, name):
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
msg = ("won't request existing .idx, try `bup bloom --check %s`"
% path_msg(fn))
raise ClientError(msg)
with atomically_replaced_file(fn, 'wb') as f:
self.send_index(name, f, lambda size: None)
def test_atomically_replaced_file(tmpdir):
target_file = os.path.join(tmpdir, b'test-atomic-write')
with atomically_replaced_file(target_file, mode='w') as f:
f.write('asdf')
WVPASSEQ(f.mode, 'w')
f = open(target_file, 'r')
WVPASSEQ(f.read(), 'asdf')
try:
with atomically_replaced_file(target_file, mode='w') as f:
f.write('wxyz')
raise Exception()
except:
pass
with open(target_file) as f:
WVPASSEQ(f.read(), 'asdf')
with atomically_replaced_file(target_file, mode='wb') as f:
f.write(os.urandom(20))
WVPASSEQ(f.mode, 'wb')
def test_atomically_replaced_file():
with no_lingering_errors(), test_tempdir('bup-thelper-') as tmpdir:
target_file = os.path.join(tmpdir, 'test-atomic-write')
with atomically_replaced_file(target_file, mode='w') as f:
f.write('asdf')
WVPASSEQ(f.mode, 'w')
f = open(target_file, 'r')
WVPASSEQ(f.read(), 'asdf')
try:
with atomically_replaced_file(target_file, mode='w') as f:
f.write('wxyz')
raise Exception()
except:
pass
with open(target_file) as f:
WVPASSEQ(f.read(), 'asdf')
with atomically_replaced_file(target_file, mode='wb') as f:
f.write(os.urandom(20))
WVPASSEQ(f.mode, 'wb')
def test_atomically_replaced_file():
with no_lingering_errors(), test_tempdir('bup-thelper-') as tmpdir:
target_file = os.path.join(tmpdir, 'test-atomic-write')
with atomically_replaced_file(target_file, mode='w') as f:
f.write('asdf')
WVPASSEQ(f.mode, 'w')
f = open(target_file, 'r')
WVPASSEQ(f.read(), 'asdf')
try:
with atomically_replaced_file(target_file, mode='w') as f:
f.write('wxyz')
raise Exception()
except:
pass
with open(target_file) as f:
WVPASSEQ(f.read(), 'asdf')
with atomically_replaced_file(target_file, mode='wb') as f:
f.write(os.urandom(20))
WVPASSEQ(f.mode, 'wb')
def prepare_save(self):
""" Commit all of the relevant data to disk. Do as much work
as possible without actually making the changes visible."""
if self._pending_save:
raise Error('save of %r already in progress' % self._filename)
with self._cleanup:
if self._node_paths:
dir, name = os.path.split(self._filename)
self._pending_save = atomically_replaced_file(self._filename,
mode='wb',
buffering=65536)
with self._cleanup.enter_context(self._pending_save) as f:
pickle.dump(self._node_paths, f, 2)
else: # No data
self._cleanup.callback(lambda: unlink(self._filename))
self._cleanup = self._cleanup.pop_all()
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
msg = "won't request existing .idx, try `bup bloom --check %s`" % fn
raise ClientError(msg)
self.conn.write('send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert(n)
with atomically_replaced_file(fn, 'w') as f:
count = 0
for b in chunkyreader(self.conn, n):
f.write(b)
count += len(b)
qprogress('Receiving index from server: %d/%d\r' % (count, n))
self.check_ok()
def __init__(self, filename, metastore, tmax):
self.closed = False
self.rootlevel = self.level = Level([], None)
self.pending_index = None
self.f = None
self.count = 0
self.lastfile = None
self.filename = None
self.filename = filename = resolve_parent(filename)
self.metastore = metastore
self.tmax = tmax
(dir, name) = os.path.split(filename)
with ExitStack() as self.cleanup:
self.pending_index = atomically_replaced_file(self.filename,
mode='wb',
buffering=65536)
self.f = self.cleanup.enter_context(self.pending_index)
self.cleanup.enter_context(self.f)
self.f.write(INDEX_HDR)
self.cleanup = self.cleanup.pop_all()
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
msg = "won't request existing .idx, try `bup bloom --check %s`" % fn
raise ClientError(msg)
self.conn.write('send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert (n)
with atomically_replaced_file(fn, 'w') as f:
count = 0
progress('Receiving index from server: %d/%d\r' % (count, n))
for b in chunkyreader(self.conn, n):
f.write(b)
count += len(b)
qprogress('Receiving index from server: %d/%d\r' % (count, n))
progress('Receiving index from server: %d/%d, done.\n' %
(count, n))
self.check_ok()
def _do_midx(outdir, outfilename, infilenames, prefixstr,
auto=False, force=False):
global _first
if not outfilename:
assert(outdir)
sum = hexlify(Sha1(b'\0'.join(infilenames)).digest())
outfilename = b'%s/midx-%s.midx' % (outdir, sum)
inp = []
total = 0
allfilenames = []
midxs = []
try:
for name in infilenames:
ix = git.open_idx(name)
midxs.append(ix)
inp.append((
ix.map,
len(ix),
ix.sha_ofs,
isinstance(ix, midx.PackMidx) and ix.which_ofs or 0,
len(allfilenames),
))
for n in ix.idxnames:
allfilenames.append(os.path.basename(n))
total += len(ix)
inp.sort(reverse=True, key=lambda x: x[0][x[2] : x[2] + 20])
if not _first: _first = outdir
dirprefix = (_first != outdir) and git.repo_rel(outdir) + b': ' or b''
debug1('midx: %s%screating from %d files (%d objects).\n'
% (dirprefix, prefixstr, len(infilenames), total))
if (auto and (total < 1024 and len(infilenames) < 3)) \
or ((auto or force) and len(infilenames) < 2) \
or (force and not total):
debug1('midx: nothing to do.\n')
return
pages = int(total/SHA_PER_PAGE) or 1
bits = int(math.ceil(math.log(pages, 2)))
entries = 2**bits
debug1('midx: table size: %d (%d bits)\n' % (entries*4, bits))
unlink(outfilename)
with atomically_replaced_file(outfilename, 'wb') as f:
f.write(b'MIDX')
f.write(struct.pack('!II', midx.MIDX_VERSION, bits))
assert(f.tell() == 12)
f.truncate(12 + 4*entries + 20*total + 4*total)
f.flush()
fdatasync(f.fileno())
fmap = mmap_readwrite(f, close=False)
count = merge_into(fmap, bits, total, inp)
del fmap # Assume this calls msync() now.
f.seek(0, os.SEEK_END)
f.write(b'\0'.join(allfilenames))
finally:
for ix in midxs:
if isinstance(ix, midx.PackMidx):
ix.close()
midxs = None
inp = None
# This is just for testing (if you enable this, don't clear inp above)
if 0:
p = midx.PackMidx(outfilename)
assert(len(p.idxnames) == len(infilenames))
log(repr(p.idxnames) + '\n')
assert(len(p) == total)
for pe, e in p, git.idxmerge(inp, final_progress=False):
pin = next(pi)
assert(i == pin)
assert(p.exists(i))
return total, outfilename
def _do_midx(outdir, outfilename, infilenames, prefixstr):
global _first
if not outfilename:
assert(outdir)
sum = Sha1('\0'.join(infilenames)).hexdigest()
outfilename = '%s/midx-%s.midx' % (outdir, sum)
inp = []
total = 0
allfilenames = []
midxs = []
try:
for name in infilenames:
ix = git.open_idx(name)
midxs.append(ix)
inp.append((
ix.map,
len(ix),
ix.sha_ofs,
isinstance(ix, midx.PackMidx) and ix.which_ofs or 0,
len(allfilenames),
))
for n in ix.idxnames:
allfilenames.append(os.path.basename(n))
total += len(ix)
inp.sort(lambda x,y: cmp(str(y[0][y[2]:y[2]+20]),str(x[0][x[2]:x[2]+20])))
if not _first: _first = outdir
dirprefix = (_first != outdir) and git.repo_rel(outdir)+': ' or ''
debug1('midx: %s%screating from %d files (%d objects).\n'
% (dirprefix, prefixstr, len(infilenames), total))
if (opt.auto and (total < 1024 and len(infilenames) < 3)) \
or ((opt.auto or opt.force) and len(infilenames) < 2) \
or (opt.force and not total):
debug1('midx: nothing to do.\n')
return
pages = int(total/SHA_PER_PAGE) or 1
bits = int(math.ceil(math.log(pages, 2)))
entries = 2**bits
debug1('midx: table size: %d (%d bits)\n' % (entries*4, bits))
unlink(outfilename)
with atomically_replaced_file(outfilename, 'wb') as f:
f.write('MIDX')
f.write(struct.pack('!II', midx.MIDX_VERSION, bits))
assert(f.tell() == 12)
f.truncate(12 + 4*entries + 20*total + 4*total)
f.flush()
fdatasync(f.fileno())
fmap = mmap_readwrite(f, close=False)
count = merge_into(fmap, bits, total, inp)
del fmap # Assume this calls msync() now.
f.seek(0, os.SEEK_END)
f.write('\0'.join(allfilenames))
finally:
for ix in midxs:
if isinstance(ix, midx.PackMidx):
ix.close()
midxs = None
inp = None
# This is just for testing (if you enable this, don't clear inp above)
if 0:
p = midx.PackMidx(outfilename)
assert(len(p.idxnames) == len(infilenames))
print p.idxnames
assert(len(p) == total)
for pe, e in p, git.idxmerge(inp, final_progress=False):
pin = pi.next()
assert(i == pin)
assert(p.exists(i))
return total, outfilename