def _packer(self):
name = self.fp.get_name()
prepack_name = name + '.prepack'
pack_name = name + '.pack'
packed = File(pack_name)
if len(packed) > 0:
# packed contains data left from an incomplete pack attempt.
packed.seek(0)
packed.truncate()
self._write_header(packed)
# find all reachable objects. Note that when we yield, new
# commits may happen and pack_extra will contain new or modified
# OIDs.
index = {}
def gen_reachable_records():
# we order the todo queue by file offset. The hope is that the
# packed file will be mostly the same as the old file in order
# to speed up the rsync delta process.
default_rank = 2**64
pack_todo = [(0, durus.connection.ROOT_OID)]
while pack_todo or self.pack_extra:
if self.pack_extra:
oid = self.pack_extra.pop()
# note we don't check 'index' because it could be an
# object that got updated since the pack began and in
# that case we have to write the new record to the pack
# file
else:
rank, oid = heapq.heappop(pack_todo)
if oid in index:
# we already wrote this object record
continue
record = self.load(oid)
oid2, data, refdata = unpack_record(record)
assert oid == oid2
# ensure we have records for objects referenced
for ref_oid in split_oids(refdata):
item = (self.index.get(ref_oid, default_rank), ref_oid)
heapq.heappush(pack_todo, item)
yield (oid, record)
for z in self._write_transaction(
packed, gen_reachable_records(), index):
yield None # incremental pack, allow clients to be served
self._write_index(packed, index)
packed.flush()
packed.fsync()
if self.fp.is_temporary():
self.fp.close()
else:
self.fp.rename(prepack_name)
packed.rename(name)
self.fp = packed
for oid in self.index:
if oid not in index:
self.invalid.add(oid)
self.index = index
self.pack_extra = None
def get_packer(self):
if (self.pending_records or
self.pack_extra is not None or
self.shelf.get_file().is_temporary() or
self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
seen = IntSet()
items = self.gen_oid_record(start_oid=int8_to_str(0), seen=seen)
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
for oid in self.pack_extra:
seen.discard(str_to_int8(oid))
for oid in self.pack_extra:
shelf.store(self.gen_oid_record(start_oid=oid, seen=seen))
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
def get_packer(self):
if (self.pending_records or self.pack_extra is not None
or self.shelf.get_file().is_temporary()
or self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
items = self.gen_oid_record(start_oid=int8_to_str(0))
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
shelf.store(
(name, self.shelf.get_value(name)) for name in self.pack_extra)
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
