def _packer(self): name = self.fp.get_name() prepack_name = name + '.prepack' pack_name = name + '.pack' packed = File(pack_name) if len(packed) > 0: # packed contains data left from an incomplete pack attempt. packed.seek(0) packed.truncate() self._write_header(packed) # find all reachable objects. Note that when we yield, new # commits may happen and pack_extra will contain new or modified # OIDs. index = {} def gen_reachable_records(): # we order the todo queue by file offset. The hope is that the # packed file will be mostly the same as the old file in order # to speed up the rsync delta process. default_rank = 2**64 pack_todo = [(0, durus.connection.ROOT_OID)] while pack_todo or self.pack_extra: if self.pack_extra: oid = self.pack_extra.pop() # note we don't check 'index' because it could be an # object that got updated since the pack began and in # that case we have to write the new record to the pack # file else: rank, oid = heapq.heappop(pack_todo) if oid in index: # we already wrote this object record continue record = self.load(oid) oid2, data, refdata = unpack_record(record) assert oid == oid2 # ensure we have records for objects referenced for ref_oid in split_oids(refdata): item = (self.index.get(ref_oid, default_rank), ref_oid) heapq.heappush(pack_todo, item) yield (oid, record) for z in self._write_transaction( packed, gen_reachable_records(), index): yield None # incremental pack, allow clients to be served self._write_index(packed, index) packed.flush() packed.fsync() if self.fp.is_temporary(): self.fp.close() else: self.fp.rename(prepack_name) packed.rename(name) self.fp = packed for oid in self.index: if oid not in index: self.invalid.add(oid) self.index = index self.pack_extra = None
def get_packer(self): if (self.pending_records or self.pack_extra is not None or self.shelf.get_file().is_temporary() or self.shelf.get_file().is_readonly()): return (x for x in []) # Don't pack. self.pack_extra = set() file_path = self.shelf.get_file().get_name() file = File(file_path + '.pack') file.truncate() # obtains lock and clears. assert file.tell() == 0 def packer(): yield "started %s" % datetime.now() seen = IntSet() items = self.gen_oid_record(start_oid=int8_to_str(0), seen=seen) for step in Shelf.generate_shelf(file, items): yield step file.flush() file.fsync() shelf = Shelf(file) yield "base written %s" % datetime.now() # Invalidate oids that have been removed. for hole in shelf.get_offset_map().gen_holes(): yield hole oid = int8_to_str(hole) if self.shelf.get_position(oid) is not None: assert shelf.get_position(oid) is None self.invalid.add(oid) yield "invalidations identified %s" % datetime.now() for oid in self.pack_extra: seen.discard(str_to_int8(oid)) for oid in self.pack_extra: shelf.store(self.gen_oid_record(start_oid=oid, seen=seen)) file.flush() file.fsync() if not self.shelf.get_file().is_temporary(): self.shelf.get_file().rename(file_path + '.prepack') self.shelf.get_file().close() shelf.get_file().rename(file_path) self.shelf = shelf self.pack_extra = None yield "finished %s" % datetime.now() return packer()
def get_packer(self): if (self.pending_records or self.pack_extra is not None or self.shelf.get_file().is_temporary() or self.shelf.get_file().is_readonly()): return (x for x in []) # Don't pack. self.pack_extra = set() file_path = self.shelf.get_file().get_name() file = File(file_path + '.pack') file.truncate() # obtains lock and clears. assert file.tell() == 0 def packer(): yield "started %s" % datetime.now() items = self.gen_oid_record(start_oid=int8_to_str(0)) for step in Shelf.generate_shelf(file, items): yield step file.flush() file.fsync() shelf = Shelf(file) yield "base written %s" % datetime.now() # Invalidate oids that have been removed. for hole in shelf.get_offset_map().gen_holes(): yield hole oid = int8_to_str(hole) if self.shelf.get_position(oid) is not None: assert shelf.get_position(oid) is None self.invalid.add(oid) yield "invalidations identified %s" % datetime.now() shelf.store( (name, self.shelf.get_value(name)) for name in self.pack_extra) file.flush() file.fsync() if not self.shelf.get_file().is_temporary(): self.shelf.get_file().rename(file_path + '.prepack') self.shelf.get_file().close() shelf.get_file().rename(file_path) self.shelf = shelf self.pack_extra = None yield "finished %s" % datetime.now() return packer()
def a(self): f = File() f.rename(f.get_name()) assert f.is_temporary() raises(AssertionError, f.rename, f.get_name() + '.renamed') test_name = f.get_name() + '.test' assert not exists(test_name) tmp = open(test_name, 'w+b') tmp.close() g = File(test_name) assert not g.is_temporary() g.rename(g.get_name() + '.renamed') assert g.get_name() == test_name + '.renamed' f.write(as_bytes('abc')) f.seek(0) assert len(f) == 3 assert as_bytes('a') == f.read(1) assert as_bytes('bc') == f.read() f.close() assert not exists(f.get_name()) raises(OSError, f.__len__) # tmpfile removed on close h = File(g.get_name()) g.write(as_bytes('a')) g.seek(0) assert g.tell() == 0 g.seek_end() assert g.tell() == 1 assert g.has_lock assert not h.has_lock raises(IOError, h.write, as_bytes('b')) g.flush() g.fsync() g.seek(0) g.truncate() g.close() h.close() unlink(g.get_name())