コード例 #1
0
    def __init__(self, file=None, readonly=False, repair=False):
        """(file:str=None, readonly:bool=False, repair:bool=False)
        If file is empty (or None), a temporary file will be used.
        """
        self.oid = -1
        if file is None:
            file = File()
            assert not readonly
            assert not repair
        elif not hasattr(file, 'seek'):
            file = File(file, readonly=readonly)
        if not readonly:
            file.obtain_lock()
        self.fp = file
        self.pending_records = {}
        self.pack_extra = None
        self.fp.seek(0, 2)
        if self.fp.tell() != 0:
            assert self.has_format(self.fp)
        else:
            # Write header for new file.
            self.fp.seek(len(self.MAGIC))
            self._write_header(self.fp)
            self._write_index(self.fp, {})

        self.index = {}
        self._build_index(repair)
        max_oid = -1
        for oid in self.index:
            max_oid = max(max_oid, str_to_int8(oid))
        self.oid = max_oid
        self.invalid = set()
コード例 #2
0
ファイル: test_shelf.py プロジェクト: ctismer/durus
 def test_c(self):
     f = File()
     s = Shelf(f)
     f.seek(0)
     p = Shelf(f)
     f.seek(0)
     q = Shelf(f)
コード例 #3
0
 def __init__(self, file=None, items=None, repair=False, readonly=False):
     """(File:str:None, [(str:str)], boolean)
     """
     if file is None:
         file = File()
         assert not readonly
         assert not repair
     elif not hasattr(file, 'seek'):
         file = File(file, readonly=readonly)
     if not readonly:
         file.obtain_lock()
     file.seek(0, 2)  # seek end
     if file.tell() == 0:
         # The file is empty.
         for result in self.generate_shelf(file=file, items=items or []):
             pass
     else:
         assert items is None
     # The file is not empty.
     assert self.has_format(file)
     self.file = file
     self.file.seek(len(self.prefix))
     n = read_int8(self.file)  # bytes in first transaction
     self.file.seek(self.file.tell() + n)
     self.offset_map = OffsetMap(self.file)
     # Initialize the memory index.
     self.memory_index = {}
     while True:
         transaction_offsets = read_transaction_offsets(self.file,
                                                        repair=repair)
         if transaction_offsets is None:
             break
         self.memory_index.update(transaction_offsets)
     self.file.seek_end()
     self.unused_name_generator = None
コード例 #4
0
 def b(self):
     name = mktemp()
     raises(OSError, File, name, readonly=True)
     g = File(name, readonly=False)
     g.close()
     f = File(name, readonly=True)
     assert f.is_readonly()
     raises(AssertionError, f.write, 'ok')
     raises(IOError, f.file.write, 'ok')  # readonly file
     f.close()
     unlink(name)
コード例 #5
0
ファイル: utest_file.py プロジェクト: Schevo/durus
 def c(self):
     name = mktemp()
     name2 = mktemp()
     f = File(name)
     assert f.tell() == 0
     g = File(name2)
     g.close()
     f.rename(name2)
     assert exists(name2)
     assert not exists(name)
     f.close()
     unlink(name2)
コード例 #6
0
ファイル: test_shelf.py プロジェクト: ctismer/durus
 def test_a(self):
     f = File()
     s = Shelf(f)
     name1 = s.next_name()
     name2 = s.next_name()
     assert name1 != name2
     r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
     assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
     assert s.get_value(name2) == name2 + name2
     f.seek(0)
     other = Shelf(f)
     names = sorted(other.__iter__())
     index = sorted(other.iterindex())
     items = sorted(other.items())
     assert names == [name1, name2], (name1, name2, names)
     assert items == [(n, n+n) for n in names]
     assert index == [(n, other.get_position(n)) for n in names]
コード例 #7
0
ファイル: utest_shelf.py プロジェクト: cfobel/durus
 def a(self):
     f = File()
     s = Shelf(f)
     name1 = s.next_name()
     name2 = s.next_name()
     assert name1 != name2
     r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
     assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
     assert s.get_value(name2) == name2 + name2
     f.seek(0)
     other = Shelf(f)
     names = sorted(other.__iter__())
     index = sorted(other.iterindex())
     items = sorted(other.items())
     assert names == [name1, name2], (name1, name2, names)
     assert items == [(n, n+n) for n in names]
     assert index == [(n, other.get_position(n)) for n in names]
コード例 #8
0
ファイル: shelf.py プロジェクト: Schevo/durus
 def __init__(self, file=None, items=None, repair=False, readonly=False):
     """(File:str:None, [(str:str)], boolean)
     """
     if file is None:
         file = File()
         assert not readonly
         assert not repair
     elif not hasattr(file, 'seek'):
         file = File(file, readonly=readonly)
     if not readonly:
         file.obtain_lock()
     file.seek(0, 2) # seek end
     if file.tell() == 0:
         # The file is empty.
         for result in self.generate_shelf(file=file, items=items or []):
             pass
     else:
         assert items is None
     # The file is not empty.
     assert self.has_format(file)
     self.file = file
     self.file.seek(len(self.prefix))
     n = read_int8(self.file) # bytes in first transaction
     self.file.seek(self.file.tell() + n)
     self.offset_map = OffsetMap(self.file)
     # Initialize the memory index.
     self.memory_index = {}
     while True:
         transaction_offsets = read_transaction_offsets(
             self.file, repair=repair)
         if transaction_offsets is None:
             break
         self.memory_index.update(transaction_offsets)
     self.file.seek_end()
     self.unused_name_generator = None
コード例 #9
0
ファイル: file_storage.py プロジェクト: ctismer/durus
 def get_packer(self):
     if (self.pending_records or
         self.pack_extra is not None or
         self.shelf.get_file().is_temporary() or
         self.shelf.get_file().is_readonly()):
         return (x for x in []) # Don't pack.
     self.pack_extra = set()
     file_path = self.shelf.get_file().get_name()
     file = File(file_path + '.pack')
     file.truncate() # obtains lock and clears.
     assert file.tell() == 0
     def packer():
         yield "started %s" % datetime.now()
         seen = IntSet()
         items = self.gen_oid_record(start_oid=int8_to_str(0), seen=seen)
         for step in Shelf.generate_shelf(file, items):
             yield step
         file.flush()
         file.fsync()
         shelf = Shelf(file)
         yield "base written %s" % datetime.now()
         # Invalidate oids that have been removed.
         for hole in shelf.get_offset_map().gen_holes():
             yield hole
             oid = int8_to_str(hole)
             if self.shelf.get_position(oid) is not None:
                 assert shelf.get_position(oid) is None
                 self.invalid.add(oid)
         yield "invalidations identified %s" % datetime.now()
         for oid in self.pack_extra:
             seen.discard(str_to_int8(oid))
         for oid in self.pack_extra:
             shelf.store(self.gen_oid_record(start_oid=oid, seen=seen))
         file.flush()
         file.fsync()
         if not self.shelf.get_file().is_temporary():
             self.shelf.get_file().rename(file_path + '.prepack')
             self.shelf.get_file().close()
         shelf.get_file().rename(file_path)
         self.shelf = shelf
         self.pack_extra = None
         yield "finished %s" % datetime.now()
     return packer()
コード例 #10
0
ファイル: utest_file_storage.py プロジェクト: cfobel/durus
 def b(self):
     f = File(prefix='shelftest')
     name = f.get_name()
     f.close()
     s = FileStorage(name)
     c = Connection(s)
     r = c.get_root()
     for x in range(10):
         r["a%s" % x] = Persistent()
         c.commit()
     deleted_oid = r['a9']._p_oid
     del r['a9']
     c.commit()
     c.pack()
     c.abort()
     assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 10
     new_oid = s.new_oid()
     assert new_oid == deleted_oid
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(11)
コード例 #11
0
ファイル: utest_file_storage.py プロジェクト: nascheme/durus
 def b(self):
     f = File(prefix='shelftest')
     name = f.get_name()
     f.close()
     s = FileStorage(name)
     c = Connection(s)
     r = c.get_root()
     for x in range(10):
         r["a%s" % x] = Persistent()
         c.commit()
     deleted_oid = r['a9']._p_oid
     del r['a9']
     c.commit()
     c.pack()
     c.abort()
     assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 10
     new_oid = s.new_oid()
     assert new_oid == deleted_oid
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(11)
コード例 #12
0
    def get_packer(self):
        if (self.pending_records or self.pack_extra is not None
                or self.shelf.get_file().is_temporary()
                or self.shelf.get_file().is_readonly()):
            return (x for x in [])  # Don't pack.
        self.pack_extra = set()
        file_path = self.shelf.get_file().get_name()
        file = File(file_path + '.pack')
        file.truncate()  # obtains lock and clears.
        assert file.tell() == 0

        def packer():
            yield "started %s" % datetime.now()
            items = self.gen_oid_record(start_oid=int8_to_str(0))
            for step in Shelf.generate_shelf(file, items):
                yield step
            file.flush()
            file.fsync()
            shelf = Shelf(file)
            yield "base written %s" % datetime.now()
            # Invalidate oids that have been removed.
            for hole in shelf.get_offset_map().gen_holes():
                yield hole
                oid = int8_to_str(hole)
                if self.shelf.get_position(oid) is not None:
                    assert shelf.get_position(oid) is None
                    self.invalid.add(oid)
            yield "invalidations identified %s" % datetime.now()
            shelf.store(
                (name, self.shelf.get_value(name)) for name in self.pack_extra)
            file.flush()
            file.fsync()
            if not self.shelf.get_file().is_temporary():
                self.shelf.get_file().rename(file_path + '.prepack')
                self.shelf.get_file().close()
            shelf.get_file().rename(file_path)
            self.shelf = shelf
            self.pack_extra = None
            yield "finished %s" % datetime.now()

        return packer()
コード例 #13
0
ファイル: utest_shelf.py プロジェクト: cfobel/durus
 def c(self):
     f = File()
     s = Shelf(f)
     f.seek(0)
     p = Shelf(f)
     f.seek(0)
     q = Shelf(f)
コード例 #14
0
ファイル: utest_file.py プロジェクト: Schevo/durus
 def b(self):
     name = mktemp()
     raises(OSError, File, name, readonly=True)
     g = File(name, readonly=False)
     g.close()
     f = File(name, readonly=True)
     assert f.is_readonly()
     raises(AssertionError, f.write, 'ok')
     raises(IOError, f.file.write, 'ok') # readonly file
     f.close()
     unlink(name)
コード例 #15
0
ファイル: utest_file_storage.py プロジェクト: cfobel/durus
 def a(self):
     f = File(prefix='shelftest')
     name = f.get_name()
     f.close()
     s = FileStorage(name)
     c = Connection(s)
     r = c.get_root()
     for x in range(10):
         r["a%s" % x] = Persistent()
         c.commit()
     deleted_oids = [
         r['a0']._p_oid, r['a2']._p_oid, r['a7']._p_oid, r['a8']._p_oid
     ]
     del r['a0']
     del r['a2']
     del r['a7']
     del r['a8']
     c.commit()
     c.pack()
     c.abort()
     assert c.get(deleted_oids[0])._p_is_ghost()
     assert c.get(deleted_oids[1])._p_is_ghost()
     raises(KeyError, getattr, c.get(deleted_oids[0]), 'a')
     assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 7
     c.commit()
     c.pack()
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-1], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-2], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-3], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-4], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(11), repr(new_oid)
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(12), repr(new_oid)
コード例 #16
0
ファイル: utest_file_storage.py プロジェクト: cfobel/durus
 def c(self):
     f = File(prefix='shelftest')
     name = f.get_name()
     f.close()
     s = FileStorage(name)
     c = Connection(s)
     r = c.get_root()
     for x in range(10):
         r["a%s" % x] = Persistent()
         c.commit()
     deleted_oid = r['a9']._p_oid
     del r['a9']
     c.commit()
     c.pack()
     c.abort()
     r.clear()
     c.commit()
     c.pack()
     c.abort()
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(1), repr(new_oid)
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(2), repr(new_oid)
コード例 #17
0
ファイル: utest_file_storage.py プロジェクト: nascheme/durus
 def c(self):
     f = File(prefix='shelftest')
     name = f.get_name()
     f.close()
     s = FileStorage(name)
     c = Connection(s)
     r = c.get_root()
     for x in range(10):
         r["a%s" % x] = Persistent()
         c.commit()
     deleted_oid = r['a9']._p_oid
     del r['a9']
     c.commit()
     c.pack()
     c.abort()
     r.clear()
     c.commit()
     c.pack()
     c.abort()
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(1), repr(new_oid)
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(2), repr(new_oid)
コード例 #18
0
ファイル: utest_file_storage.py プロジェクト: nascheme/durus
 def a(self):
     f = File(prefix='shelftest')
     name = f.get_name()
     f.close()
     s = FileStorage(name)
     c = Connection(s)
     r = c.get_root()
     for x in range(10):
         r["a%s" % x] = Persistent()
         c.commit()
     deleted_oids = [
         r['a0']._p_oid, r['a2']._p_oid, r['a7']._p_oid, r['a8']._p_oid]
     del r['a0']
     del r['a2']
     del r['a7']
     del r['a8']
     c.commit()
     c.pack()
     c.abort()
     assert c.get(deleted_oids[0])._p_is_ghost()
     assert c.get(deleted_oids[1])._p_is_ghost()
     raises(ReadConflictError, getattr, c.get(deleted_oids[0]), 'a')
     assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 7
     c.commit()
     c.pack()
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-1], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-2], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-3], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == deleted_oids[-4], (new_oid, deleted_oids)
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(11), repr(new_oid)
     new_oid = s.new_oid()
     assert new_oid == int8_to_str(12), repr(new_oid)
コード例 #19
0
ファイル: utest_utils.py プロジェクト: cfobel/durus
 def d(self):
     file = File()
     word_array = WordArray(file=file, number_of_words=1, bytes_per_word=8)
     file.seek(0)
     word_array2 = WordArray(file=file, number_of_words=1, bytes_per_word=8)
コード例 #20
0
 def c(self):
     name = mktemp()
     name2 = mktemp()
     f = File(name)
     assert f.tell() == 0
     g = File(name2)
     g.close()
     f.rename(name2)
     assert exists(name2)
     assert not exists(name)
     f.close()
     unlink(name2)
コード例 #21
0
 def _packer(self):
     name = self.fp.get_name()
     prepack_name = name + '.prepack'
     pack_name = name + '.pack'
     packed = File(pack_name)
     if len(packed) > 0:
         # packed contains data left from an incomplete pack attempt.
         packed.seek(0)
         packed.truncate()
     self._write_header(packed)
     # find all reachable objects.  Note that when we yield, new
     # commits may happen and pack_extra will contain new or modified
     # OIDs.
     index = {}
     def gen_reachable_records():
         pack_todo = [durus.connection.ROOT_OID]
         while pack_todo or self.pack_extra:
             if self.pack_extra:
                 oid = self.pack_extra.pop()
                 # note we don't check 'index' because it could be an
                 # object that got updated since the pack began and in
                 # that case we have to write the new record to the pack
                 # file
             else:
                 oid = heapq.heappop(pack_todo)
                 if oid in index:
                     # we already wrote this object record
                     continue
             record = self.load(oid)
             oid2, data, refdata = unpack_record(record)
             assert oid == oid2
             # ensure we have records for objects referenced
             for ref_oid in split_oids(refdata):
                 heapq.heappush(pack_todo, ref_oid)
             yield (oid, record)
     for z in self._write_transaction(
         packed, gen_reachable_records(), index):
         yield None # incremental pack, allow clients to be served
     self._write_index(packed, index)
     packed.flush()
     packed.fsync()
     if self.fp.is_temporary():
         self.fp.close()
     else:
         self.fp.rename(prepack_name)
     packed.rename(name)
     self.fp = packed
     for oid in self.index:
         if oid not in index:
             self.invalid.add(oid)
     self.index = index
     self.pack_extra = None
コード例 #22
0
def random_filename():
    f = File(prefix='schevodurus')
    n = f.get_name()
    f.close()
    return n
コード例 #23
0
ファイル: utest_shelf.py プロジェクト: cfobel/durus
 def e(self):
     f = File()
     n1 = int8_to_str(0)
     n2 = int8_to_str(1)
     s = Shelf(f, items=[(n1, 'record1'), (n2, 'record2')])
コード例 #24
0
ファイル: utest_utils.py プロジェクト: cfobel/durus
 def b(self):
     file = File()
     int_array = IntArray(file=file, number_of_ints=10, maximum_int=10)
     file.seek(0)
     int_array2 = IntArray(file=file)
     assert len(int_array2) == 10
コード例 #25
0
ファイル: utest_file.py プロジェクト: Schevo/durus
 def a(self):
     f = File()
     f.rename(f.get_name())
     assert f.is_temporary()
     raises(AssertionError, f.rename, f.get_name() + '.renamed')
     test_name = f.get_name() + '.test'
     assert not exists(test_name)
     tmp = open(test_name, 'w+b')
     tmp.close()
     g = File(test_name)
     assert not g.is_temporary()
     g.rename(g.get_name() + '.renamed')
     assert g.get_name() == test_name + '.renamed'
     f.write(as_bytes('abc'))
     f.seek(0)
     assert len(f) == 3
     assert as_bytes('a') == f.read(1)
     assert as_bytes('bc') == f.read()
     f.close()
     assert not exists(f.get_name())
     raises(OSError, f.__len__) # tmpfile removed on close
     h = File(g.get_name())
     g.write(as_bytes('a'))
     g.seek(0)
     assert g.tell() == 0
     g.seek_end()
     assert g.tell() == 1
     assert g.has_lock
     assert not h.has_lock
     raises(IOError, h.write, as_bytes('b'))
     g.flush()
     g.fsync()
     g.seek(0)
     g.truncate()
     g.close()
     h.close()
     unlink(g.get_name())
コード例 #26
0
ファイル: utest_utils.py プロジェクト: cfobel/durus
 def c(self):
     file = File()
     int_array = IntArray(file=file, number_of_ints=0, maximum_int=0)
     file.seek(0)
     int_array2 = IntArray(file=file)
コード例 #27
0
 def a(self):
     f = File()
     f.rename(f.get_name())
     assert f.is_temporary()
     raises(AssertionError, f.rename, f.get_name() + '.renamed')
     test_name = f.get_name() + '.test'
     assert not exists(test_name)
     tmp = open(test_name, 'w+b')
     tmp.close()
     g = File(test_name)
     assert not g.is_temporary()
     g.rename(g.get_name() + '.renamed')
     assert g.get_name() == test_name + '.renamed'
     f.write(as_bytes('abc'))
     f.seek(0)
     assert len(f) == 3
     assert as_bytes('a') == f.read(1)
     assert as_bytes('bc') == f.read()
     f.close()
     assert not exists(f.get_name())
     raises(OSError, f.__len__)  # tmpfile removed on close
     h = File(g.get_name())
     g.write(as_bytes('a'))
     g.seek(0)
     assert g.tell() == 0
     g.seek_end()
     assert g.tell() == 1
     assert g.has_lock
     assert not h.has_lock
     raises(IOError, h.write, as_bytes('b'))
     g.flush()
     g.fsync()
     g.seek(0)
     g.truncate()
     g.close()
     h.close()
     unlink(g.get_name())