Пример #1
0
    def check_storage_tools(self):
        connection = Connection(self._get_storage())
        root = connection.get_root()
        root['a'] = Persistent()
        root['b'] = Persistent()
        connection.commit()
        index = get_reference_index(connection.get_storage())
        assert index == {
            int8_to_str(1): [int8_to_str(0)],
            int8_to_str(2): [int8_to_str(0)]
        }
        census = get_census(connection.get_storage())
        assert census == {
            as_bytes('PersistentDict'): 1,
            as_bytes('Persistent'): 2
        }
        references = list(
            gen_referring_oid_record(connection.get_storage(), int8_to_str(1)))
        assert references == [(int8_to_str(0),
                               connection.get_storage().load(int8_to_str(0)))]

        class Fake(object):
            pass

        s = Fake()
        s.__class__ = Storage
        raises(RuntimeError, s.__init__)
        raises(NotImplementedError, s.load, None)
        raises(NotImplementedError, s.begin)
        raises(NotImplementedError, s.store, None, None)
        raises(NotImplementedError, s.end)
        raises(NotImplementedError, s.sync)
        g = s.gen_oid_record()
        raises(NotImplementedError, next, g)
Пример #2
0
 def check_file_storage(self):
     name = mktemp()
     b = FileStorage(name)
     assert b.new_oid() == int8_to_str(0)
     assert b.new_oid() == int8_to_str(1)
     assert b.new_oid() == int8_to_str(2)
     raises(KeyError, b.load, int8_to_str(0))
     record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
     b.begin()
     b.store(int8_to_str(0), record)
     b.end()
     b.sync()
     b.begin()
     b.store(int8_to_str(1), pack_record(
         int8_to_str(1), as_bytes('no'), as_bytes('')))
     b.end()
     assert len(list(b.gen_oid_record(start_oid=int8_to_str(0)))) == 1
     assert len(list(b.gen_oid_record())) == 2
     b.pack()
     b.close()
     unlink(name + '.prepack')
     raises(ValueError, b.pack) # storage closed
     unlink(name + '.pack')
     raises(ValueError, b.load, int8_to_str(0)) # storage closed
     unlink(name)
Пример #3
0
 def check_client_storage(self):
     b = ClientStorage(address=self.address)
     c = ClientStorage(address=self.address)
     oid = b.new_oid()
     assert oid == int8_to_str(0), repr(oid)
     oid = b.new_oid()
     assert oid == int8_to_str(1), repr(oid)
     oid = b.new_oid()
     assert oid == int8_to_str(2), repr(oid)
     raises(KeyError, b.load, int8_to_str(0))
     record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
     b.begin()
     b.store(int8_to_str(0), record)
     assert b.end() is None
     b.load(int8_to_str(0))
     assert b.sync() == []
     b.begin()
     b.store(
         int8_to_str(1),
         pack_record(int8_to_str(1), as_bytes('no'), as_bytes('')))
     b.end()
     assert len(list(b.gen_oid_record())) == 1
     records = b.bulk_load([int8_to_str(0), int8_to_str(1)])
     assert len(list(records)) == 2
     records = b.bulk_load([int8_to_str(0), int8_to_str(1), int8_to_str(2)])
     raises(DurusKeyError, list, records)
     b.pack()
     assert len(list(b.gen_oid_record())) == 1
     raises(ReadConflictError, c.load, int8_to_str(0))
     raises(ReadConflictError, c.load, int8_to_str(0))
     assert set(c.sync()) == set([int8_to_str(0), int8_to_str(1)])
     assert record == c.load(int8_to_str(0))
     b.close()
     c.close()
Пример #4
0
 def check_storage_tools(self):
     connection = Connection(self._get_storage())
     root = connection.get_root()
     root['a'] = Persistent()
     root['b'] = Persistent()
     connection.commit()
     index = get_reference_index(connection.get_storage())
     assert index == {
         int8_to_str(1): [int8_to_str(0)], int8_to_str(2): [int8_to_str(0)]}
     census = get_census(connection.get_storage())
     assert census == {as_bytes('PersistentDict'):1, as_bytes('Persistent'):2}
     references = list(gen_referring_oid_record(connection.get_storage(),
                                                int8_to_str(1)))
     assert references == [
         (int8_to_str(0), connection.get_storage().load(int8_to_str(0)))]
     class Fake(object):
         pass
     s = Fake()
     s.__class__ = Storage
     raises(RuntimeError, s.__init__)
     raises(NotImplementedError, s.load, None)
     raises(NotImplementedError, s.begin)
     raises(NotImplementedError, s.store, None, None)
     raises(NotImplementedError, s.end)
     raises(NotImplementedError, s.sync)
     g = s.gen_oid_record()
     raises(NotImplementedError, next, g)
Пример #5
0
 def check_file_storage(self):
     name = mktemp()
     b = FileStorage(name)
     assert b.new_oid() == int8_to_str(0)
     assert b.new_oid() == int8_to_str(1)
     assert b.new_oid() == int8_to_str(2)
     raises(KeyError, b.load, int8_to_str(0))
     record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
     b.begin()
     b.store(int8_to_str(0), record)
     b.end()
     b.sync()
     b.begin()
     b.store(int8_to_str(1),
             pack_record(int8_to_str(1), as_bytes('no'), as_bytes('')))
     b.end()
     assert len(list(b.gen_oid_record(start_oid=int8_to_str(0)))) == 1
     assert len(list(b.gen_oid_record())) == 2
     b.pack()
     b.close()
     unlink(name + '.prepack')
     raises(ValueError, b.pack)  # storage closed
     unlink(name + '.pack')
     raises(ValueError, b.load, int8_to_str(0))  # storage closed
     unlink(name)
Пример #6
0
 def gen_items(records):
     for record in records:
         oid, data, refdata = unpack_record(record)
         yield str_to_int8(oid), as_bytes(data), as_bytes(refdata)
         if self.pack_extra is not None:
             # ensure object and refs are marked alive and not removed
             self.pack_extra.append(oid)
Пример #7
0
 def b(self):
     b = ByteArray(size=50)
     raises(IndexError, b.__getitem__, 50)
     raises(IndexError, b.__setitem__, 50, as_bytes('x'))
     raises(ValueError, b.__setitem__, 1, as_bytes('xx'))
     raises(ValueError, b.__setitem__, 1, as_bytes(''))
     raises(IndexError, b.__getslice__, 0, 51)
     raises(IndexError, b.__setslice__, 0, 51, as_bytes('x') * 51)
     raises(ValueError, b.__setslice__, 0, 50, as_bytes('x') * 49)
Пример #8
0
 def check_record_pack_unpack(self):
     oid = as_bytes('0'*8)
     data = as_bytes('sample')
     reflist = ['1'*8, '2'*8]
     reflist =  list(map(as_bytes, reflist))
     refs = join_bytes(reflist)
     result=unpack_record(pack_record(oid, data, refs))
     assert result[0] == oid
     assert result[1] == data
     assert split_oids(result[2]) == reflist
     assert split_oids('') == []
Пример #9
0
 def end(self, handle_invalidations=None):
     write(self.s, 'C')
     n = read_int4(self.s)
     oid_list = []
     if n != 0:
         packed_oids = read(self.s, n*8)
         oid_list = split_oids(packed_oids)
         try:
             handle_invalidations(oid_list)
         except ConflictError:
             self.transaction_new_oids.reverse()
             self.oid_pool.extend(self.transaction_new_oids)
             assert len(self.oid_pool) == len(set(self.oid_pool))
             self.begin() # clear out records and transaction_new_oids.
             write_int4(self.s, 0) # Tell server we are done.
             raise
     tdata = []
     for oid, record in iteritems(self.records):
         tdata.append(int4_to_str(8 + len(record)))
         tdata.append(as_bytes(oid))
         tdata.append(record)
     tdata = join_bytes(tdata)
     write_int4_str(self.s, tdata)
     self.records.clear()
     if len(tdata) > 0:
         status = read(self.s, 1)
         if status == STATUS_OKAY:
             pass
         elif status == STATUS_INVALID:
             raise WriteConflictError()
         else:
             raise ProtocolError('server returned invalid status %r' % status)
Пример #10
0
 def end(self, handle_invalidations=None):
     write(self.s, 'C')
     n = read_int4(self.s)
     oid_list = []
     if n != 0:
         packed_oids = read(self.s, n * 8)
         oid_list = split_oids(packed_oids)
         try:
             handle_invalidations(oid_list)
         except ConflictError:
             self.transaction_new_oids.reverse()
             self.oid_pool.extend(self.transaction_new_oids)
             assert len(self.oid_pool) == len(set(self.oid_pool))
             self.begin()  # clear out records and transaction_new_oids.
             write_int4(self.s, 0)  # Tell server we are done.
             raise
     tdata = []
     for oid, record in iteritems(self.records):
         tdata.append(int4_to_str(8 + len(record)))
         tdata.append(as_bytes(oid))
         tdata.append(record)
     tdata = join_bytes(tdata)
     write_int4_str(self.s, tdata)
     self.records.clear()
     if len(tdata) > 0:
         status = read(self.s, 1)
         if status == STATUS_OKAY:
             pass
         elif status == STATUS_INVALID:
             raise WriteConflictError()
         else:
             raise ProtocolError('server returned invalid status %r' %
                                 status)
Пример #11
0
 def d(self):
     s = BytesIO()
     for x in ("", "a", "ab", "a" * 1000):
         x = as_bytes(x)
         s.seek(0)
         write_int4_str(s, x)
         s.seek(0)
         assert x == read_int4_str(s)
Пример #12
0
 def check_object_reader(self):
     class FakeConnection:
         pass
     self.r = r = ObjectReader(FakeConnection())
     root = ('\x80\x02cdurus.persistent_dict\nPersistentDict\nq\x01.'
         '\x80\x02}q\x02U\x04dataq\x03}q\x04s.\x00\x00\x00\x00')
     root = as_bytes(root)
     assert r.get_ghost(root)._p_is_ghost()
Пример #13
0
 def b(self):
     n = 1000
     s = BytesIO()
     word_array = WordArray(file=s, bytes_per_word=8, number_of_words=n)
     for x in xrange(n):
         word_array[x] = int8_to_str(x)
     assert word_array[-1] == int8_to_str(n - 1)
     for x in xrange(n):
         assert x == str_to_int8(word_array[x])
         word_array[x] = int8_to_str(2*x)
         assert x == str_to_int8(word_array[x]) / 2
     assert len(word_array) == n
     assert raises(IndexError, word_array.__getitem__, n + 1)
     s.seek(0)
     word_array2 = WordArray(file=s)
     word_array2[-1] = as_bytes('mmmmmmmm')
     assert word_array2[-1] == as_bytes('mmmmmmmm')
Пример #14
0
 def b(self):
     s = BytesIO()
     for x in ('', 'a', 'ab', 'a' * 1000):
         x = as_bytes(x)
         s.seek(0)
         write(s, x)
         s.seek(0)
         assert x == read(s, len(x))
Пример #15
0
 def d(self):
     s = BytesIO()
     for x in ('', 'a', 'ab', 'a' * 1000):
         x = as_bytes(x)
         s.seek(0)
         write_int4_str(s, x)
         s.seek(0)
         assert x == read_int4_str(s)
Пример #16
0
 def b(self):
     s = BytesIO()
     for x in ("", "a", "ab", "a" * 1000):
         x = as_bytes(x)
         s.seek(0)
         write(s, x)
         s.seek(0)
         assert x == read(s, len(x))
Пример #17
0
 def e(self):
     s = BytesIO()
     durus.utils.TRACE = True
     for x in ('', 'a', 'ab', 'a' * 1000):
         x = as_bytes(x)
         s.seek(0)
         write_int8_str(s, x)
         s.seek(0)
         assert x == read_int8_str(s)
     durus.utils.TRACE = False
Пример #18
0
 def check_memory_storage(self):
     b = MemoryStorage()
     assert b.new_oid() == int8_to_str(0)
     assert b.new_oid() == int8_to_str(1)
     assert b.new_oid() == int8_to_str(2)
     raises(KeyError, b.load, int8_to_str(0))
     record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
     b.begin()
     b.store(int8_to_str(0), record)
     b.end()
     b.sync()
     b.begin()
     b.store(
         int8_to_str(1),
         pack_record(int8_to_str(1), as_bytes('no'), as_bytes('')))
     b.end()
     assert len(list(b.gen_oid_record())) == 1
     assert record == b.load(int8_to_str(0))
     records = b.bulk_load([int8_to_str(0), int8_to_str(1)])
     assert len(list(records)) == 2
     records = b.bulk_load([int8_to_str(0), int8_to_str(1), int8_to_str(2)])
     raises(KeyError, list, records)
Пример #19
0
 def a(self):
     f = File()
     f.rename(f.get_name())
     assert f.is_temporary()
     raises(AssertionError, f.rename, f.get_name() + '.renamed')
     test_name = f.get_name() + '.test'
     assert not exists(test_name)
     tmp = open(test_name, 'w+b')
     tmp.close()
     g = File(test_name)
     assert not g.is_temporary()
     g.rename(g.get_name() + '.renamed')
     assert g.get_name() == test_name + '.renamed'
     f.write(as_bytes('abc'))
     f.seek(0)
     assert len(f) == 3
     assert as_bytes('a') == f.read(1)
     assert as_bytes('bc') == f.read()
     f.close()
     assert not exists(f.get_name())
     raises(OSError, f.__len__) # tmpfile removed on close
     h = File(g.get_name())
     g.write(as_bytes('a'))
     g.seek(0)
     assert g.tell() == 0
     g.seek_end()
     assert g.tell() == 1
     assert g.has_lock
     assert not h.has_lock
     raises(IOError, h.write, as_bytes('b'))
     g.flush()
     g.fsync()
     g.seek(0)
     g.truncate()
     g.close()
     h.close()
     unlink(g.get_name())
Пример #20
0
 def a(self):
     f = File()
     f.rename(f.get_name())
     assert f.is_temporary()
     raises(AssertionError, f.rename, f.get_name() + '.renamed')
     test_name = f.get_name() + '.test'
     assert not exists(test_name)
     tmp = open(test_name, 'w+b')
     tmp.close()
     g = File(test_name)
     assert not g.is_temporary()
     g.rename(g.get_name() + '.renamed')
     assert g.get_name() == test_name + '.renamed'
     f.write(as_bytes('abc'))
     f.seek(0)
     assert len(f) == 3
     assert as_bytes('a') == f.read(1)
     assert as_bytes('bc') == f.read()
     f.close()
     assert not exists(f.get_name())
     raises(OSError, f.__len__)  # tmpfile removed on close
     h = File(g.get_name())
     g.write(as_bytes('a'))
     g.seek(0)
     assert g.tell() == 0
     g.seek_end()
     assert g.tell() == 1
     assert g.has_lock
     assert not h.has_lock
     raises(IOError, h.write, as_bytes('b'))
     g.flush()
     g.fsync()
     g.seek(0)
     g.truncate()
     g.close()
     h.close()
     unlink(g.get_name())
Пример #21
0
 def get_stored_pickle(self, oid):
     """(oid:str) -> str
     Retrieve the pickle from storage.  Will raise ReadConflictError if
     the oid is invalid.
     """
     assert oid not in self.invalid_oids, "still conflicted: missing abort()"
     try:
         record = self.storage.load(oid)
     except ReadConflictError:
         invalid_oids = self.storage.sync()
         self._handle_invalidations(invalid_oids, read_oid=oid)
         record = self.storage.load(oid)
     oid2, data, refdata = unpack_record(record)
     assert as_bytes(oid) == oid2, (oid, oid2)
     return data
Пример #22
0
def touch_every_reference(connection, *words):
    """(connection:Connection, *words:(str))
    Mark as changed, every object whose pickled class/state contains any
    of the given words.  This is useful when you move or rename a class,
    so that all references can be updated.
    """
    get = connection.get
    reader = ObjectReader(connection)
    words = [as_bytes(w) for w in words]
    for oid, record in connection.get_storage().gen_oid_record():
        record_oid, data, refs = unpack_record(record)
        state = reader.get_state_pickle(data)
        for word in words:
            if word in data or word in state:
                get(oid)._p_note_change()
Пример #23
0
def touch_every_reference(connection, *words):
    """(connection:Connection, *words:(str))
    Mark as changed, every object whose pickled class/state contains any
    of the given words.  This is useful when you move or rename a class,
    so that all references can be updated.
    """
    get = connection.get
    reader = ObjectReader(connection)
    words = [as_bytes(w) for w in words]
    for oid, record in connection.get_storage().gen_oid_record():
        record_oid, data, refs = unpack_record(record)
        state = reader.get_state_pickle(data)
        for word in words:
            if word in data or word in state:
                get(oid)._p_note_change()
Пример #24
0
 def a(self):
     for sample in (["a"], ["a", "b"], ["ab", "cd", "ef"]):
         sample = [as_bytes(x) for x in sample]
         s = BytesIO()
         number_of_words = len(sample)
         bytes_per_word = 0
         if sample:
             bytes_per_word = len(sample[0])
         word_array = WordArray(file=s, bytes_per_word=bytes_per_word, number_of_words=number_of_words)
         for j, word in enumerate(sample):
             word_array[j] = word
         assert list(word_array) == sample, (list(word_array), sample)
     assert raises(ValueError, word_array.__setitem__, 1, "sdf")
     assert raises(IndexError, word_array.__setitem__, 10, "sf")
     assert raises(IndexError, word_array.__getitem__, -10)
Пример #25
0
 def get_stored_pickle(self, oid):
     """(oid:str) -> str
     Retrieve the pickle from storage.  Will raise ReadConflictError if
     the oid is invalid.
     """
     assert oid not in self.invalid_oids, "still conflicted: missing abort()"
     try:
         record = self.storage.load(oid)
     except ReadConflictError:
         invalid_oids = self.storage.sync()
         self._handle_invalidations(invalid_oids, read_oid=oid)
         record = self.storage.load(oid)
     oid2, data, refdata = unpack_record(record)
     assert as_bytes(oid) == oid2, (oid, oid2)
     return data
Пример #26
0
 def check_repair(self):
     name = mktemp()
     g = FileStorage(name)
     g.close()
     f = open(name, 'r+b')
     f.seek(0, 2)
     p = f.tell()
     f.write(as_bytes('b'))
     f.flush()
     raises(ShortRead, FileStorage, name, readonly=True)
     h = FileStorage(name, repair=True)
     f.seek(0, 2)
     assert p == f.tell()
     f.close()
     h.close()
     unlink(name)
Пример #27
0
 def check_repair(self):
     name = mktemp()
     g = FileStorage(name)
     g.close()
     f = open(name, 'r+b')
     f.seek(0, 2)
     p = f.tell()
     f.write(as_bytes('b'))
     f.flush()
     raises(ShortRead, FileStorage, name, readonly=True)
     h = FileStorage(name, repair=True)
     f.seek(0, 2)
     assert p == f.tell()
     f.close()
     h.close()
     unlink(name)
Пример #28
0
 def a(self):
     s = BytesIO()
     b = ByteArray(size=10000, file=s)
     assert list(b) == [as_bytes('\x00') for j in xrange(10000)], list(b)
     for j in xrange(10000):
         assert as_bytes('\x00') == b[j]
     for j in xrange(10000):
         b[j] = as_bytes('!')
     for j in xrange(10000):
         assert as_bytes('!') == b[j]
     assert b[0:3] == as_bytes('!!!')
     assert b[47:50] == as_bytes('!!!'), repr(b[47:50])
     s = BytesIO()
     b2 = ByteArray(file=s)
     b2.set_size(10000, init_byte=as_bytes('\xff'))
     for j in xrange(10000):
         assert as_bytes('\xff') == b2[j], (j, b2[j])
     s.seek(0)
     raises(AssertionError, ByteArray, size=20000, file=s)
Пример #29
0
 def a(self):
     for sample in (['a'], ['a', 'b'], ['ab', 'cd', 'ef']):
         sample = [as_bytes(x) for x in sample]
         s = BytesIO()
         number_of_words = len(sample)
         bytes_per_word = 0
         if sample:
             bytes_per_word = len(sample[0])
         word_array = WordArray(
             file=s,
             bytes_per_word=bytes_per_word,
             number_of_words=number_of_words)
         for j, word in enumerate(sample):
             word_array[j] = word
         assert list(word_array) == sample, (list(word_array), sample)
     assert raises(ValueError, word_array.__setitem__, 1, 'sdf')
     assert raises(IndexError, word_array.__setitem__, 10, 'sf')
     assert raises(IndexError, word_array.__getitem__, -10)
Пример #30
0
 def recv(x, n):
     return as_bytes(choice(['a', 'bb'])[:n])
Пример #31
0
 def c(self):
     s = BytesIO(as_bytes('asdfasdfadsf'))
     s.seek(0)
     assert raises(ShortRead, WordArray, file=s)
Пример #32
0
 def recv(x, n):
     return as_bytes(choice(['a', 'bb'])[:n])
Пример #33
0
 def __init__(self, *args):
     self.io = BytesIO(join_bytes(as_bytes(a) for a in args))
Пример #34
0
class Shelf(object):
    """
    A Shelf wraps a file and uses it to hold a mapping.
    Each item in the mapping associates a string "name"
    with a string "value".  It would be like an ordinary mapping,
    except that the Shelf also maintains and provides outside
    access to the "position" for each name, which can be used
    directly to lookup the current value for a name. 
    When new values are assigned to a set of names, the changes
    occur in batches.  
    
    The Shelf class is written for use by ShelfStorage.
    The names of the Shelf are oids.
    The values of the Shelf are object records.
    The batches of changes are transactions.
    
    Here is the sequence of parts in a Shelf file:
    1) a prefix string that distinguishes the file format from that of other 
       file storages;
    2) an initial transaction;
    3) an offset mapping;
    4) a sequence of zero or more additional transactions.
    
    A transaction consists of the following:
    1) an number of bytes remaining in this transaction;
    2) a sequence of zero or more object records.

    An offset mapping consists of the following:
    1) the number of bytes remaining in this offset mapping; 
    2) the number of bytes in each entry in the mapping;
    3) the number of entries in the mapping.
 
    An object record consists of the following:
    1) the number of bytes in rest of the record;
    2) the record, as produced by durus.serialize.pack_record().

    A record produced by durus.serialize.pack_record() is as follows:
        1) an 8 byte oid;
        2) the number (4-byte, unsigned, big-endian int) of bytes in the 
           following;
        3) the pickle of the object's class followed by the (possibly zlib 
           compressed) pickle of the object's state (pickles produced in 
           sequence using the same pickler, with pickle protocol 2);
        4) a sequence of zero or more oids of persistent objects referenced 
           in the pickled object state.
    
    Except as noted, all numbers are stored as 8-byte unsigned big-endian ints.

    After the initial construction of a Shelf is completed, all subsequent
    writing happens at the end of the file.
    """
    prefix = as_bytes("SHELF-1\n")

    def __init__(self, file=None, items=None, repair=False, readonly=False):
        """(File:str:None, [(str:str)], boolean)
        """
        if file is None:
            file = File()
            assert not readonly
            assert not repair
        elif not hasattr(file, 'seek'):
            file = File(file, readonly=readonly)
        if not readonly:
            file.obtain_lock()
        file.seek(0, 2)  # seek end
        if file.tell() == 0:
            # The file is empty.
            for result in self.generate_shelf(file=file, items=items or []):
                pass
        else:
            assert items is None
        # The file is not empty.
        assert self.has_format(file)
        self.file = file
        self.file.seek(len(self.prefix))
        n = read_int8(self.file)  # bytes in first transaction
        self.file.seek(self.file.tell() + n)
        self.offset_map = OffsetMap(self.file)
        # Initialize the memory index.
        self.memory_index = {}
        while True:
            transaction_offsets = read_transaction_offsets(self.file,
                                                           repair=repair)
            if transaction_offsets is None:
                break
            self.memory_index.update(transaction_offsets)
        self.file.seek_end()
        self.unused_name_generator = None

    @classmethod
    def has_format(klass, file):
        file.seek(0)
        try:
            prefix = read(file, len(klass.prefix))
        except ShortRead:
            return False
        return klass.prefix == prefix

    @classmethod
    def generate_shelf(klass, file, items):
        """(File, [(str, str)])
        This returns a generator that writes a new Shelf into file,
        iterating once through the given items.
        The use of an iterator makes it possible to build a new Shelf 
        incrementally.
        """
        file.seek_end()
        if not file.tell() == 0:
            raise ValueError("Expected %s to be empty." % file)
        write(file, klass.prefix)
        if not items:
            # Just write an empty transaction.
            write_int8(file, 0)
            # Write an empty index array.
            offset_map = OffsetMap(file)
        else:
            # Write a transaction here with the given items.
            transaction_start = file.tell()
            # Write a placeholder for the length.
            write_int8(file, 0)
            # Loop over the items, writing their records.
            # Keep track of max_key and max_offset.
            max_key = 0
            max_offset = 0
            n = 0
            for name, value in items:
                max_key = max(max_key, str_to_int8(name))
                max_offset = max(max_offset, file.tell())
                write_int8(file, len(name) + len(value))
                write(file, name)
                write(file, value)
                n += 1
                yield n
            transaction_end = file.tell()
            # Write the correct transaction length.
            file.seek(transaction_start)
            write_int8(file, transaction_end - transaction_start - 8)
            # Write the empty array with the calculated dimensions.
            file.seek(transaction_end)
            for step in OffsetMap.generate(file, max_key, max_offset):
                yield step
            offset_map = OffsetMap(file)
            # Now read through the records and record the offsets in the array.
            file.seek(transaction_start + 8)
            while file.tell() < transaction_end:
                position = file.tell()
                record_length = read_int8(file)
                name = read(file, 8)
                k = str_to_int8(name)
                offset_map[k] = position
                file.seek(position + 8 + record_length)
                n -= 1
                yield n
        for index in offset_map.gen_stitch():
            yield index

    def next_name(self):
        """() -> str
        Return the next element in a sequence of names.
        Names returned have not been used, and they have not been returned
        by previous calls to this function.
        """
        if self.unused_name_generator is None:

            def generate_unused_names():
                for j in self.offset_map.gen_holes():
                    name = int8_to_str(j)
                    if name not in self.memory_index:
                        yield name
                # Now continue with values above those in the offset map.
                j = self.offset_map.get_array_size()
                while True:
                    name = int8_to_str(j)
                    if name not in self.memory_index:
                        yield name
                    j += 1

            self.unused_name_generator = generate_unused_names()
        return next(self.unused_name_generator)

    def store(self, name_value_sequence):
        """([(str, str)]) -> [(str, int|None, int)]
        Record all of the items in the sequence.
        Return a list of triples, each giving a name, an old position (or None
        if this is a new name), and a new position.
        """
        self.file.seek_end()
        start = self.file.tell()
        write_int8(self.file, 0)
        result = []
        index = {}
        try:
            for name, value in name_value_sequence:
                new_position = self.file.tell()
                old_position = self.get_position(name)
                index[name] = new_position
                result.append((name, old_position, new_position))
                write_int8(self.file, len(name) + len(value))
                write(self.file, name)
                write(self.file, value)
        except:
            # Revert before raising.
            self.file.seek(start)
            self.file.truncate()
            raise
        end = self.file.tell()
        self.file.seek(start)
        write_int8(self.file, end - start - 8)
        self.file.seek(end)
        self.memory_index.update(index)
        return result

    def get_position(self, name):
        """(str) -> int
        Return the position of the most recent value with this name.
        """
        if len(name) != 8:
            raise ValueError("Expected a string with 8 bytes.")
        p = self.memory_index.get(name, None)
        if p is not None:
            return p
        current = self.file.tell()
        result = self.offset_map.get(str_to_int8(name), None)
        self.file.seek(current)
        if result is None or result >= self.offset_map.get_start():
            return None
        else:
            return result

    def get_item_at_position(self, position):
        """(int) -> str, str
        """
        self.file.seek(position)
        record = read_int8_str(self.file)
        return record[:8], record[8:]

    def get_value(self, name):
        position = self.get_position(name)
        if position is None:
            return None
        else:
            item = self.get_item_at_position(position)
            assert item[0] == name
            return item[1]

    def iterindex(self):
        for n, position in iteritems(self.offset_map):
            if position < self.offset_map.get_start():
                name = int8_to_str(n)
                if name not in self.memory_index:
                    yield name, position
        for item in list(self.memory_index.items()):
            yield item

    def __iter__(self):
        for name, position in self.iterindex():
            yield name

    def iteritems(self):
        for name, position in self.iterindex():
            item = self.get_item_at_position(position)
            assert item[0] == name, (name, item)
            yield item

    items = iteritems

    def __contains__(self, name):
        return self.get_position(name) != None

    def get_offset_map(self):
        return self.offset_map

    def get_file(self):
        return self.file

    def close(self):
        self.file.close()
Пример #35
0
 def recv(self, n):
     if n > 10:
         return as_bytes('')
     return as_bytes('x')
Пример #36
0
 def test_d(self):
     s = BytesIO(as_bytes('nope'))
     assert raises(AssertionError, Shelf, s, readonly=True)
     s = BytesIO(as_bytes("SHELF-1\nbogus"))
     assert raises(ShortRead, Shelf, s, readonly=True)
Пример #37
0
 def test_b(self):
     s = Shelf()
     assert s.get_value(as_bytes('okokokok')) is None
     assert raises(ValueError, s.get_value, as_bytes('okok'))
Пример #38
0
 def b(self):
     s = Shelf()
     assert s.get_value(as_bytes('okokokok')) is None
     assert raises(ValueError, s.get_value, as_bytes('okok'))
Пример #39
0
from durus.utils import read_int4, read_int4_str, write_int4_str
from durus.utils import join_bytes, write_all, next, as_bytes
from os.path import exists
from time import sleep
import errno
import select
import socket
import sys

import os
if os.name != 'nt':
    from grp import getgrnam, getgrgid
    from os import unlink, stat, chown, geteuid, getegid, umask, getpid
    from pwd import getpwnam, getpwuid

STATUS_OKAY = as_bytes('O')
STATUS_KEYERROR = as_bytes('K')
STATUS_INVALID = as_bytes('I')

TIMEOUT = 10
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 2972
DEFAULT_GCBYTES = 0


class _Client(object):
    def __init__(self, s, addr):
        self.s = s
        self.addr = addr
        self.invalid = set()
        self.unused_oids = set()
Пример #40
0
    data = record[12:data_end]
    refs = record[data_end:]
    return oid, data, refs

def split_oids(s):
    """(s:str) -> [str]
    s is a packed string of oids.  Return a list of oid strings.
    """
    if not s:
        return []
    num, extra = divmod(len(s), 8)
    assert extra == 0, s
    fmt = '8s' * num
    return list(struct.unpack('>' + fmt, s))

NEWLINE = as_bytes('\n')

def extract_class_name(record):
    try:
        oid, state, refs = unpack_record(record)
        return state.split(NEWLINE, 2)[1]
    except IndexError:
        return "?"

if sys.version_info[0] < 3:
    def method(a, b):
        return MethodType(a, b, object)
else:
    def method(a, b):
        return MethodType(a, b)
Пример #41
0
 def d(self):
     s = BytesIO(as_bytes('nope'))
     assert raises(AssertionError, Shelf, s, readonly=True)
     s = BytesIO(as_bytes("SHELF-1\nbogus"))
     assert raises(ShortRead, Shelf, s, readonly=True)
Пример #42
0
 def _p_format_oid(self):
     oid = self._p_oid
     return str(oid and str_to_int8(as_bytes(oid)))
Пример #43
0
    return oid, data, refs


def split_oids(s):
    """(s:str) -> [str]
    s is a packed string of oids.  Return a list of oid strings.
    """
    if not s:
        return []
    num, extra = divmod(len(s), 8)
    assert extra == 0, s
    fmt = '8s' * num
    return list(struct.unpack('>' + fmt, s))


NEWLINE = as_bytes('\n')


def extract_class_name(record):
    try:
        oid, state, refs = unpack_record(record)
        return state.split(NEWLINE, 2)[1]
    except IndexError:
        return "?"


if sys.version < "3":

    def method(a, b):
        return MethodType(a, b, object)
else: