def check_storage_tools(self):
connection = Connection(self._get_storage())
root = connection.get_root()
root['a'] = Persistent()
root['b'] = Persistent()
connection.commit()
index = get_reference_index(connection.get_storage())
assert index == {
int8_to_str(1): [int8_to_str(0)],
int8_to_str(2): [int8_to_str(0)]
}
census = get_census(connection.get_storage())
assert census == {
as_bytes('PersistentDict'): 1,
as_bytes('Persistent'): 2
}
references = list(
gen_referring_oid_record(connection.get_storage(), int8_to_str(1)))
assert references == [(int8_to_str(0),
connection.get_storage().load(int8_to_str(0)))]
class Fake(object):
pass
s = Fake()
s.__class__ = Storage
raises(RuntimeError, s.__init__)
raises(NotImplementedError, s.load, None)
raises(NotImplementedError, s.begin)
raises(NotImplementedError, s.store, None, None)
raises(NotImplementedError, s.end)
raises(NotImplementedError, s.sync)
g = s.gen_oid_record()
raises(NotImplementedError, next, g)
def check_file_storage(self):
name = mktemp()
b = FileStorage(name)
assert b.new_oid() == int8_to_str(0)
assert b.new_oid() == int8_to_str(1)
assert b.new_oid() == int8_to_str(2)
raises(KeyError, b.load, int8_to_str(0))
record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
b.begin()
b.store(int8_to_str(0), record)
b.end()
b.sync()
b.begin()
b.store(int8_to_str(1), pack_record(
int8_to_str(1), as_bytes('no'), as_bytes('')))
b.end()
assert len(list(b.gen_oid_record(start_oid=int8_to_str(0)))) == 1
assert len(list(b.gen_oid_record())) == 2
b.pack()
b.close()
unlink(name + '.prepack')
raises(ValueError, b.pack) # storage closed
unlink(name + '.pack')
raises(ValueError, b.load, int8_to_str(0)) # storage closed
unlink(name)
def check_client_storage(self):
b = ClientStorage(address=self.address)
c = ClientStorage(address=self.address)
oid = b.new_oid()
assert oid == int8_to_str(0), repr(oid)
oid = b.new_oid()
assert oid == int8_to_str(1), repr(oid)
oid = b.new_oid()
assert oid == int8_to_str(2), repr(oid)
raises(KeyError, b.load, int8_to_str(0))
record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
b.begin()
b.store(int8_to_str(0), record)
assert b.end() is None
b.load(int8_to_str(0))
assert b.sync() == []
b.begin()
b.store(
int8_to_str(1),
pack_record(int8_to_str(1), as_bytes('no'), as_bytes('')))
b.end()
assert len(list(b.gen_oid_record())) == 1
records = b.bulk_load([int8_to_str(0), int8_to_str(1)])
assert len(list(records)) == 2
records = b.bulk_load([int8_to_str(0), int8_to_str(1), int8_to_str(2)])
raises(DurusKeyError, list, records)
b.pack()
assert len(list(b.gen_oid_record())) == 1
raises(ReadConflictError, c.load, int8_to_str(0))
raises(ReadConflictError, c.load, int8_to_str(0))
assert set(c.sync()) == set([int8_to_str(0), int8_to_str(1)])
assert record == c.load(int8_to_str(0))
b.close()
c.close()
def check_storage_tools(self):
connection = Connection(self._get_storage())
root = connection.get_root()
root['a'] = Persistent()
root['b'] = Persistent()
connection.commit()
index = get_reference_index(connection.get_storage())
assert index == {
int8_to_str(1): [int8_to_str(0)], int8_to_str(2): [int8_to_str(0)]}
census = get_census(connection.get_storage())
assert census == {as_bytes('PersistentDict'):1, as_bytes('Persistent'):2}
references = list(gen_referring_oid_record(connection.get_storage(),
int8_to_str(1)))
assert references == [
(int8_to_str(0), connection.get_storage().load(int8_to_str(0)))]
class Fake(object):
pass
s = Fake()
s.__class__ = Storage
raises(RuntimeError, s.__init__)
raises(NotImplementedError, s.load, None)
raises(NotImplementedError, s.begin)
raises(NotImplementedError, s.store, None, None)
raises(NotImplementedError, s.end)
raises(NotImplementedError, s.sync)
g = s.gen_oid_record()
raises(NotImplementedError, next, g)
def check_file_storage(self):
name = mktemp()
b = FileStorage(name)
assert b.new_oid() == int8_to_str(0)
assert b.new_oid() == int8_to_str(1)
assert b.new_oid() == int8_to_str(2)
raises(KeyError, b.load, int8_to_str(0))
record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
b.begin()
b.store(int8_to_str(0), record)
b.end()
b.sync()
b.begin()
b.store(int8_to_str(1),
pack_record(int8_to_str(1), as_bytes('no'), as_bytes('')))
b.end()
assert len(list(b.gen_oid_record(start_oid=int8_to_str(0)))) == 1
assert len(list(b.gen_oid_record())) == 2
b.pack()
b.close()
unlink(name + '.prepack')
raises(ValueError, b.pack) # storage closed
unlink(name + '.pack')
raises(ValueError, b.load, int8_to_str(0)) # storage closed
unlink(name)
def gen_items(records):
for record in records:
oid, data, refdata = unpack_record(record)
yield str_to_int8(oid), as_bytes(data), as_bytes(refdata)
if self.pack_extra is not None:
# ensure object and refs are marked alive and not removed
self.pack_extra.append(oid)
def b(self):
b = ByteArray(size=50)
raises(IndexError, b.__getitem__, 50)
raises(IndexError, b.__setitem__, 50, as_bytes('x'))
raises(ValueError, b.__setitem__, 1, as_bytes('xx'))
raises(ValueError, b.__setitem__, 1, as_bytes(''))
raises(IndexError, b.__getslice__, 0, 51)
raises(IndexError, b.__setslice__, 0, 51, as_bytes('x') * 51)
raises(ValueError, b.__setslice__, 0, 50, as_bytes('x') * 49)
def check_record_pack_unpack(self):
oid = as_bytes('0'*8)
data = as_bytes('sample')
reflist = ['1'*8, '2'*8]
reflist = list(map(as_bytes, reflist))
refs = join_bytes(reflist)
result=unpack_record(pack_record(oid, data, refs))
assert result[0] == oid
assert result[1] == data
assert split_oids(result[2]) == reflist
assert split_oids('') == []
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n*8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' % status)
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n * 8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' %
status)
def d(self):
s = BytesIO()
for x in ("", "a", "ab", "a" * 1000):
x = as_bytes(x)
s.seek(0)
write_int4_str(s, x)
s.seek(0)
assert x == read_int4_str(s)
def check_object_reader(self):
class FakeConnection:
pass
self.r = r = ObjectReader(FakeConnection())
root = ('\x80\x02cdurus.persistent_dict\nPersistentDict\nq\x01.'
'\x80\x02}q\x02U\x04dataq\x03}q\x04s.\x00\x00\x00\x00')
root = as_bytes(root)
assert r.get_ghost(root)._p_is_ghost()
def b(self):
n = 1000
s = BytesIO()
word_array = WordArray(file=s, bytes_per_word=8, number_of_words=n)
for x in xrange(n):
word_array[x] = int8_to_str(x)
assert word_array[-1] == int8_to_str(n - 1)
for x in xrange(n):
assert x == str_to_int8(word_array[x])
word_array[x] = int8_to_str(2*x)
assert x == str_to_int8(word_array[x]) / 2
assert len(word_array) == n
assert raises(IndexError, word_array.__getitem__, n + 1)
s.seek(0)
word_array2 = WordArray(file=s)
word_array2[-1] = as_bytes('mmmmmmmm')
assert word_array2[-1] == as_bytes('mmmmmmmm')
def b(self):
s = BytesIO()
for x in ('', 'a', 'ab', 'a' * 1000):
x = as_bytes(x)
s.seek(0)
write(s, x)
s.seek(0)
assert x == read(s, len(x))
def d(self):
s = BytesIO()
for x in ('', 'a', 'ab', 'a' * 1000):
x = as_bytes(x)
s.seek(0)
write_int4_str(s, x)
s.seek(0)
assert x == read_int4_str(s)
def b(self):
s = BytesIO()
for x in ("", "a", "ab", "a" * 1000):
x = as_bytes(x)
s.seek(0)
write(s, x)
s.seek(0)
assert x == read(s, len(x))
def e(self):
s = BytesIO()
durus.utils.TRACE = True
for x in ('', 'a', 'ab', 'a' * 1000):
x = as_bytes(x)
s.seek(0)
write_int8_str(s, x)
s.seek(0)
assert x == read_int8_str(s)
durus.utils.TRACE = False
def check_memory_storage(self):
b = MemoryStorage()
assert b.new_oid() == int8_to_str(0)
assert b.new_oid() == int8_to_str(1)
assert b.new_oid() == int8_to_str(2)
raises(KeyError, b.load, int8_to_str(0))
record = pack_record(int8_to_str(0), as_bytes('ok'), as_bytes(''))
b.begin()
b.store(int8_to_str(0), record)
b.end()
b.sync()
b.begin()
b.store(
int8_to_str(1),
pack_record(int8_to_str(1), as_bytes('no'), as_bytes('')))
b.end()
assert len(list(b.gen_oid_record())) == 1
assert record == b.load(int8_to_str(0))
records = b.bulk_load([int8_to_str(0), int8_to_str(1)])
assert len(list(records)) == 2
records = b.bulk_load([int8_to_str(0), int8_to_str(1), int8_to_str(2)])
raises(KeyError, list, records)
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def get_stored_pickle(self, oid):
"""(oid:str) -> str
Retrieve the pickle from storage. Will raise ReadConflictError if
the oid is invalid.
"""
assert oid not in self.invalid_oids, "still conflicted: missing abort()"
try:
record = self.storage.load(oid)
except ReadConflictError:
invalid_oids = self.storage.sync()
self._handle_invalidations(invalid_oids, read_oid=oid)
record = self.storage.load(oid)
oid2, data, refdata = unpack_record(record)
assert as_bytes(oid) == oid2, (oid, oid2)
return data
def touch_every_reference(connection, *words):
"""(connection:Connection, *words:(str))
Mark as changed, every object whose pickled class/state contains any
of the given words. This is useful when you move or rename a class,
so that all references can be updated.
"""
get = connection.get
reader = ObjectReader(connection)
words = [as_bytes(w) for w in words]
for oid, record in connection.get_storage().gen_oid_record():
record_oid, data, refs = unpack_record(record)
state = reader.get_state_pickle(data)
for word in words:
if word in data or word in state:
get(oid)._p_note_change()
def touch_every_reference(connection, *words):
"""(connection:Connection, *words:(str))
Mark as changed, every object whose pickled class/state contains any
of the given words. This is useful when you move or rename a class,
so that all references can be updated.
"""
get = connection.get
reader = ObjectReader(connection)
words = [as_bytes(w) for w in words]
for oid, record in connection.get_storage().gen_oid_record():
record_oid, data, refs = unpack_record(record)
state = reader.get_state_pickle(data)
for word in words:
if word in data or word in state:
get(oid)._p_note_change()
def a(self):
for sample in (["a"], ["a", "b"], ["ab", "cd", "ef"]):
sample = [as_bytes(x) for x in sample]
s = BytesIO()
number_of_words = len(sample)
bytes_per_word = 0
if sample:
bytes_per_word = len(sample[0])
word_array = WordArray(file=s, bytes_per_word=bytes_per_word, number_of_words=number_of_words)
for j, word in enumerate(sample):
word_array[j] = word
assert list(word_array) == sample, (list(word_array), sample)
assert raises(ValueError, word_array.__setitem__, 1, "sdf")
assert raises(IndexError, word_array.__setitem__, 10, "sf")
assert raises(IndexError, word_array.__getitem__, -10)
def get_stored_pickle(self, oid):
"""(oid:str) -> str
Retrieve the pickle from storage. Will raise ReadConflictError if
the oid is invalid.
"""
assert oid not in self.invalid_oids, "still conflicted: missing abort()"
try:
record = self.storage.load(oid)
except ReadConflictError:
invalid_oids = self.storage.sync()
self._handle_invalidations(invalid_oids, read_oid=oid)
record = self.storage.load(oid)
oid2, data, refdata = unpack_record(record)
assert as_bytes(oid) == oid2, (oid, oid2)
return data
def check_repair(self):
name = mktemp()
g = FileStorage(name)
g.close()
f = open(name, 'r+b')
f.seek(0, 2)
p = f.tell()
f.write(as_bytes('b'))
f.flush()
raises(ShortRead, FileStorage, name, readonly=True)
h = FileStorage(name, repair=True)
f.seek(0, 2)
assert p == f.tell()
f.close()
h.close()
unlink(name)
def check_repair(self):
name = mktemp()
g = FileStorage(name)
g.close()
f = open(name, 'r+b')
f.seek(0, 2)
p = f.tell()
f.write(as_bytes('b'))
f.flush()
raises(ShortRead, FileStorage, name, readonly=True)
h = FileStorage(name, repair=True)
f.seek(0, 2)
assert p == f.tell()
f.close()
h.close()
unlink(name)
def a(self):
s = BytesIO()
b = ByteArray(size=10000, file=s)
assert list(b) == [as_bytes('\x00') for j in xrange(10000)], list(b)
for j in xrange(10000):
assert as_bytes('\x00') == b[j]
for j in xrange(10000):
b[j] = as_bytes('!')
for j in xrange(10000):
assert as_bytes('!') == b[j]
assert b[0:3] == as_bytes('!!!')
assert b[47:50] == as_bytes('!!!'), repr(b[47:50])
s = BytesIO()
b2 = ByteArray(file=s)
b2.set_size(10000, init_byte=as_bytes('\xff'))
for j in xrange(10000):
assert as_bytes('\xff') == b2[j], (j, b2[j])
s.seek(0)
raises(AssertionError, ByteArray, size=20000, file=s)
def a(self):
for sample in (['a'], ['a', 'b'], ['ab', 'cd', 'ef']):
sample = [as_bytes(x) for x in sample]
s = BytesIO()
number_of_words = len(sample)
bytes_per_word = 0
if sample:
bytes_per_word = len(sample[0])
word_array = WordArray(
file=s,
bytes_per_word=bytes_per_word,
number_of_words=number_of_words)
for j, word in enumerate(sample):
word_array[j] = word
assert list(word_array) == sample, (list(word_array), sample)
assert raises(ValueError, word_array.__setitem__, 1, 'sdf')
assert raises(IndexError, word_array.__setitem__, 10, 'sf')
assert raises(IndexError, word_array.__getitem__, -10)
def recv(x, n):
return as_bytes(choice(['a', 'bb'])[:n])
def c(self):
s = BytesIO(as_bytes('asdfasdfadsf'))
s.seek(0)
assert raises(ShortRead, WordArray, file=s)
def recv(x, n):
return as_bytes(choice(['a', 'bb'])[:n])
def __init__(self, *args):
self.io = BytesIO(join_bytes(as_bytes(a) for a in args))
class Shelf(object):
"""
A Shelf wraps a file and uses it to hold a mapping.
Each item in the mapping associates a string "name"
with a string "value". It would be like an ordinary mapping,
except that the Shelf also maintains and provides outside
access to the "position" for each name, which can be used
directly to lookup the current value for a name.
When new values are assigned to a set of names, the changes
occur in batches.
The Shelf class is written for use by ShelfStorage.
The names of the Shelf are oids.
The values of the Shelf are object records.
The batches of changes are transactions.
Here is the sequence of parts in a Shelf file:
1) a prefix string that distinguishes the file format from that of other
file storages;
2) an initial transaction;
3) an offset mapping;
4) a sequence of zero or more additional transactions.
A transaction consists of the following:
1) an number of bytes remaining in this transaction;
2) a sequence of zero or more object records.
An offset mapping consists of the following:
1) the number of bytes remaining in this offset mapping;
2) the number of bytes in each entry in the mapping;
3) the number of entries in the mapping.
An object record consists of the following:
1) the number of bytes in rest of the record;
2) the record, as produced by durus.serialize.pack_record().
A record produced by durus.serialize.pack_record() is as follows:
1) an 8 byte oid;
2) the number (4-byte, unsigned, big-endian int) of bytes in the
following;
3) the pickle of the object's class followed by the (possibly zlib
compressed) pickle of the object's state (pickles produced in
sequence using the same pickler, with pickle protocol 2);
4) a sequence of zero or more oids of persistent objects referenced
in the pickled object state.
Except as noted, all numbers are stored as 8-byte unsigned big-endian ints.
After the initial construction of a Shelf is completed, all subsequent
writing happens at the end of the file.
"""
prefix = as_bytes("SHELF-1\n")
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(self.file,
repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
@classmethod
def has_format(klass, file):
file.seek(0)
try:
prefix = read(file, len(klass.prefix))
except ShortRead:
return False
return klass.prefix == prefix
@classmethod
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def next_name(self):
"""() -> str
Return the next element in a sequence of names.
Names returned have not been used, and they have not been returned
by previous calls to this function.
"""
if self.unused_name_generator is None:
def generate_unused_names():
for j in self.offset_map.gen_holes():
name = int8_to_str(j)
if name not in self.memory_index:
yield name
# Now continue with values above those in the offset map.
j = self.offset_map.get_array_size()
while True:
name = int8_to_str(j)
if name not in self.memory_index:
yield name
j += 1
self.unused_name_generator = generate_unused_names()
return next(self.unused_name_generator)
def store(self, name_value_sequence):
"""([(str, str)]) -> [(str, int|None, int)]
Record all of the items in the sequence.
Return a list of triples, each giving a name, an old position (or None
if this is a new name), and a new position.
"""
self.file.seek_end()
start = self.file.tell()
write_int8(self.file, 0)
result = []
index = {}
try:
for name, value in name_value_sequence:
new_position = self.file.tell()
old_position = self.get_position(name)
index[name] = new_position
result.append((name, old_position, new_position))
write_int8(self.file, len(name) + len(value))
write(self.file, name)
write(self.file, value)
except:
# Revert before raising.
self.file.seek(start)
self.file.truncate()
raise
end = self.file.tell()
self.file.seek(start)
write_int8(self.file, end - start - 8)
self.file.seek(end)
self.memory_index.update(index)
return result
def get_position(self, name):
"""(str) -> int
Return the position of the most recent value with this name.
"""
if len(name) != 8:
raise ValueError("Expected a string with 8 bytes.")
p = self.memory_index.get(name, None)
if p is not None:
return p
current = self.file.tell()
result = self.offset_map.get(str_to_int8(name), None)
self.file.seek(current)
if result is None or result >= self.offset_map.get_start():
return None
else:
return result
def get_item_at_position(self, position):
"""(int) -> str, str
"""
self.file.seek(position)
record = read_int8_str(self.file)
return record[:8], record[8:]
def get_value(self, name):
position = self.get_position(name)
if position is None:
return None
else:
item = self.get_item_at_position(position)
assert item[0] == name
return item[1]
def iterindex(self):
for n, position in iteritems(self.offset_map):
if position < self.offset_map.get_start():
name = int8_to_str(n)
if name not in self.memory_index:
yield name, position
for item in list(self.memory_index.items()):
yield item
def __iter__(self):
for name, position in self.iterindex():
yield name
def iteritems(self):
for name, position in self.iterindex():
item = self.get_item_at_position(position)
assert item[0] == name, (name, item)
yield item
items = iteritems
def __contains__(self, name):
return self.get_position(name) != None
def get_offset_map(self):
return self.offset_map
def get_file(self):
return self.file
def close(self):
self.file.close()
def recv(self, n):
if n > 10:
return as_bytes('')
return as_bytes('x')
def test_d(self):
s = BytesIO(as_bytes('nope'))
assert raises(AssertionError, Shelf, s, readonly=True)
s = BytesIO(as_bytes("SHELF-1\nbogus"))
assert raises(ShortRead, Shelf, s, readonly=True)
def test_b(self):
s = Shelf()
assert s.get_value(as_bytes('okokokok')) is None
assert raises(ValueError, s.get_value, as_bytes('okok'))
def b(self):
s = Shelf()
assert s.get_value(as_bytes('okokokok')) is None
assert raises(ValueError, s.get_value, as_bytes('okok'))
from durus.utils import read_int4, read_int4_str, write_int4_str
from durus.utils import join_bytes, write_all, next, as_bytes
from os.path import exists
from time import sleep
import errno
import select
import socket
import sys
import os
if os.name != 'nt':
from grp import getgrnam, getgrgid
from os import unlink, stat, chown, geteuid, getegid, umask, getpid
from pwd import getpwnam, getpwuid
STATUS_OKAY = as_bytes('O')
STATUS_KEYERROR = as_bytes('K')
STATUS_INVALID = as_bytes('I')
TIMEOUT = 10
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 2972
DEFAULT_GCBYTES = 0
class _Client(object):
def __init__(self, s, addr):
self.s = s
self.addr = addr
self.invalid = set()
self.unused_oids = set()
data = record[12:data_end]
refs = record[data_end:]
return oid, data, refs
def split_oids(s):
"""(s:str) -> [str]
s is a packed string of oids. Return a list of oid strings.
"""
if not s:
return []
num, extra = divmod(len(s), 8)
assert extra == 0, s
fmt = '8s' * num
return list(struct.unpack('>' + fmt, s))
NEWLINE = as_bytes('\n')
def extract_class_name(record):
try:
oid, state, refs = unpack_record(record)
return state.split(NEWLINE, 2)[1]
except IndexError:
return "?"
if sys.version_info[0] < 3:
def method(a, b):
return MethodType(a, b, object)
else:
def method(a, b):
return MethodType(a, b)
def d(self):
s = BytesIO(as_bytes('nope'))
assert raises(AssertionError, Shelf, s, readonly=True)
s = BytesIO(as_bytes("SHELF-1\nbogus"))
assert raises(ShortRead, Shelf, s, readonly=True)
def _p_format_oid(self):
oid = self._p_oid
return str(oid and str_to_int8(as_bytes(oid)))
return oid, data, refs
def split_oids(s):
"""(s:str) -> [str]
s is a packed string of oids. Return a list of oid strings.
"""
if not s:
return []
num, extra = divmod(len(s), 8)
assert extra == 0, s
fmt = '8s' * num
return list(struct.unpack('>' + fmt, s))
NEWLINE = as_bytes('\n')
def extract_class_name(record):
try:
oid, state, refs = unpack_record(record)
return state.split(NEWLINE, 2)[1]
except IndexError:
return "?"
if sys.version < "3":
def method(a, b):
return MethodType(a, b, object)
else: