def test_c(self):
f = File()
s = Shelf(f)
f.seek(0)
p = Shelf(f)
f.seek(0)
q = Shelf(f)
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(self.file,
repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(
self.file, repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def c(self):
f = File()
s = Shelf(f)
f.seek(0)
p = Shelf(f)
f.seek(0)
q = Shelf(f)
def _packer(self):
name = self.fp.get_name()
prepack_name = name + '.prepack'
pack_name = name + '.pack'
packed = File(pack_name)
if len(packed) > 0:
# packed contains data left from an incomplete pack attempt.
packed.seek(0)
packed.truncate()
self._write_header(packed)
# find all reachable objects. Note that when we yield, new
# commits may happen and pack_extra will contain new or modified
# OIDs.
index = {}
def gen_reachable_records():
# we order the todo queue by file offset. The hope is that the
# packed file will be mostly the same as the old file in order
# to speed up the rsync delta process.
default_rank = 2**64
pack_todo = [(0, durus.connection.ROOT_OID)]
while pack_todo or self.pack_extra:
if self.pack_extra:
oid = self.pack_extra.pop()
# note we don't check 'index' because it could be an
# object that got updated since the pack began and in
# that case we have to write the new record to the pack
# file
else:
rank, oid = heapq.heappop(pack_todo)
if oid in index:
# we already wrote this object record
continue
record = self.load(oid)
oid2, data, refdata = unpack_record(record)
assert oid == oid2
# ensure we have records for objects referenced
for ref_oid in split_oids(refdata):
item = (self.index.get(ref_oid, default_rank), ref_oid)
heapq.heappush(pack_todo, item)
yield (oid, record)
for z in self._write_transaction(
packed, gen_reachable_records(), index):
yield None # incremental pack, allow clients to be served
self._write_index(packed, index)
packed.flush()
packed.fsync()
if self.fp.is_temporary():
self.fp.close()
else:
self.fp.rename(prepack_name)
packed.rename(name)
self.fp = packed
for oid in self.index:
if oid not in index:
self.invalid.add(oid)
self.index = index
self.pack_extra = None
def test_a(self):
f = File()
s = Shelf(f)
name1 = s.next_name()
name2 = s.next_name()
assert name1 != name2
r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
assert s.get_value(name2) == name2 + name2
f.seek(0)
other = Shelf(f)
names = sorted(other.__iter__())
index = sorted(other.iterindex())
items = sorted(other.items())
assert names == [name1, name2], (name1, name2, names)
assert items == [(n, n+n) for n in names]
assert index == [(n, other.get_position(n)) for n in names]
def a(self):
f = File()
s = Shelf(f)
name1 = s.next_name()
name2 = s.next_name()
assert name1 != name2
r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
assert s.get_value(name2) == name2 + name2
f.seek(0)
other = Shelf(f)
names = sorted(other.__iter__())
index = sorted(other.iterindex())
items = sorted(other.items())
assert names == [name1, name2], (name1, name2, names)
assert items == [(n, n+n) for n in names]
assert index == [(n, other.get_position(n)) for n in names]
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def c(self):
file = File()
int_array = IntArray(file=file, number_of_ints=0, maximum_int=0)
file.seek(0)
int_array2 = IntArray(file=file)
def b(self):
file = File()
int_array = IntArray(file=file, number_of_ints=10, maximum_int=10)
file.seek(0)
int_array2 = IntArray(file=file)
assert len(int_array2) == 10
def d(self):
file = File()
word_array = WordArray(file=file, number_of_words=1, bytes_per_word=8)
file.seek(0)
word_array2 = WordArray(file=file, number_of_words=1, bytes_per_word=8)
