def __init__(self, file=None, readonly=False, repair=False):
"""(file:str=None, readonly:bool=False, repair:bool=False)
If file is empty (or None), a temporary file will be used.
"""
self.oid = -1
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
self.fp = file
self.pending_records = {}
self.pack_extra = None
self.fp.seek(0, 2)
if self.fp.tell() != 0:
assert self.has_format(self.fp)
else:
# Write header for new file.
self.fp.seek(len(self.MAGIC))
self._write_header(self.fp)
self._write_index(self.fp, {})
self.index = {}
self._build_index(repair)
max_oid = -1
for oid in self.index:
max_oid = max(max_oid, str_to_int8(oid))
self.oid = max_oid
self.invalid = set()
def test_c(self):
f = File()
s = Shelf(f)
f.seek(0)
p = Shelf(f)
f.seek(0)
q = Shelf(f)
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(self.file,
repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def b(self):
name = mktemp()
raises(OSError, File, name, readonly=True)
g = File(name, readonly=False)
g.close()
f = File(name, readonly=True)
assert f.is_readonly()
raises(AssertionError, f.write, 'ok')
raises(IOError, f.file.write, 'ok') # readonly file
f.close()
unlink(name)
def c(self):
name = mktemp()
name2 = mktemp()
f = File(name)
assert f.tell() == 0
g = File(name2)
g.close()
f.rename(name2)
assert exists(name2)
assert not exists(name)
f.close()
unlink(name2)
def test_a(self):
f = File()
s = Shelf(f)
name1 = s.next_name()
name2 = s.next_name()
assert name1 != name2
r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
assert s.get_value(name2) == name2 + name2
f.seek(0)
other = Shelf(f)
names = sorted(other.__iter__())
index = sorted(other.iterindex())
items = sorted(other.items())
assert names == [name1, name2], (name1, name2, names)
assert items == [(n, n+n) for n in names]
assert index == [(n, other.get_position(n)) for n in names]
def a(self):
f = File()
s = Shelf(f)
name1 = s.next_name()
name2 = s.next_name()
assert name1 != name2
r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
assert s.get_value(name2) == name2 + name2
f.seek(0)
other = Shelf(f)
names = sorted(other.__iter__())
index = sorted(other.iterindex())
items = sorted(other.items())
assert names == [name1, name2], (name1, name2, names)
assert items == [(n, n+n) for n in names]
assert index == [(n, other.get_position(n)) for n in names]
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(
self.file, repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def get_packer(self):
if (self.pending_records or
self.pack_extra is not None or
self.shelf.get_file().is_temporary() or
self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
seen = IntSet()
items = self.gen_oid_record(start_oid=int8_to_str(0), seen=seen)
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
for oid in self.pack_extra:
seen.discard(str_to_int8(oid))
for oid in self.pack_extra:
shelf.store(self.gen_oid_record(start_oid=oid, seen=seen))
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
def b(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 10
new_oid = s.new_oid()
assert new_oid == deleted_oid
new_oid = s.new_oid()
assert new_oid == int8_to_str(11)
def b(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 10
new_oid = s.new_oid()
assert new_oid == deleted_oid
new_oid = s.new_oid()
assert new_oid == int8_to_str(11)
def get_packer(self):
if (self.pending_records or self.pack_extra is not None
or self.shelf.get_file().is_temporary()
or self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
items = self.gen_oid_record(start_oid=int8_to_str(0))
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
shelf.store(
(name, self.shelf.get_value(name)) for name in self.pack_extra)
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
def c(self):
f = File()
s = Shelf(f)
f.seek(0)
p = Shelf(f)
f.seek(0)
q = Shelf(f)
def b(self):
name = mktemp()
raises(OSError, File, name, readonly=True)
g = File(name, readonly=False)
g.close()
f = File(name, readonly=True)
assert f.is_readonly()
raises(AssertionError, f.write, 'ok')
raises(IOError, f.file.write, 'ok') # readonly file
f.close()
unlink(name)
def a(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oids = [
r['a0']._p_oid, r['a2']._p_oid, r['a7']._p_oid, r['a8']._p_oid
]
del r['a0']
del r['a2']
del r['a7']
del r['a8']
c.commit()
c.pack()
c.abort()
assert c.get(deleted_oids[0])._p_is_ghost()
assert c.get(deleted_oids[1])._p_is_ghost()
raises(KeyError, getattr, c.get(deleted_oids[0]), 'a')
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 7
c.commit()
c.pack()
new_oid = s.new_oid()
assert new_oid == deleted_oids[-1], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-2], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-3], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-4], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == int8_to_str(11), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(12), repr(new_oid)
def c(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
r.clear()
c.commit()
c.pack()
c.abort()
new_oid = s.new_oid()
assert new_oid == int8_to_str(1), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(2), repr(new_oid)
def c(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
r.clear()
c.commit()
c.pack()
c.abort()
new_oid = s.new_oid()
assert new_oid == int8_to_str(1), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(2), repr(new_oid)
def a(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oids = [
r['a0']._p_oid, r['a2']._p_oid, r['a7']._p_oid, r['a8']._p_oid]
del r['a0']
del r['a2']
del r['a7']
del r['a8']
c.commit()
c.pack()
c.abort()
assert c.get(deleted_oids[0])._p_is_ghost()
assert c.get(deleted_oids[1])._p_is_ghost()
raises(ReadConflictError, getattr, c.get(deleted_oids[0]), 'a')
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 7
c.commit()
c.pack()
new_oid = s.new_oid()
assert new_oid == deleted_oids[-1], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-2], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-3], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-4], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == int8_to_str(11), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(12), repr(new_oid)
def d(self):
file = File()
word_array = WordArray(file=file, number_of_words=1, bytes_per_word=8)
file.seek(0)
word_array2 = WordArray(file=file, number_of_words=1, bytes_per_word=8)
def c(self):
name = mktemp()
name2 = mktemp()
f = File(name)
assert f.tell() == 0
g = File(name2)
g.close()
f.rename(name2)
assert exists(name2)
assert not exists(name)
f.close()
unlink(name2)
def _packer(self):
name = self.fp.get_name()
prepack_name = name + '.prepack'
pack_name = name + '.pack'
packed = File(pack_name)
if len(packed) > 0:
# packed contains data left from an incomplete pack attempt.
packed.seek(0)
packed.truncate()
self._write_header(packed)
# find all reachable objects. Note that when we yield, new
# commits may happen and pack_extra will contain new or modified
# OIDs.
index = {}
def gen_reachable_records():
pack_todo = [durus.connection.ROOT_OID]
while pack_todo or self.pack_extra:
if self.pack_extra:
oid = self.pack_extra.pop()
# note we don't check 'index' because it could be an
# object that got updated since the pack began and in
# that case we have to write the new record to the pack
# file
else:
oid = heapq.heappop(pack_todo)
if oid in index:
# we already wrote this object record
continue
record = self.load(oid)
oid2, data, refdata = unpack_record(record)
assert oid == oid2
# ensure we have records for objects referenced
for ref_oid in split_oids(refdata):
heapq.heappush(pack_todo, ref_oid)
yield (oid, record)
for z in self._write_transaction(
packed, gen_reachable_records(), index):
yield None # incremental pack, allow clients to be served
self._write_index(packed, index)
packed.flush()
packed.fsync()
if self.fp.is_temporary():
self.fp.close()
else:
self.fp.rename(prepack_name)
packed.rename(name)
self.fp = packed
for oid in self.index:
if oid not in index:
self.invalid.add(oid)
self.index = index
self.pack_extra = None
def random_filename():
f = File(prefix='schevodurus')
n = f.get_name()
f.close()
return n
def e(self):
f = File()
n1 = int8_to_str(0)
n2 = int8_to_str(1)
s = Shelf(f, items=[(n1, 'record1'), (n2, 'record2')])
def b(self):
file = File()
int_array = IntArray(file=file, number_of_ints=10, maximum_int=10)
file.seek(0)
int_array2 = IntArray(file=file)
assert len(int_array2) == 10
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def c(self):
file = File()
int_array = IntArray(file=file, number_of_ints=0, maximum_int=0)
file.seek(0)
int_array2 = IntArray(file=file)
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
