def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(self.file,
repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def b(self):
name = mktemp()
raises(OSError, File, name, readonly=True)
g = File(name, readonly=False)
g.close()
f = File(name, readonly=True)
assert f.is_readonly()
raises(AssertionError, f.write, 'ok')
raises(IOError, f.file.write, 'ok') # readonly file
f.close()
unlink(name)
def c(self):
name = mktemp()
name2 = mktemp()
f = File(name)
assert f.tell() == 0
g = File(name2)
g.close()
f.rename(name2)
assert exists(name2)
assert not exists(name)
f.close()
unlink(name2)
def c(self):
f = File()
s = Shelf(f)
f.seek(0)
p = Shelf(f)
f.seek(0)
q = Shelf(f)
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def a(self):
f = File()
s = Shelf(f)
name1 = s.next_name()
name2 = s.next_name()
assert name1 != name2
r = s.store([(name1, name1 + name1), (name2, name2 + name2)])
assert s.get_value(name1) == name1 + name1, (name1, s.get_value(name1))
assert s.get_value(name2) == name2 + name2
f.seek(0)
other = Shelf(f)
names = sorted(other.__iter__())
index = sorted(other.iterindex())
items = sorted(other.items())
assert names == [name1, name2], (name1, name2, names)
assert items == [(n, n+n) for n in names]
assert index == [(n, other.get_position(n)) for n in names]
def b(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 10
new_oid = s.new_oid()
assert new_oid == deleted_oid
new_oid = s.new_oid()
assert new_oid == int8_to_str(11)
def get_packer(self):
if (self.pending_records or self.pack_extra is not None
or self.shelf.get_file().is_temporary()
or self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
items = self.gen_oid_record(start_oid=int8_to_str(0))
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
shelf.store(
(name, self.shelf.get_value(name)) for name in self.pack_extra)
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
def a(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oids = [
r['a0']._p_oid, r['a2']._p_oid, r['a7']._p_oid, r['a8']._p_oid
]
del r['a0']
del r['a2']
del r['a7']
del r['a8']
c.commit()
c.pack()
c.abort()
assert c.get(deleted_oids[0])._p_is_ghost()
assert c.get(deleted_oids[1])._p_is_ghost()
raises(KeyError, getattr, c.get(deleted_oids[0]), 'a')
assert len([repr(oid) for oid, record in s.gen_oid_record()]) == 7
c.commit()
c.pack()
new_oid = s.new_oid()
assert new_oid == deleted_oids[-1], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-2], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-3], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == deleted_oids[-4], (new_oid, deleted_oids)
new_oid = s.new_oid()
assert new_oid == int8_to_str(11), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(12), repr(new_oid)
def c(self):
f = File(prefix='shelftest')
name = f.get_name()
f.close()
s = FileStorage(name)
c = Connection(s)
r = c.get_root()
for x in range(10):
r["a%s" % x] = Persistent()
c.commit()
deleted_oid = r['a9']._p_oid
del r['a9']
c.commit()
c.pack()
c.abort()
r.clear()
c.commit()
c.pack()
c.abort()
new_oid = s.new_oid()
assert new_oid == int8_to_str(1), repr(new_oid)
new_oid = s.new_oid()
assert new_oid == int8_to_str(2), repr(new_oid)
def e(self):
f = File()
n1 = int8_to_str(0)
n2 = int8_to_str(1)
s = Shelf(f, items=[(n1, 'record1'), (n2, 'record2')])
