def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(
self.file, repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(self.file,
repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def c(self):
name = mktemp()
name2 = mktemp()
f = File(name)
assert f.tell() == 0
g = File(name2)
g.close()
f.rename(name2)
assert exists(name2)
assert not exists(name)
f.close()
unlink(name2)
def c(self):
name = mktemp()
name2 = mktemp()
f = File(name)
assert f.tell() == 0
g = File(name2)
g.close()
f.rename(name2)
assert exists(name2)
assert not exists(name)
f.close()
unlink(name2)
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def a(self):
f = File()
f.rename(f.get_name())
assert f.is_temporary()
raises(AssertionError, f.rename, f.get_name() + '.renamed')
test_name = f.get_name() + '.test'
assert not exists(test_name)
tmp = open(test_name, 'w+b')
tmp.close()
g = File(test_name)
assert not g.is_temporary()
g.rename(g.get_name() + '.renamed')
assert g.get_name() == test_name + '.renamed'
f.write(as_bytes('abc'))
f.seek(0)
assert len(f) == 3
assert as_bytes('a') == f.read(1)
assert as_bytes('bc') == f.read()
f.close()
assert not exists(f.get_name())
raises(OSError, f.__len__) # tmpfile removed on close
h = File(g.get_name())
g.write(as_bytes('a'))
g.seek(0)
assert g.tell() == 0
g.seek_end()
assert g.tell() == 1
assert g.has_lock
assert not h.has_lock
raises(IOError, h.write, as_bytes('b'))
g.flush()
g.fsync()
g.seek(0)
g.truncate()
g.close()
h.close()
unlink(g.get_name())
def get_packer(self):
if (self.pending_records or
self.pack_extra is not None or
self.shelf.get_file().is_temporary() or
self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
seen = IntSet()
items = self.gen_oid_record(start_oid=int8_to_str(0), seen=seen)
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
for oid in self.pack_extra:
seen.discard(str_to_int8(oid))
for oid in self.pack_extra:
shelf.store(self.gen_oid_record(start_oid=oid, seen=seen))
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
def get_packer(self):
if (self.pending_records or self.pack_extra is not None
or self.shelf.get_file().is_temporary()
or self.shelf.get_file().is_readonly()):
return (x for x in []) # Don't pack.
self.pack_extra = set()
file_path = self.shelf.get_file().get_name()
file = File(file_path + '.pack')
file.truncate() # obtains lock and clears.
assert file.tell() == 0
def packer():
yield "started %s" % datetime.now()
items = self.gen_oid_record(start_oid=int8_to_str(0))
for step in Shelf.generate_shelf(file, items):
yield step
file.flush()
file.fsync()
shelf = Shelf(file)
yield "base written %s" % datetime.now()
# Invalidate oids that have been removed.
for hole in shelf.get_offset_map().gen_holes():
yield hole
oid = int8_to_str(hole)
if self.shelf.get_position(oid) is not None:
assert shelf.get_position(oid) is None
self.invalid.add(oid)
yield "invalidations identified %s" % datetime.now()
shelf.store(
(name, self.shelf.get_value(name)) for name in self.pack_extra)
file.flush()
file.fsync()
if not self.shelf.get_file().is_temporary():
self.shelf.get_file().rename(file_path + '.prepack')
self.shelf.get_file().close()
shelf.get_file().rename(file_path)
self.shelf = shelf
self.pack_extra = None
yield "finished %s" % datetime.now()
return packer()
