def decode_db(self, fp, append_queue=False):
"""Decodes the encoded database, as found in the file-like
`fp` object. Overwrites colletion_id and the database"""
(length,) = int_encoding.unpack(fp.read(4))
collection_id = fp.read(length)
col_filename = os.path.join(self.dir, 'new_collection_id.txt')
col_fp = open_create(col_filename)
try:
col_fp.write(collection_id)
finally:
col_fp.close()
(length,) = int_encoding.unpack(fp.read(4))
collection_secret = fp.read(length)
col_filename = os.path.join(self.dir, 'new_collection_secret.txt')
col_fp = open_create(col_filename)
try:
col_fp.write(collection_secret)
finally:
col_fp.close()
(length,) = int_encoding.unpack(fp.read(4))
db_name = os.path.join(self.dir, 'new_database')
queue_filename = os.path.join(self.dir, 'queue')
queue_index_fp = None
if os.path.exists(queue_filename + '.index'):
queue_index_fp = open(queue_filename + '.index', 'rb')
lock_file(queue_index_fp, LOCK_EX, 0, 0, os.SEEK_SET)
new_fp = open_create(db_name + '.index')
try:
self._copy_chunked(fp, new_fp, length)
if queue_index_fp is not None:
new_fp.write(queue_index_fp.read())
finally:
new_fp.close()
(length,) = int_encoding.unpack(fp.read(4))
new_fp = open_create(db_name)
try:
self._copy_chunked(fp, new_fp, length)
if append_queue and os.path.exists(queue_filename):
with open(queue_filename, 'rb') as copy_fp:
## FIXME: chunk
new_fp.write(copy_fp.read())
finally:
new_fp.close()
for name in 'new_collection_id.txt', 'new_collection_secret.txt', 'new_database.index', 'new_database':
os.rename(os.path.join(self.dir, name),
os.path.join(self.dir, name[4:]))
if append_queue:
## FIXME: also not atomic:
for name in 'queue', 'queue.index':
name = os.path.join(self.dir, name)
if os.path.exists(name):
os.unlink(name)
if queue_index_fp is not None:
lock_file(queue_index_fp, LOCK_UN, 0, 0, os.SEEK_SET)
def overwrite(self, data_filename, index_filename):
"""Overwrites this database with the given files"""
lock_file(self.index_fp, LOCK_EX)
self.index_fp.seek(0)
# First make sure no one can do anything:
self.index_fp.truncate()
self.data_fp.close()
os.unlink(self.data_filename)
os.rename(data_filename, self.data_filename)
self.data_fp = open(self.data_filename, 'r+b')
fp = open(index_filename, 'rb')
shutil.copyfile(fp, self.index_fp)
fp.close()
os.unlink(index_filename)
def extend(self, datas, expect_latest=None):
"""Appends the data to the database, returning the integer
counter for the first item in the data
"""
lock_file(self.index_fp, LOCK_EX)
try:
count = self._read_last_count()
if expect_latest is not None and count > expect_latest:
raise ExpectationFailed
count += 1
first_datas = count
self.index_fp.seek(0, os.SEEK_END)
self.data_fp.seek(0, os.SEEK_END)
pos = self.data_fp.tell()
for data in datas:
assert isinstance(data, str)
length = len(data)
self.data_fp.write(data)
self.index_fp.write(triple_encoding.pack(length, pos, count))
count += 1
pos += length
return first_datas
finally:
lock_file(self.index_fp, LOCK_UN)
def lock_complete(fp):
lock_file(fp, LOCK_EX, 0, 0, os.SEEK_SET)
yield
lock_file(fp, LOCK_UN, 0, 0, os.SEEK_SET)
def lock_append(fp):
lock_file(fp, LOCK_EX, 0, 0, os.SEEK_END)
yield
lock_file(fp, LOCK_UN, 0, 0, os.SEEK_END)
except IOError, e:
if e.errno != 2:
raise
## File does not exist
try:
fd = os.open(index_filename,
os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_EXLOCK)
except IOError, e:
if e.errno != 17:
raise
## File was created while we were trying to create it, which is fine
self.index_fp = open(index_filename, 'r+b')
else:
self.index_fp = os.fdopen(fd, 'r+b')
self.index_fp.write(triple_encoding.pack(0, 0, 0))
lock_file(self.index_fp, LOCK_UN, 0, 0, os.SEEK_SET)
fd = os.open(data_filename, os.O_RDWR | os.O_CREAT)
self.data_fp = os.fdopen(fd, 'r+b')
def _read_last_count(self):
"""Reads the counter of the last item appended"""
self.index_fp.seek(-4, os.SEEK_END)
if self.index_fp.tell() % 4:
raise Exception("Misaligned length of index file %s" %
self.index_filename)
chunk = self.index_fp.read(4)
if not chunk:
# The file has been truncated, there's not even the 0/0/0 record
raise TruncatedFile()
return int_encoding.unpack(chunk)[0]
def lock_complete(fp):
lock_file(fp, LOCK_EX, 0, 0, os.SEEK_SET)
yield
lock_file(fp, LOCK_UN, 0, 0, os.SEEK_SET)
def lock_append(fp):
lock_file(fp, LOCK_EX, 0, 0, os.SEEK_END)
yield
lock_file(fp, LOCK_UN, 0, 0, os.SEEK_END)
self.index_fp = open(index_filename, 'r+b')
except IOError, e:
if e.errno != 2:
raise
## File does not exist
try:
fd = os.open(index_filename, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_EXLOCK)
except IOError, e:
if e.errno != 17:
raise
## File was created while we were trying to create it, which is fine
self.index_fp = open(index_filename, 'r+b')
else:
self.index_fp = os.fdopen(fd, 'r+b')
self.index_fp.write(triple_encoding.pack(0, 0, 0))
lock_file(self.index_fp, LOCK_UN, 0, 0, os.SEEK_SET)
fd = os.open(data_filename, os.O_RDWR | os.O_CREAT)
self.data_fp = os.fdopen(fd, 'r+b')
def _read_last_count(self):
"""Reads the counter of the last item appended"""
self.index_fp.seek(-4, os.SEEK_END)
if self.index_fp.tell() % 4:
raise Exception("Misaligned length of index file %s" % self.index_filename)
chunk = self.index_fp.read(4)
if not chunk:
# The file has been truncated, there's not even the 0/0/0 record
raise TruncatedFile()
return int_encoding.unpack(chunk)[0]
def _seek_index(self, seek_count):
