Beispiel #1
0
 def decode_db(self, fp, append_queue=False):
     """Decodes the encoded database, as found in the file-like
     `fp` object.  Overwrites colletion_id and the database"""
     (length,) = int_encoding.unpack(fp.read(4))
     collection_id = fp.read(length)
     col_filename = os.path.join(self.dir, 'new_collection_id.txt')
     col_fp = open_create(col_filename)
     try:
         col_fp.write(collection_id)
     finally:
         col_fp.close()
     (length,) = int_encoding.unpack(fp.read(4))
     collection_secret = fp.read(length)
     col_filename = os.path.join(self.dir, 'new_collection_secret.txt')
     col_fp = open_create(col_filename)
     try:
         col_fp.write(collection_secret)
     finally:
         col_fp.close()
     (length,) = int_encoding.unpack(fp.read(4))
     db_name = os.path.join(self.dir, 'new_database')
     queue_filename = os.path.join(self.dir, 'queue')
     queue_index_fp = None
     if os.path.exists(queue_filename + '.index'):
         queue_index_fp = open(queue_filename + '.index', 'rb')
         lock_file(queue_index_fp, LOCK_EX, 0, 0, os.SEEK_SET)
     new_fp = open_create(db_name + '.index')
     try:
         self._copy_chunked(fp, new_fp, length)
         if queue_index_fp is not None:
             new_fp.write(queue_index_fp.read())
     finally:
         new_fp.close()
     (length,) = int_encoding.unpack(fp.read(4))
     new_fp = open_create(db_name)
     try:
         self._copy_chunked(fp, new_fp, length)
         if append_queue and os.path.exists(queue_filename):
             with open(queue_filename, 'rb') as copy_fp:
                 ## FIXME: chunk
                 new_fp.write(copy_fp.read())
     finally:
         new_fp.close()
     for name in 'new_collection_id.txt', 'new_collection_secret.txt', 'new_database.index', 'new_database':
         os.rename(os.path.join(self.dir, name),
                   os.path.join(self.dir, name[4:]))
     if append_queue:
         ## FIXME: also not atomic:
         for name in 'queue', 'queue.index':
             name = os.path.join(self.dir, name)
             if os.path.exists(name):
                 os.unlink(name)
     if queue_index_fp is not None:
         lock_file(queue_index_fp, LOCK_UN, 0, 0, os.SEEK_SET)
Beispiel #2
0
 def overwrite(self, data_filename, index_filename):
     """Overwrites this database with the given files"""
     lock_file(self.index_fp, LOCK_EX)
     self.index_fp.seek(0)
     # First make sure no one can do anything:
     self.index_fp.truncate()
     self.data_fp.close()
     os.unlink(self.data_filename)
     os.rename(data_filename, self.data_filename)
     self.data_fp = open(self.data_filename, 'r+b')
     fp = open(index_filename, 'rb')
     shutil.copyfile(fp, self.index_fp)
     fp.close()
     os.unlink(index_filename)
Beispiel #3
0
 def extend(self, datas, expect_latest=None):
     """Appends the data to the database, returning the integer
     counter for the first item in the data
     """
     lock_file(self.index_fp, LOCK_EX)
     try:
         count = self._read_last_count()
         if expect_latest is not None and count > expect_latest:
             raise ExpectationFailed
         count += 1
         first_datas = count
         self.index_fp.seek(0, os.SEEK_END)
         self.data_fp.seek(0, os.SEEK_END)
         pos = self.data_fp.tell()
         for data in datas:
             assert isinstance(data, str)
             length = len(data)
             self.data_fp.write(data)
             self.index_fp.write(triple_encoding.pack(length, pos, count))
             count += 1
             pos += length
         return first_datas
     finally:
         lock_file(self.index_fp, LOCK_UN)
Beispiel #4
0
def lock_complete(fp):
    lock_file(fp, LOCK_EX, 0, 0, os.SEEK_SET)
    yield
    lock_file(fp, LOCK_UN, 0, 0, os.SEEK_SET)
Beispiel #5
0
def lock_append(fp):
    lock_file(fp, LOCK_EX, 0, 0, os.SEEK_END)
    yield
    lock_file(fp, LOCK_UN, 0, 0, os.SEEK_END)
Beispiel #6
0
        except IOError, e:
            if e.errno != 2:
                raise
            ## File does not exist
            try:
                fd = os.open(index_filename,
                             os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_EXLOCK)
            except IOError, e:
                if e.errno != 17:
                    raise
                ## File was created while we were trying to create it, which is fine
                self.index_fp = open(index_filename, 'r+b')
            else:
                self.index_fp = os.fdopen(fd, 'r+b')
                self.index_fp.write(triple_encoding.pack(0, 0, 0))
                lock_file(self.index_fp, LOCK_UN, 0, 0, os.SEEK_SET)
        fd = os.open(data_filename, os.O_RDWR | os.O_CREAT)
        self.data_fp = os.fdopen(fd, 'r+b')

    def _read_last_count(self):
        """Reads the counter of the last item appended"""
        self.index_fp.seek(-4, os.SEEK_END)
        if self.index_fp.tell() % 4:
            raise Exception("Misaligned length of index file %s" %
                            self.index_filename)
        chunk = self.index_fp.read(4)
        if not chunk:
            # The file has been truncated, there's not even the 0/0/0 record
            raise TruncatedFile()
        return int_encoding.unpack(chunk)[0]
Beispiel #7
0
def lock_complete(fp):
    lock_file(fp, LOCK_EX, 0, 0, os.SEEK_SET)
    yield
    lock_file(fp, LOCK_UN, 0, 0, os.SEEK_SET)
Beispiel #8
0
def lock_append(fp):
    lock_file(fp, LOCK_EX, 0, 0, os.SEEK_END)
    yield
    lock_file(fp, LOCK_UN, 0, 0, os.SEEK_END)
Beispiel #9
0
            self.index_fp = open(index_filename, 'r+b')
        except IOError, e:
            if e.errno != 2:
                raise
            ## File does not exist
            try:
                fd = os.open(index_filename, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_EXLOCK)
            except IOError, e:
                if e.errno != 17:
                    raise
                ## File was created while we were trying to create it, which is fine
                self.index_fp = open(index_filename, 'r+b')
            else:
                self.index_fp = os.fdopen(fd, 'r+b')
                self.index_fp.write(triple_encoding.pack(0, 0, 0))
                lock_file(self.index_fp, LOCK_UN, 0, 0, os.SEEK_SET)
        fd = os.open(data_filename, os.O_RDWR | os.O_CREAT)
        self.data_fp = os.fdopen(fd, 'r+b')

    def _read_last_count(self):
        """Reads the counter of the last item appended"""
        self.index_fp.seek(-4, os.SEEK_END)
        if self.index_fp.tell() % 4:
            raise Exception("Misaligned length of index file %s" % self.index_filename)
        chunk = self.index_fp.read(4)
        if not chunk:
            # The file has been truncated, there's not even the 0/0/0 record
            raise TruncatedFile()
        return int_encoding.unpack(chunk)[0]

    def _seek_index(self, seek_count):