예제 #1
0
파일: s3.py 프로젝트: wamdam/backy2
 def _writer(self, id_):
     """ A threaded background writer """
     while True:
         entry = self._write_queue.get()
         if entry is None or self.fatal_error:
             logger.debug("Writer {} finishing.".format(id_))
             break
         uid, data = entry
         time.sleep(self.write_throttling.consume(len(data)))
         t1 = time.time()
         key = self.bucket.new_key(uid)
         try:
             r = key.set_contents_from_string(data)
         except (
                 OSError,
                 boto.exception.BotoServerError,
                 boto.exception.S3ResponseError,
                 ) as e:
             # OSError happens when the S3 host is gone (i.e. network died,
             # host down, ...). boto tries hard to recover, however after
             # several attempts it will give up and raise.
             # BotoServerError happens, when there is no server.
             # S3ResponseError sometimes happens, when the cluster is about
             # to shutdown. Hard to reproduce because the writer must write
             # in exactly this moment.
             # We let the backup job die here fataly.
             self.fatal_error = e
             logger.error('Fatal error, dying: {}'.format(e))
             #exit('Fatal error: {}'.format(e))  # this only raises SystemExit
             os._exit(11)
         t2 = time.time()
         assert r == len(data)
         self._write_queue.task_done()
         logger.debug('Writer {} wrote data async. uid {} in {:.2f}s (Queue size is {})'.format(id_, uid, t2-t1, self._write_queue.qsize()))
예제 #2
0
파일: minio.py 프로젝트: royaflash/backy2
    def _writer(self, id_):
        """ A threaded background writer """
        #bucket = None
        client = None
        while True:
            self.writer_thread_status[id_] = STATUS_QUEUE
            entry = self._write_queue.get()
            self.writer_thread_status[id_] = STATUS_NOTHING
            if entry is None or self.last_exception:
                logger.debug("Writer {} finishing.".format(id_))
                break
            if client is None:
                client = self._get_client()
            uid, enc_envkey, enc_version, enc_nonce, data, callback = entry

            self.writer_thread_status[id_] = STATUS_THROTTLING
            time.sleep(self.write_throttling.consume(len(data)))
            self.writer_thread_status[id_] = STATUS_NOTHING

            try:
                self.writer_thread_status[id_] = STATUS_WRITING
                client.put_object(self.bucket_name, uid, io.BytesIO(data),
                                  len(data))
                #client.upload_fileobj(io.BytesIO(data), Key=uid, Bucket=self._bucket_name)
                self.writer_thread_status[id_] = STATUS_NOTHING
                #if random.random() > 0.9:
                #    raise ValueError("This is a test")
            except Exception as e:
                self.last_exception = e
            else:
                if callback:
                    callback(uid, enc_envkey, enc_version, enc_nonce)
                self._write_queue.task_done()
예제 #3
0
파일: minio.py 프로젝트: royaflash/backy2
 def _reader(self, id_):
     """ A threaded background reader """
     client = None
     while True:
         block = self._read_queue.get()  # contains block
         if block is None or self.last_exception:
             logger.debug("Reader {} finishing.".format(id_))
             break
         if client is None:
             client = self._get_client()
         t1 = time.time()
         try:
             self.reader_thread_status[id_] = STATUS_READING
             data = self.read_raw(block, client)
             self.reader_thread_status[id_] = STATUS_NOTHING
             #except FileNotFoundError:
         except Exception as e:
             self.last_exception = e
         else:
             self._read_data_queue.put((block, data))
             t2 = time.time()
             self._read_queue.task_done()
             logger.debug(
                 'Reader {} read data async. uid {} in {:.2f}s (Queue size is {})'
                 .format(id_, block.uid, t2 - t1, self._read_queue.qsize()))
예제 #4
0
파일: s3.py 프로젝트: m4k3r-org/backy2
    def rm_many(self, uids):
        """ Deletes many uids from the data backend and returns a list
        of uids that couldn't be deleted.
        """
        # "The request contains a list of up to 1000 keys that you want to delete."
        no_deletes = []
        for chunk in chunks(uids, 1000):
            logger.debug("About to delete {} objects from the backend.".format(
                len(chunk)))
            objects = [{'Key': uid} for uid in chunk]
            response = self.bucket.delete_objects(
                Delete={
                    'Objects': objects,
                    #'Quiet': True|False
                },
                RequestPayer='requester',
            )
            # {'Deleted': [{'Key': 'a04ab9bcc0BK6vATCi95Bwb4Djriiy5B'},

            deleted_objects = [d['Key'] for d in response['Deleted']]
            not_found_objects = set(chunk) - set(deleted_objects)
            no_deletes.extend(not_found_objects)
            logger.debug("Deleted {} keys, {} were not found.".format(
                len(deleted_objects), len(not_found_objects)))
        return no_deletes
예제 #5
0
파일: sql.py 프로젝트: mennozon/backy2
    def set_block(self, id, version_uid, block_uid, checksum, size, valid, _commit=True, _upsert=True):
        """ Upsert a block (or insert only when _upsert is False - this is only
        a performance improvement)
        """
        valid = 1 if valid else 0
        block = None
        if _upsert:
            block = self.session.query(Block).filter_by(id=id, version_uid=version_uid).first()

        if block:
            block.uid = block_uid
            block.checksum = checksum
            block.size = size
            block.valid = valid
            block.date = datetime.datetime.now()
        else:
            block = Block(
                id=id,
                version_uid=version_uid,
                uid=block_uid,
                checksum=checksum,
                size=size,
                valid=valid
                )
            self.session.add(block)
        self._flush_block_counter += 1
        if self._flush_block_counter % self.FLUSH_EVERY_N_BLOCKS == 0:
            t1 = time.time()
            self.session.flush()  # saves some ram
            t2 = time.time()
            logger.debug('Flushed meta backend in {:.2f}s'.format(t2-t1))
        if _commit:
            self.session.commit()
예제 #6
0
 def _writer(self, id_=0):
     """ A threaded background writer """
     while True:
         entry = self._write_queue.get()
         if entry is None:
             logger.debug("Writer {} finishing.".format(id_))
             break
         uid, data = entry
         path = os.path.join(self.path, self._path(uid))
         filename = self._filename(uid)
         time.sleep(self.write_throttling.consume(len(data)))
         t1 = time.time()
         try:
             with open(filename, 'wb') as f:
                 r = f.write(data)
         except FileNotFoundError:
             makedirs(path)
             with open(filename, 'wb') as f:
                 r = f.write(data)
         t2 = time.time()
         assert r == len(data)
         self._write_queue.task_done()
         logger.debug(
             'Writer {} wrote data async. uid {} in {:.2f}s (Queue size is {})'
             .format(id_, uid, t2 - t1, self._write_queue.qsize()))
예제 #7
0
 def _writer(self, id_):
     """ A threaded background writer """
     while True:
         entry = self._write_queue.get()
         if entry is None or self.last_exception:
             logger.debug("Writer {} finishing.".format(id_))
             break
         uid, data, callback = entry
         self.writer_thread_status[id_] = STATUS_THROTTLING
         time.sleep(self.write_throttling.consume(len(data)))
         self.writer_thread_status[id_] = STATUS_NOTHING
         t1 = time.time()
         try:
             # storing data to key uid
             self.writer_thread_status[id_] = STATUS_WRITING
             #time.sleep(.1)
             self.writer_thread_status[id_] = STATUS_NOTHING
         except Exception as e:
             self.last_exception = e
         else:
             t2 = time.time()
             # assert r == len(data)
             if callback:
                 callback(uid)
             self._write_queue.task_done()
예제 #8
0
파일: sql.py 프로젝트: royaflash/backy2
 def set_block(self, id, version_uid, block_uid, checksum, size, valid, enc_envkey=b'', enc_version=0, enc_nonce=None, _commit=True):
     """ insert a block
     """
     valid = 1 if valid else 0
     if enc_envkey:
         enc_envkey = binascii.hexlify(enc_envkey).decode('ascii')
     if enc_nonce:
         enc_nonce = binascii.hexlify(enc_nonce).decode('ascii')
     block = Block(
         id=id,
         version_uid=version_uid,
         date=datetime.datetime.utcnow(),  # as func.now creates timezone stamps...
         uid=block_uid,
         checksum=checksum,
         size=size,
         valid=valid,
         enc_envkey=enc_envkey,
         enc_version=enc_version,
         enc_nonce=enc_nonce,
         )
     self.session.add(block)
     self._flush_block_counter += 1
     if self._flush_block_counter % self.FLUSH_EVERY_N_BLOCKS == 0:
         t1 = time.time()
         self.session.flush()  # saves some ram
         t2 = time.time()
         logger.debug('Flushed meta backend in {:.2f}s'.format(t2-t1))
     if _commit:
         self.session.commit()
     return block
예제 #9
0
파일: s3.py 프로젝트: ckauhaus/backy2
 def save(self, data):
     uid = self._uid()
     t1 = time.time()
     key = self.bucket.new_key(uid)
     try:
         r = key.set_contents_from_string(data)
     except (
             OSError,
             boto.exception.BotoServerError,
             boto.exception.S3ResponseError,
             ) as e:
         # OSError happens when the S3 host is gone (i.e. network died,
         # host down, ...). boto tries hard to recover, however after
         # several attempts it will give up and raise.
         # BotoServerError happens, when there is no server.
         # S3ResponseError sometimes happens, when the cluster is about
         # to shutdown. Hard to reproduce because the writer must write
         # in exactly this moment.
         # We let the backup job die here fataly.
         logger.error('Fatal error, dying: {}'.format(e))
         #exit('Fatal error: {}'.format(e))  # this only raises SystemExit
         os._exit(13)
     t2 = time.time()
     assert r == len(data)
     logger.debug('Wrote data uid {} in {:.2f}s'.format(uid, t2-t1))
     return uid
예제 #10
0
파일: nbd.py 프로젝트: kalkin/backy2
 def write(self, version_uid, offset, data):
     """ Copy on write backup writer """
     dataio = BytesIO(data)
     cow = self.cow[version_uid]
     write_list = self._block_list(version_uid, offset, len(data))
     for block, _offset, length in write_list:
         if block is None:
             logger.warning(
                 'Tried to save data beyond device (offset {})'.format(
                     offset))
             continue  # raise? That'd be a write outside the device...
         if block.id in cow:
             # the block is already copied, so update it.
             block_uid = cow[block.id]
             self._update(block_uid, dataio.read(length), _offset)
             logger.debug('Updated cow changed block {} into {})'.format(
                 block.id, block_uid))
         else:
             # read the block from the original, update it and write it back
             write_data = BytesIO(
                 self.backy.data_backend.read_raw(block.uid))
             write_data.seek(_offset)
             write_data.write(dataio.read(length))
             write_data.seek(0)
             block_uid = self._save(write_data.read())
             cow[block.id] = block_uid
             logger.debug('Wrote cow changed block {} into {})'.format(
                 block.id, block_uid))
예제 #11
0
파일: sql.py 프로젝트: royaflash/backy2
 def expire_version(self, version_uid, expire):
     version = self.get_version(version_uid)
     version.expire = expire
     self.session.commit()
     logger.debug('Set expire for version (UID {})'.format(
         version_uid,
         ))
예제 #12
0
    def _writer(self, id_=0):
        """ A threaded background writer """
        while True:
            entry = self._write_queue.get()
            if entry is None or self.last_exception:
                logger.debug("Writer {} finishing.".format(id_))
                break
            uid, enc_envkey, enc_version, enc_nonce, data, callback = entry

            path = os.path.join(self.path, self._path(uid))
            filename = self._filename(uid)
            self.writer_thread_status[id_] = STATUS_THROTTLING
            time.sleep(self.write_throttling.consume(len(data)))
            self.writer_thread_status[id_] = STATUS_NOTHING
            t1 = time.time()
            try:
                try:
                    self.writer_thread_status[id_] = STATUS_WRITING
                    with open(filename, 'wb') as f:
                        r = f.write(data)
                    self.writer_thread_status[id_] = STATUS_NOTHING
                except FileNotFoundError:
                    self.writer_thread_status[id_] = STATUS_WRITING
                    makedirs(path)
                    with open(filename, 'wb') as f:
                        r = f.write(data)
                    self.writer_thread_status[id_] = STATUS_NOTHING
                assert r == len(data)
            except Exception as e:
                self.last_exception = e
            else:
                t2 = time.time()
                if callback:
                    callback(uid, enc_envkey, enc_version, enc_nonce)
                self._write_queue.task_done()
예제 #13
0
파일: sql.py 프로젝트: wamdam/backy2
    def set_block(self, id, version_uid, block_uid, checksum, size, valid, _commit=True, _upsert=True):
        """ Upsert a block (or insert only when _upsert is False - this is only
        a performance improvement)
        """
        valid = 1 if valid else 0
        block = None
        if _upsert:
            block = self.session.query(Block).filter_by(id=id, version_uid=version_uid).first()

        if block:
            block.uid = block_uid
            block.checksum = checksum
            block.size = size
            block.valid = valid
            block.date = datetime.datetime.now()
        else:
            block = Block(
                id=id,
                version_uid=version_uid,
                uid=block_uid,
                checksum=checksum,
                size=size,
                valid=valid
                )
            self.session.add(block)
        self._flush_block_counter += 1
        if self._flush_block_counter % self.FLUSH_EVERY_N_BLOCKS == 0:
            t1 = time.time()
            self.session.flush()  # saves some ram
            t2 = time.time()
            logger.debug('Flushed meta backend in {:.2f}s'.format(t2-t1))
        if _commit:
            self.session.commit()
예제 #14
0
파일: sql.py 프로젝트: mennozon/backy2
 def unprotect_version(self, uid):
     version = self.get_version(uid)
     version.protected = 0
     self.session.commit()
     logger.debug('Marked version unprotected (UID {})'.format(
         uid,
         ))
예제 #15
0
파일: sql.py 프로젝트: wamdam/backy2
 def set_version_valid(self, uid):
     version = self.get_version(uid)
     version.valid = 1
     self.session.commit()
     logger.debug('Marked version valid (UID {})'.format(
         uid,
         ))
예제 #16
0
파일: null.py 프로젝트: royaflash/backy2
    def _reader(self, id_):
        """ self._inqueue contains block_ids to be read.
        self._outqueue contains (block_id, data, data_checksum)
        """
        while True:
            entry = self._inqueue.get()
            if entry is None:
                logger.debug("IO {} finishing.".format(id_))
                self._outqueue.put(None)  # also let the outqueue end
                self._inqueue.task_done()
                break
            block_id, read, metadata = entry
            if not read:
                self._outqueue.put((block_id, None, None, metadata))
            else:
                start_offset = block_id * self.block_size
                end_offset = min(block_id * self.block_size + self.block_size, self._size)
                block_size = end_offset - start_offset
                self.writer_thread_status[id_] = STATUS_READING
                data = generate_block(block_id, block_size)
                self.writer_thread_status[id_] = STATUS_NOTHING

                if not data:
                    raise RuntimeError('EOF reached on source when there should be data.')

                data_checksum = self.hash_function(data).hexdigest()

                self._outqueue.put((block_id, data, data_checksum, metadata))
            self._inqueue.task_done()
예제 #17
0
    def _writer(self, id_):
        """ self._write_queue contains a list of (Block, data) to be written.
        """
        with open(self.io_name, 'rb+') as _write_file:
            while True:
                entry = self._write_queue.get()
                if entry is None:
                    logger.debug("IO writer {} finishing.".format(id_))
                    self._write_queue.task_done()
                    break
                block, data, callback = entry

                offset = block.id * self.block_size

                self.writer_thread_status[id_] = STATUS_SEEKING
                _write_file.seek(offset)
                self.writer_thread_status[id_] = STATUS_WRITING
                written = _write_file.write(data)
                posix_fadvise(_write_file.fileno(), offset, offset + written,
                              os.POSIX_FADV_DONTNEED)
                self.writer_thread_status[id_] = STATUS_NOTHING
                assert written == len(data)
                if callback:
                    callback()

                self._write_queue.task_done()
예제 #18
0
파일: backy.py 프로젝트: ckauhaus/backy2
 def cleanup_fast(self, dt=3600):
     """ Delete unreferenced blob UIDs """
     if not self.locking.lock('backy-cleanup-fast'):
         raise LockError('Another backy cleanup is running.')
     delete_candidates = self.meta_backend.get_delete_candidates(dt=dt)
     try:
         for candidates in grouper(100, delete_candidates):
             # 100 is the number that works here smoothly within about 10-30s
             # per batch. With more than 70s there's a timeout and the job
             # is re-sent. Maybe we must either learn the best amount here
             # or we make this configurable...
             logger.debug('Cleanup: Removing UIDs {}'.format(', '.join(candidates)))
             try:
                 self.data_backend.rm_many(candidates)
             except FileNotFoundError:
                 continue
     except:
         logger.error('Error during cleanup. Reverting metadata changes.')
         self.meta_backend.revert_delete_candidates(delete_candidates)
         self.locking.unlock('backy-cleanup-fast')
         raise
     else:
         self.meta_backend.remove_delete_candidates(delete_candidates)
     logger.info('Cleanup: Removed {} blobs'.format(len(delete_candidates)))
     self.locking.unlock('backy-cleanup-fast')
예제 #19
0
파일: sql.py 프로젝트: mennozon/backy2
 def set_version_valid(self, uid):
     version = self.get_version(uid)
     version.valid = 1
     self.session.commit()
     logger.debug('Marked version valid (UID {})'.format(
         uid,
         ))
예제 #20
0
파일: sql.py 프로젝트: wamdam/backy2
    def get_delete_candidates(self, dt=3600):
        _stat_i = 0
        _stat_remove_from_delete_candidates = 0
        _stat_delete_candidates = 0
        while True:
            delete_candidates = self.session.query(
                DeletedBlock
            ).filter(
                DeletedBlock.time < (inttime() - dt)
            ).limit(250).all()  # http://stackoverflow.com/questions/7389759/memory-efficient-built-in-sqlalchemy-iterator-generator
            if not delete_candidates:
                break

            _remove_from_delete_candidate_uids = set()
            _delete_candidates = set()
            for candidate in delete_candidates:
                _stat_i += 1
                if _stat_i%1000 == 0:
                    logger.info("Cleanup-fast: {} false positives, {} data deletions.".format(
                        _stat_remove_from_delete_candidates,
                        _stat_delete_candidates,
                        ))

                block = self.session.query(
                    Block
                ).filter(
                    Block.uid == candidate.uid
                ).limit(1).scalar()
                if block:
                    _remove_from_delete_candidate_uids.add(candidate.uid)
                    _stat_remove_from_delete_candidates += 1
                else:
                    _delete_candidates.add(candidate.uid)
                    _stat_delete_candidates += 1

            if _remove_from_delete_candidate_uids:
                logger.debug("Cleanup-fast: Removing {} false positive delete candidates".format(len(_remove_from_delete_candidate_uids)))
                self.session.query(
                    DeletedBlock
                ).filter(
                    DeletedBlock.uid.in_(_remove_from_delete_candidate_uids)
                ).delete(synchronize_session=False)

            if _delete_candidates:
                logger.debug("Cleanup-fast: Sending {} delete candidates for final deletion".format(len(_delete_candidates)))
                self.session.query(
                    DeletedBlock
                ).filter(
                    DeletedBlock.uid.in_(_delete_candidates)
                ).delete(synchronize_session=False)
                yield(_delete_candidates)

        logger.info("Cleanup-fast: Cleanup finished. {} false positives, {} data deletions.".format(
            _stat_remove_from_delete_candidates,
            _stat_delete_candidates,
            ))
예제 #21
0
    def get_delete_candidates(self, dt=3600):
        _stat_i = 0
        _stat_remove_from_delete_candidates = 0
        _stat_delete_candidates = 0
        while True:
            delete_candidates = self.session.query(DeletedBlock).filter(
                DeletedBlock.time < (inttime() - dt)
            ).limit(250).all(
            )  # http://stackoverflow.com/questions/7389759/memory-efficient-built-in-sqlalchemy-iterator-generator
            if not delete_candidates:
                break

            _remove_from_delete_candidate_uids = set()
            _delete_candidates = set()
            for candidate in delete_candidates:
                _stat_i += 1
                if _stat_i % 1000 == 0:
                    logger.info(
                        "Cleanup-fast: {} false positives, {} data deletions.".
                        format(
                            _stat_remove_from_delete_candidates,
                            _stat_delete_candidates,
                        ))

                block = self.session.query(Block).filter(
                    Block.uid == candidate.uid).limit(1).scalar()
                if block:
                    _remove_from_delete_candidate_uids.add(candidate.uid)
                    _stat_remove_from_delete_candidates += 1
                else:
                    _delete_candidates.add(candidate.uid)
                    _stat_delete_candidates += 1

            if _remove_from_delete_candidate_uids:
                logger.debug(
                    "Cleanup-fast: Removing {} false positive delete candidates"
                    .format(len(_remove_from_delete_candidate_uids)))
                self.session.query(DeletedBlock).filter(
                    DeletedBlock.uid.in_(_remove_from_delete_candidate_uids)
                ).delete(synchronize_session=False)

            if _delete_candidates:
                logger.debug(
                    "Cleanup-fast: Sending {} delete candidates for final deletion"
                    .format(len(_delete_candidates)))
                self.session.query(DeletedBlock).filter(
                    DeletedBlock.uid.in_(_delete_candidates)).delete(
                        synchronize_session=False)
                yield (_delete_candidates)

        logger.info(
            "Cleanup-fast: Cleanup finished. {} false positives, {} data deletions."
            .format(
                _stat_remove_from_delete_candidates,
                _stat_delete_candidates,
            ))
예제 #22
0
파일: nbd.py 프로젝트: wamdam/backy2
 def read(self, version_uid, offset, length):
     read_list = self._block_list(version_uid, offset, length)
     data = []
     for block, offset, length in read_list:
         logger.debug('Reading block {}:{}:{}'.format(block, offset, length))
         if block is None:
             data.append(b'\0'*length)
         else:
             data.append(self._read(block.uid, offset, length))
     return b''.join(data)
예제 #23
0
    def set_block_enc_envkey(self, block, enc_envkey):
        block.enc_envkey = binascii.hexlify(enc_envkey).decode('ascii')
        self.session.add(block)
        self._flush_block_counter += 1

        if self._flush_block_counter % self.FLUSH_EVERY_N_BLOCKS == 0:
            t1 = time.time()
            self.session.flush()  # saves some ram
            t2 = time.time()
            logger.debug('Flushed meta backend in {:.2f}s'.format(t2 - t1))
        return block
예제 #24
0
파일: nbd.py 프로젝트: kalkin/backy2
 def read(self, version_uid, offset, length):
     read_list = self._block_list(version_uid, offset, length)
     data = []
     for block, offset, length in read_list:
         logger.debug('Reading block {}:{}:{}'.format(
             block, offset, length))
         if block is None:
             data.append(b'\0' * length)
         else:
             data.append(self._read(block.uid, offset, length))
     return b''.join(data)
예제 #25
0
파일: backy.py 프로젝트: kalkin/backy2
    def cleanup_fast(self, dt=3600):
        """ Delete unreferenced blob UIDs """
        if not self.locking.lock('backy-cleanup-fast'):
            raise LockError('Another backy cleanup is running.')

        for uid_list in self.meta_backend.get_delete_candidates(dt):
            logger.debug('Cleanup-fast: Deleting UIDs from data backend: {}'.format(uid_list))
            no_del_uids = []
            no_del_uids = self.data_backend.rm_many(uid_list)
            if no_del_uids:
                logger.info('Cleanup-fast: Unable to delete these UIDs from data backend: {}'.format(uid_list))
        self.locking.unlock('backy-cleanup-fast')
예제 #26
0
파일: backy.py 프로젝트: wamdam/backy2
    def cleanup_fast(self, dt=3600):
        """ Delete unreferenced blob UIDs """
        if not self.locking.lock('backy-cleanup-fast'):
            raise LockError('Another backy cleanup is running.')

        for uid_list in self.meta_backend.get_delete_candidates():
            logger.debug('Cleanup-fast: Deleting UIDs from data backend: {}'.format(uid_list))
            no_del_uids = []
            no_del_uids = self.data_backend.rm_many(uid_list)
            if no_del_uids:
                logger.info('Cleanup-fast: Unable to delete these UIDs from data backend: {}'.format(uid_list))
        self.locking.unlock('backy-cleanup-fast')
예제 #27
0
파일: file.py 프로젝트: ckauhaus/backy2
 def save(self, data):
     uid = self._uid()
     path = os.path.join(self.path, self._path(uid))
     filename = self._filename(uid)
     t1 = time.time()
     try:
         with open(filename, 'wb') as f:
             r = f.write(data)
     except FileNotFoundError:
         makedirs(path)
         with open(filename, 'wb') as f:
             r = f.write(data)
     t2 = time.time()
     assert r == len(data)
     logger.debug('Wrote data uid {} in {:.2f}s'.format(uid, t2-t1))
     return uid
예제 #28
0
 def _reader(self, id_):
     """ A threaded background reader """
     while True:
         block = self._read_queue.get()  # contains block
         if block is None or self.fatal_error:
             logger.debug("Reader {} finishing.".format(id_))
             break
         t1 = time.time()
         try:
             data = self.read_raw(block.uid)
         except FileNotFoundError:
             self._read_data_queue.put((block, None))  # catch this!
         else:
             self._read_data_queue.put((block, data))
             t2 = time.time()
             self._read_queue.task_done()
             logger.debug('Reader {} read data async. uid {} in {:.2f}s (Queue size is {})'.format(id_, block.uid, t2-t1, self._read_queue.qsize()))
예제 #29
0
파일: s3.py 프로젝트: wamdam/backy2
 def _reader(self, id_):
     """ A threaded background reader """
     while True:
         block = self._read_queue.get()  # contains block
         if block is None or self.fatal_error:
             logger.debug("Reader {} finishing.".format(id_))
             break
         t1 = time.time()
         try:
             data = self.read_raw(block.uid)
         except FileNotFoundError:
             self._read_data_queue.put((block, None))  # catch this!
         else:
             self._read_data_queue.put((block, data))
             t2 = time.time()
             self._read_queue.task_done()
             logger.debug('Reader {} read data async. uid {} in {:.2f}s (Queue size is {})'.format(id_, block.uid, t2-t1, self._read_queue.qsize()))
예제 #30
0
파일: null.py 프로젝트: royaflash/backy2
    def _writer(self, id_):
        """ self._write_queue contains a list of (Block, data) to be written.
        """
        while True:
            entry = self._write_queue.get()
            if entry is None:
                logger.debug("IO writer {} finishing.".format(id_))
                break
            block, data, callback = entry

            self.writer_thread_status[id_] = STATUS_WRITING
            # write nothing
            #time.sleep(.1)
            self.writer_thread_status[id_] = STATUS_NOTHING
            if callback:
                callback()

            self._write_queue.task_done()
예제 #31
0
파일: nbd.py 프로젝트: kalkin/backy2
    def fixate(self, cow_version_uid):
        # save blocks into version
        logger.info('Fixating version {} with {} blocks (PLEASE WAIT)'.format(
            cow_version_uid, len(self.cow[cow_version_uid].items())))
        for block_id, block_uid in self.cow[cow_version_uid].items():
            logger.debug('Fixating block {} uid {}'.format(
                block_id, block_uid))
            data = self._read(block_uid)
            checksum = self.hash_function(data).hexdigest()
            if not self.backy.data_backend._SUPPORTS_PARTIAL_WRITES:
                # dump changed data
                new_uid = self.backy.data_backend.save(data, _sync=True)
                logger.debug(
                    'Stored block {} with local uid {} to uid {}'.format(
                        block_id, block_uid, new_uid))
                block_uid = new_uid

            self.backy.meta_backend.set_block(block_id,
                                              cow_version_uid,
                                              block_uid,
                                              checksum,
                                              len(data),
                                              valid=1,
                                              _commit=False)
        self.backy.meta_backend.set_version_valid(cow_version_uid)
        self.backy.meta_backend._commit()
        logger.info('Fixation done. Deleting temporary data (PLEASE WAIT)')
        # TODO: Delete COW blocks and also those from block_cache
        if self.backy.data_backend._SUPPORTS_PARTIAL_WRITES:
            for block_uid in self.block_cache:
                # TODO if this block is in the current version (and in no other?)
                # rm this block from cache
                # rm block uid from self.block_cache
                pass
            for block_id, block_uid in self.cow[cow_version_uid].items():
                # TODO: rm block_uid from cache
                pass
        else:
            # backends that support partial writes will be written to directly.
            # So there's no need to cleanup.
            pass
        del (self.cow[cow_version_uid])
        logger.info('Finished.')
예제 #32
0
    def _writer(self, id_):
        """ self._write_queue contains a list of (Block, data) to be written.
        """
        while True:
            entry = self._write_queue.get()
            if entry is None:
                logger.debug("IO writer {} finishing.".format(id_))
                break
            block, data, callback = entry

            offset = block.id * self.block_size
            self.writer_thread_status[id_] = STATUS_WRITING
            written = self._write_rbd.write(
                data, offset, rados.LIBRADOS_OP_FLAG_FADVISE_DONTNEED)
            assert written == len(data)
            self.writer_thread_status[id_] = STATUS_NOTHING
            if callback:
                callback()

            self._write_queue.task_done()
예제 #33
0
파일: backy.py 프로젝트: ckauhaus/backy2
 def cleanup_full(self, prefix=None):
     """ Delete unreferenced blob UIDs starting with <prefix> """
     # in this mode, we compare all existing uids in data and meta.
     # make sure, no other backy will start
     if not self.locking.lock('backy'):
         self.locking.unlock('backy')
         raise LockError('Other backy instances are running.')
     # make sure, no other backy is running
     if len(find_other_procs(self.process_name)) > 1:
         raise LockError('Other backy instances are running.')
     active_blob_uids = set(self.data_backend.get_all_blob_uids(prefix))
     active_block_uids = set(self.meta_backend.get_all_block_uids(prefix))
     delete_candidates = active_blob_uids.difference(active_block_uids)
     for delete_candidate in delete_candidates:
         logger.debug('Cleanup: Removing UID {}'.format(delete_candidate))
         try:
             self.data_backend.rm(delete_candidate)
         except FileNotFoundError:
             continue
     logger.info('Cleanup: Removed {} blobs'.format(len(delete_candidates)))
     self.locking.unlock('backy')
예제 #34
0
 def cleanup_full(self, prefix=None):
     """ Delete unreferenced blob UIDs starting with <prefix> """
     # in this mode, we compare all existing uids in data and meta.
     # make sure, no other backy will start
     if not self.locking.lock('backy'):
         self.locking.unlock('backy')
         raise LockError('Other backy instances are running.')
     # make sure, no other backy is running
     if len(find_other_procs(self.process_name)) > 1:
         raise LockError('Other backy instances are running.')
     active_blob_uids = set(self.data_backend.get_all_blob_uids(prefix))
     active_block_uids = set(self.meta_backend.get_all_block_uids(prefix))
     delete_candidates = active_blob_uids.difference(active_block_uids)
     for delete_candidate in delete_candidates:
         logger.debug('Cleanup: Removing UID {}'.format(delete_candidate))
         try:
             self.data_backend.rm(delete_candidate)
         except FileNotFoundError:
             continue
     logger.info('Cleanup: Removed {} blobs'.format(len(delete_candidates)))
     self.locking.unlock('backy')
예제 #35
0
파일: file.py 프로젝트: wamdam/backy2
    def _reader(self, id_):
        """ self._inqueue contains Blocks.
        self._outqueue contains (block, data, data_checksum)
        """
        with open(self.io_name, 'rb') as source_file:
            while True:
                block = self._inqueue.get()
                if block is None:
                    logger.debug("IO {} finishing.".format(id_))
                    self._outqueue.put(None)  # also let the outqueue end
                    break
                offset = block.id * self.block_size
                t1 = time.time()
                source_file.seek(offset)
                t2 = time.time()
                data = source_file.read(self.block_size)
                t3 = time.time()
                # throw away cache
                posix_fadvise(source_file.fileno(), offset, offset + self.block_size, os.POSIX_FADV_DONTNEED)
                if not data:
                    raise RuntimeError('EOF reached on source when there should be data.')

                data_checksum = self.hash_function(data).hexdigest()
                if not block.valid:
                    logger.debug('IO {} re-read block (because it was invalid) {} (checksum {})'.format(id_, block.id, data_checksum))
                else:
                    logger.debug('IO {} read block {} (len {}, checksum {}...) in {:.2f}s (seek in {:.2f}s) (Inqueue size: {}, Outqueue size: {})'.format(id_, block.id, len(data), data_checksum[:16], t3-t1, t2-t1, self._inqueue.qsize(), self._outqueue.qsize()))

                self._outqueue.put((block, data, data_checksum))
                self._inqueue.task_done()
예제 #36
0
    def _reader(self, id_):
        """ self._inqueue contains block_ids to be read.
        self._outqueue contains (block_id, data, data_checksum)
        """
        with open(self.io_name, 'rb') as source_file:
            while True:
                entry = self._inqueue.get()
                if entry is None:
                    logger.debug("IO {} finishing.".format(id_))
                    self._outqueue.put(None)  # also let the outqueue end
                    self._inqueue.task_done()
                    break
                block_id, read, metadata = entry
                if not read:
                    self._outqueue.put((block_id, None, None, metadata))
                else:
                    offset = block_id * self.block_size
                    t1 = time.time()
                    self.reader_thread_status[id_] = STATUS_SEEKING
                    source_file.seek(offset)
                    t2 = time.time()
                    self.reader_thread_status[id_] = STATUS_READING
                    data = source_file.read(self.block_size)
                    t3 = time.time()
                    # throw away cache
                    self.reader_thread_status[id_] = STATUS_FADVISE
                    posix_fadvise(source_file.fileno(), offset,
                                  offset + self.block_size,
                                  os.POSIX_FADV_DONTNEED)
                    self.reader_thread_status[id_] = STATUS_NOTHING
                    if not data:
                        raise RuntimeError(
                            'EOF reached on source when there should be data.')

                    data_checksum = self.hash_function(data).hexdigest()

                    self._outqueue.put(
                        (block_id, data, data_checksum, metadata))
                self._inqueue.task_done()
예제 #37
0
    def _writer(self, id_):
        """ A threaded background writer """
        while True:
            entry = self._write_queue.get()
            if entry is None or self.fatal_error:
                logger.debug("Writer {} finishing.".format(id_))
                break
            uid, data = entry
            time.sleep(self.write_throttling.consume(len(data)))
            t1 = time.time()

            try:
                # res = self.conn.create_blob_from_text(
                #     container_name=self.container_name,
                #     blob_name=uid,
                #     text=data,
                #     validate_content=True,
                #     encoding='ascii'
                # )
                string_data = data
                if not isinstance(string_data, bytes):
                    string_data = string_data.encode("utf-8")
                fp = BytesIO(string_data)
                res = self.conn.create_blob_from_bytes(
                    container_name=self.container_name,
                    blob_name=uid,
                    blob=fp.getvalue(),
                    validate_content=True,
                )
            except (OSError, Exception) as e:
                # We let the backup job die here fataly.
                self.fatal_error = e
                logger.error('Fatal error, dying: {}'.format(e))
                print('Error on Write File', e)
                #exit('Fatal error: {}'.format(e))  # this only raises SystemExit
                os._exit(11)
            t2 = time.time()
            self._write_queue.task_done()
            logger.debug('Writer {} wrote data async. uid {} in {:.2f}s (Queue size is {})'.format(id_, uid, t2-t1, self._write_queue.qsize()))
예제 #38
0
파일: file.py 프로젝트: ckauhaus/backy2
 def _writer(self, id_=0):
     """ A threaded background writer """
     while True:
         entry = self._queue.get()
         if entry is None:
             logger.debug("Writer {} finishing.".format(id_))
             break
         uid, data = entry
         path = os.path.join(self.path, self._path(uid))
         filename = self._filename(uid)
         t1 = time.time()
         try:
             with open(filename, 'wb') as f:
                 r = f.write(data)
         except FileNotFoundError:
             makedirs(path)
             with open(filename, 'wb') as f:
                 r = f.write(data)
         t2 = time.time()
         assert r == len(data)
         self._queue.task_done()
         logger.debug('Writer {} wrote data async. uid {} in {:.2f}s (Queue size is {})'.format(id_, uid, t2-t1, self._queue.qsize()))
예제 #39
0
파일: nbd.py 프로젝트: ckauhaus/backy2
 def write(self, version_uid, offset, data):
     """ Copy on write backup writer """
     dataio = BytesIO(data)
     cow = self.cow[version_uid]
     write_list = self._block_list(version_uid, offset, len(data))
     for block, _offset, length in write_list:
         if block is None:
             logger.warning('Tried to save data beyond device (offset {})'.format(offset))
             continue  # raise? That'd be a write outside the device...
         if block.id in cow:
             # the block is already copied, so update it.
             block_uid = cow[block.id]
             self._update(block_uid, dataio.read(length), _offset)
             logger.debug('Updated cow changed block {} into {})'.format(block.id, block_uid))
         else:
             # read the block from the original, update it and write it back
             write_data = BytesIO(self.backy.data_backend.read(block.uid))
             write_data.seek(_offset)
             write_data.write(dataio.read(length))
             write_data.seek(0)
             block_uid = self._save(write_data.read())
             cow[block.id] = block_uid
             logger.debug('Wrote cow changed block {} into {})'.format(block.id, block_uid))
예제 #40
0
파일: nbd.py 프로젝트: ckauhaus/backy2
    def fixate(self, cow_version_uid):
        # save blocks into version
        logger.info('Fixating version {} with {} blocks (PLEASE WAIT)'.format(
            cow_version_uid,
            len(self.cow[cow_version_uid].items())
            ))
        for block_id, block_uid in self.cow[cow_version_uid].items():
            logger.debug('Fixating block {} uid {}'.format(block_id, block_uid))
            data = self._read(block_uid)
            checksum = self.hash_function(data).hexdigest()
            if not self.backy.data_backend._SUPPORTS_PARTIAL_WRITES:
                # dump changed data
                new_uid = self.backy.data_backend.save(data, _sync=True)
                logger.debug('Stored block {} with local uid {} to uid {}'.format(block_id, block_uid, new_uid))
                block_uid = new_uid

            self.backy.meta_backend.set_block(block_id, cow_version_uid, block_uid, checksum, len(data), valid=1, _commit=False)
        self.backy.meta_backend.set_version_valid(cow_version_uid)
        self.backy.meta_backend._commit()
        logger.info('Fixation done. Deleting temporary data (PLEASE WAIT)')
        # TODO: Delete COW blocks and also those from block_cache
        if self.backy.data_backend._SUPPORTS_PARTIAL_WRITES:
            for block_uid in self.block_cache:
                # TODO if this block is in the current version (and in no other?)
                # rm this block from cache
                # rm block uid from self.block_cache
                pass
            for block_id, block_uid in self.cow[cow_version_uid].items():
                # TODO: rm block_uid from cache
                pass
        else:
            # backends that support partial writes will be written to directly.
            # So there's no need to cleanup.
            pass
        del(self.cow[cow_version_uid])
        logger.info('Finished.')
예제 #41
0
파일: file.py 프로젝트: ckauhaus/backy2
    def get(self):
        try:
            block = self._read_list.pop()
        except IndexError:
            raise  # pop from an empty list

        offset = block.id * self.block_size
        t1 = time.time()
        self.source_file.seek(offset)
        t2 = time.time()
        data = self.source_file.read(self.block_size)
        t3 = time.time()
        # throw away cache
        posix_fadvise(self.source_file.fileno(), offset, offset + self.block_size, os.POSIX_FADV_DONTNEED)
        if not data:
            raise RuntimeError('EOF reached on source when there should be data.')

        data_checksum = self.hash_function(data).hexdigest()
        if not block.valid:
            logger.debug('Re-read block (because it was invalid) {} (checksum {})'.format(block.id, data_checksum))
        else:
            logger.debug('Read block {} (len {}, checksum {}...) in {:.2f}s (seek in {:.2f}s)'.format(block.id, len(data), data_checksum[:16], t3-t1, t2-t1))

        return block, data, data_checksum
예제 #42
0
 def _reader(self, id_):
     """ A threaded background reader """
     while True:
         block = self._read_queue.get()  # contains block
         if block is None or self.last_exception:
             logger.debug("Reader {} finishing.".format(id_))
             break
         t1 = time.time()
         try:
             self.reader_thread_status[id_] = STATUS_READING
             data = self.read_raw(block.id, block.size)
             self.reader_thread_status[id_] = STATUS_THROTTLING
         except Exception as e:
             self.last_exception = e
         else:
             time.sleep(self.read_throttling.consume(len(data)))
             self.reader_thread_status[id_] = STATUS_NOTHING
             #time.sleep(.5)
             self._read_data_queue.put((block, data))
             t2 = time.time()
             self._read_queue.task_done()
             logger.debug(
                 'Reader {} read data async. uid {} in {:.2f}s (Queue size is {})'
                 .format(id_, block.uid, t2 - t1, self._read_queue.qsize()))
예제 #43
0
 def _writer(self, id_):
     """ A threaded background writer """
     while True:
         entry = self._write_queue.get()
         if entry is None or self.fatal_error:
             logger.debug("Writer {} finishing.".format(id_))
             break
         uid, data = entry
         time.sleep(self.write_throttling.consume(len(data)))
         t1 = time.time()
         key = self.bucket.new_key(uid)
         try:
             r = key.set_contents_from_string(data)
         except (
                 OSError,
                 boto.exception.BotoServerError,
                 boto.exception.S3ResponseError,
         ) as e:
             # OSError happens when the S3 host is gone (i.e. network died,
             # host down, ...). boto tries hard to recover, however after
             # several attempts it will give up and raise.
             # BotoServerError happens, when there is no server.
             # S3ResponseError sometimes happens, when the cluster is about
             # to shutdown. Hard to reproduce because the writer must write
             # in exactly this moment.
             # We let the backup job die here fataly.
             self.fatal_error = e
             logger.error('Fatal error, dying: {}'.format(e))
             #exit('Fatal error: {}'.format(e))  # this only raises SystemExit
             os._exit(11)
         t2 = time.time()
         assert r == len(data)
         self._write_queue.task_done()
         logger.debug(
             'Writer {} wrote data async. uid {} in {:.2f}s (Queue size is {})'
             .format(id_, uid, t2 - t1, self._write_queue.qsize()))
예제 #44
0
    def _reader(self, id_):
        """ self._inqueue contains block_ids to be read.
        self._outqueue contains (block_id, data, data_checksum)
        """
        ioctx = self.cluster.open_ioctx(self.pool_name)
        with rbd.Image(ioctx,
                       self.image_name,
                       self.snapshot_name,
                       read_only=True) as image:
            while True:
                entry = self._inqueue.get()
                if entry is None:
                    logger.debug("IO {} finishing.".format(id_))
                    self._outqueue.put(None)  # also let the outqueue end
                    self._inqueue.task_done()
                    break
                block_id, read, metadata = entry
                if not read:
                    self._outqueue.put((block_id, None, None, metadata))
                else:
                    offset = block_id * self.block_size
                    t1 = time.time()
                    self.reader_thread_status[id_] = STATUS_READING
                    data = image.read(offset, self.block_size,
                                      rados.LIBRADOS_OP_FLAG_FADVISE_DONTNEED)
                    self.reader_thread_status[id_] = STATUS_NOTHING
                    t2 = time.time()
                    if not data:
                        raise RuntimeError(
                            'EOF reached on source when there should be data.')

                    data_checksum = self.hash_function(data).hexdigest()

                    self._outqueue.put(
                        (block_id, data, data_checksum, metadata))
                self._inqueue.task_done()
예제 #45
0
    def restore(self, version_uid, target, sparse=False, force=False):
        if not self.locking.lock(version_uid):
            raise LockError('Version {} is locked.'.format(version_uid))

        version = self.meta_backend.get_version(version_uid)  # raise if version not exists
        notify(self.process_name, 'Restoring Version {}. Getting blocks.'.format(version_uid))
        blocks = self.meta_backend.get_blocks_by_version(version_uid)

        io = self.get_io_by_source(target)
        io.open_w(target, version.size_bytes, force)

        read_jobs = 0
        for i, block in enumerate(blocks):
            if block.uid:
                self.data_backend.read(block.deref())  # adds a read job
                read_jobs += 1
            elif not sparse:
                io.write(block, b'\0'*block.size)
                logger.debug('Restored sparse block {} successfully ({} bytes).'.format(
                    block.id,
                    block.size,
                    ))
            else:
                logger.debug('Ignored sparse block {}.'.format(
                    block.id,
                    ))
            notify(self.process_name, 'Restoring Version {} to {} PREPARING AND SPARSE BLOCKS ({:.1f}%)'.format(version_uid, target, (i + 1) / len(blocks) * 100))

        done_jobs = 0
        _log_every_jobs = read_jobs // 200 + 1  # about every half percent
        for i in range(read_jobs):
            block, offset, length, data = self.data_backend.read_get()
            assert len(data) == block.size
            data_checksum = self.hash_function(data).hexdigest()
            io.write(block, data)
            if data_checksum != block.checksum:
                logger.error('Checksum mismatch during restore for block '
                    '{} (is: {} should-be: {}, block-valid: {}). Block '
                    'restored is invalid. Continuing.'.format(
                        block.id,
                        data_checksum,
                        block.checksum,
                        block.valid,
                        ))
                self.meta_backend.set_blocks_invalid(block.uid, block.checksum)
            else:
                logger.debug('Restored block {} successfully ({} bytes).'.format(
                    block.id,
                    block.size,
                    ))

            notify(self.process_name, 'Restoring Version {} to {} ({:.1f}%)'.format(version_uid, target, (i + 1) / read_jobs * 100))
            if i % _log_every_jobs == 0 or i + 1 == read_jobs:
                logger.info('Restored {}/{} blocks ({:.1f}%)'.format(i + 1, read_jobs, (i + 1) / read_jobs * 100))
        self.locking.unlock(version_uid)
예제 #46
0
파일: backy.py 프로젝트: ckauhaus/backy2
 def restore(self, version_uid, target, sparse=False):
     if not self.locking.lock(version_uid):
         raise LockError('Version {} is locked.'.format(version_uid))
     version = self.meta_backend.get_version(version_uid)  # raise if version not exists
     blocks = self.meta_backend.get_blocks_by_version(version_uid)
     with open(target, 'wb') as f:
         for block in blocks:
             f.seek(block.id * self.block_size)
             if block.uid:
                 data = self.data_backend.read(block.uid)
                 assert len(data) == block.size
                 data_checksum = self.hash_function(data).hexdigest()
                 written = f.write(data)
                 assert written == len(data)
                 if data_checksum != block.checksum:
                     logger.error('Checksum mismatch during restore for block '
                         '{} (is: {} should-be: {}, block-valid: {}). Block '
                         'restored is invalid. Continuing.'.format(
                             block.id,
                             data_checksum,
                             block.checksum,
                             block.valid,
                             ))
                     self.meta_backend.set_blocks_invalid(block.uid, block.checksum)
                 else:
                     logger.debug('Restored block {} successfully ({} bytes).'.format(
                         block.id,
                         block.size,
                         ))
             elif not sparse:
                 f.write(b'\0'*block.size)
                 logger.debug('Restored sparse block {} successfully ({} bytes).'.format(
                     block.id,
                     block.size,
                     ))
             else:
                 logger.debug('Ignored sparse block {}.'.format(
                     block.id,
                     ))
         if f.tell() != version.size_bytes:
             # write last byte with \0, because this can only happen when
             # the last block was left over in sparse mode.
             last_block = blocks[-1]
             f.seek(last_block.id * self.block_size + last_block.size - 1)
             f.write(b'\0')
     self.locking.unlock(version_uid)
예제 #47
0
파일: backy.py 프로젝트: wamdam/backy2
    def restore(self, version_uid, target, sparse=False, force=False):
        if not self.locking.lock(version_uid):
            raise LockError('Version {} is locked.'.format(version_uid))

        version = self.meta_backend.get_version(version_uid)  # raise if version not exists
        blocks = self.meta_backend.get_blocks_by_version(version_uid)

        io = self.get_io_by_source(target)
        io.open_w(target, version.size_bytes, force)

        for block in blocks:
            if block.uid:
                data = self.data_backend.read(block, sync=True)
                assert len(data) == block.size
                data_checksum = self.hash_function(data).hexdigest()
                io.write(block, data)
                if data_checksum != block.checksum:
                    logger.error('Checksum mismatch during restore for block '
                        '{} (is: {} should-be: {}, block-valid: {}). Block '
                        'restored is invalid. Continuing.'.format(
                            block.id,
                            data_checksum,
                            block.checksum,
                            block.valid,
                            ))
                    self.meta_backend.set_blocks_invalid(block.uid, block.checksum)
                else:
                    logger.debug('Restored block {} successfully ({} bytes).'.format(
                        block.id,
                        block.size,
                        ))
            elif not sparse:
                io.write(block, b'\0'*block.size)
                logger.debug('Restored sparse block {} successfully ({} bytes).'.format(
                    block.id,
                    block.size,
                    ))
            else:
                logger.debug('Ignored sparse block {}.'.format(
                    block.id,
                    ))
        self.locking.unlock(version_uid)
예제 #48
0
파일: rbd.py 프로젝트: zenland/backy2
    def _reader(self, id_):
        """ self._inqueue contains Blocks.
        self._outqueue contains (block, data, data_checksum)
        """
        ioctx = self.cluster.open_ioctx(self.pool_name)
        with rbd.Image(ioctx,
                       self.image_name,
                       self.snapshot_name,
                       read_only=True) as image:
            while True:
                block = self._inqueue.get()
                if block is None:
                    logger.debug("IO {} finishing.".format(id_))
                    self._outqueue.put(None)  # also let the outqueue end
                    break
                offset = block.id * self.block_size
                t1 = time.time()
                data = image.read(offset, self.block_size,
                                  rados.LIBRADOS_OP_FLAG_FADVISE_DONTNEED)
                t2 = time.time()
                # throw away cache
                if not data:
                    raise RuntimeError(
                        'EOF reached on source when there should be data.')

                data_checksum = self.hash_function(data).hexdigest()
                if not block.valid:
                    logger.debug(
                        'IO {} re-read block (because it was invalid) {} (checksum {})'
                        .format(id_, block.id, data_checksum))
                else:
                    logger.debug(
                        'IO {} read block {} (checksum {}...) in {:.2f}s) '
                        '(Inqueue size: {}, Outqueue size: {})'.format(
                            id_, block.id, data_checksum[:16], t2 - t1,
                            self._inqueue.qsize(), self._outqueue.qsize()))

                self._outqueue.put((block, data, data_checksum))
                self._inqueue.task_done()
예제 #49
0
파일: rbd.py 프로젝트: ckauhaus/backy2
    def _reader(self, id_):
        """ self._inqueue contains Blocks.
        self._outqueue contains (block, data, data_checksum)
        """
        ioctx = self.cluster.open_ioctx(self.pool_name)
        with rbd.Image(ioctx, self.image_name, self.snapshot_name, read_only=True) as image:
            while True:
                block = self._inqueue.get()
                if block is None:
                    logger.debug("Reader {} finishing.".format(id_))
                    self._outqueue.put(None)  # also let the outqueue end
                    break
                offset = block.id * self.block_size
                t1 = time.time()
                data = image.read(offset, self.block_size, rados.LIBRADOS_OP_FLAG_FADVISE_DONTNEED)
                t2 = time.time()
                # throw away cache
                if not data:
                    raise RuntimeError('EOF reached on source when there should be data.')

                data_checksum = self.hash_function(data).hexdigest()
                if not block.valid:
                    logger.debug('Reader {} re-read block (because it was invalid) {} (checksum {})'.format(id_, block.id, data_checksum))
                else:
                    logger.debug('Reader {} read block {} (checksum {}...) in {:.2f}s) '
                        '(Inqueue size: {}, Outqueue size: {})'.format(
                            id_,
                            block.id,
                            data_checksum[:16],
                            t2-t1,
                            self._inqueue.qsize(),
                            self._outqueue.qsize()
                            ))

                self._outqueue.put((block, data, data_checksum))
                self._inqueue.task_done()
예제 #50
0
파일: backy.py 프로젝트: ckauhaus/backy2
    def scrub(self, version_uid, source=None, percentile=100):
        """ Returns a boolean (state). If False, there were errors, if True
        all was ok
        """
        if not self.locking.lock(version_uid):
            raise LockError('Version {} is locked.'.format(version_uid))
        self.meta_backend.get_version(version_uid)  # raise if version not exists
        blocks = self.meta_backend.get_blocks_by_version(version_uid)
        if source:
            self.reader.open(source)

        state = True
        for block in blocks:
            if block.uid:
                if percentile < 100 and random.randint(1, 100) > percentile:
                    logger.debug('Scrub of block {} (UID {}) skipped (percentile is {}).'.format(
                        block.id,
                        block.uid,
                        percentile,
                        ))
                    continue
                try:
                    data = self.data_backend.read(block.uid)
                except FileNotFoundError as e:
                    logger.error('Blob not found: {}'.format(str(e)))
                    self.meta_backend.set_blocks_invalid(block.uid, block.checksum)
                    state = False
                    continue
                if len(data) != block.size:
                    logger.error('Blob has wrong size: {} is: {} should be: {}'.format(
                        block.uid,
                        len(data),
                        block.size,
                        ))
                    self.meta_backend.set_blocks_invalid(block.uid, block.checksum)
                    state = False
                    continue
                data_checksum = self.hash_function(data).hexdigest()
                if data_checksum != block.checksum:
                    logger.error('Checksum mismatch during scrub for block '
                        '{} (UID {}) (is: {} should-be: {}).'.format(
                            block.id,
                            block.uid,
                            data_checksum,
                            block.checksum,
                            ))
                    self.meta_backend.set_blocks_invalid(block.uid, block.checksum)
                    state = False
                    continue
                else:
                    if source:
                        source_data = self.reader.read(block, sync=True)
                        if source_data != data:
                            logger.error('Source data has changed for block {} '
                                '(UID {}) (is: {} should-be: {}). NOT setting '
                                'this block invalid, because the source looks '
                                'wrong.'.format(
                                    block.id,
                                    block.uid,
                                    self.hash_function(source_data).hexdigest(),
                                    data_checksum,
                                    ))
                            state = False
                            # We are not setting the block invalid here because
                            # when the block is there AND the checksum is good,
                            # then the source is invalid.
                    logger.debug('Scrub of block {} (UID {}) ok.'.format(
                        block.id,
                        block.uid,
                        ))
            else:
                logger.debug('Scrub of block {} (UID {}) skipped (sparse).'.format(
                    block.id,
                    block.uid,
                    ))
        if state == True:
            self.meta_backend.set_version_valid(version_uid)
        else:
            # version is set invalid by set_blocks_invalid.
            logger.error('Marked version invalid because it has errors: {}'.format(version_uid))
        if source:
            self.reader.close()  # wait for all readers

        self.locking.unlock(version_uid)
        return state
예제 #51
0
파일: backy.py 프로젝트: ckauhaus/backy2
    def backup(self, name, source, hints, from_version):
        """ Create a backup from source.
        If hints are given, they must be tuples of (offset, length, exists)
        where offset and length are integers and exists is a boolean. Then, only
        data within hints will be backed up.
        Otherwise, the backup reads source and looks if checksums match with
        the target.
        """
        stats = {
                'version_size_bytes': 0,
                'version_size_blocks': 0,
                'bytes_read': 0,
                'blocks_read': 0,
                'bytes_written': 0,
                'blocks_written': 0,
                'bytes_found_dedup': 0,
                'blocks_found_dedup': 0,
                'bytes_sparse': 0,
                'blocks_sparse': 0,
                'start_time': time.time(),
            }
        self.reader.open(source)
        source_size = self.reader.size()

        size = math.ceil(source_size / self.block_size)
        stats['version_size_bytes'] = source_size
        stats['version_size_blocks'] = size

        # Sanity check: check hints for validity, i.e. too high offsets, ...
        if hints:
            max_offset = max([h[0]+h[1] for h in hints])
            if max_offset > source_size:
                raise ValueError('Hints have higher offsets than source file.')

        if hints:
            sparse_blocks = blocks_from_hints([hint for hint in hints if not hint[2]], self.block_size)
            read_blocks = blocks_from_hints([hint for hint in hints if hint[2]], self.block_size)
        else:
            sparse_blocks = []
            read_blocks = range(size)
        sparse_blocks = set(sparse_blocks)
        read_blocks = set(read_blocks)

        try:
            version_uid = self._prepare_version(name, source_size, from_version)
        except RuntimeError as e:
            logger.error(str(e))
            logger.error('Backy exiting.')
            # TODO: Don't exit here, exit in Commands
            exit(4)
        except LockError as e:
            logger.error(str(e))
            logger.error('Backy exiting.')
            # TODO: Don't exit here, exit in Commands
            exit(99)
        if not self.locking.lock(version_uid):
            logger.error('Version {} is locked.'.format(version_uid))
            logger.error('Backy exiting.')
            # TODO: Don't exit here, exit in Commands
            exit(99)

        blocks = self.meta_backend.get_blocks_by_version(version_uid)

        if from_version and hints:
            # SANITY CHECK:
            # Check some blocks outside of hints if they are the same in the
            # from_version backup and in the current backup. If they
            # don't, either hints are wrong (e.g. from a wrong snapshot diff)
            # or source doesn't match. In any case, the resulting backup won't
            # be good.
            logger.info('Starting sanity check with 1% of the blocks. Reading...')
            ignore_blocks = list(set(range(size)) - read_blocks - sparse_blocks)
            random.shuffle(ignore_blocks)
            num_check_blocks = max(10, len(ignore_blocks) // 100)  # 1%, but at least 10
            check_block_ids = ignore_blocks[:num_check_blocks]
            num_reading = 0
            for block in blocks:
                if block.id in check_block_ids and block.uid:  # no uid = sparse block in backup. Can't check.
                    self.reader.read(block)
                    num_reading += 1
            for i in range(num_reading):
                # this is source file data
                source_block, source_data, source_data_checksum = self.reader.get()
                # check metadata checksum with the newly read one
                if source_block.checksum != source_data_checksum:
                    logger.error("Source and backup don't match in regions outside of the hints.")
                    logger.error("Looks like the hints don't match or the source is different.")
                    logger.error("Found wrong source data at block {}: offset {} with max. length {}".format(
                        source_block.id,
                        source_block.id * self.block_size,
                        self.block_size
                        ))
                    # remove version
                    self.meta_backend.rm_version(version_uid)
                    sys.exit(5)
            logger.info('Finished sanity check. Checked {} blocks.'.format(num_reading))

        read_jobs = 0
        for block in blocks:
            if block.id in read_blocks or not block.valid:
                self.reader.read(block)  # adds a read job.
                read_jobs += 1
            elif block.id in sparse_blocks:
                # This "elif" is very important. Because if the block is in read_blocks
                # AND sparse_blocks, it *must* be read.
                self.meta_backend.set_block(block.id, version_uid, None, None, block.size, valid=1, _commit=False)
                stats['blocks_sparse'] += 1
                stats['bytes_sparse'] += block.size
                logger.debug('Skipping block (sparse) {}'.format(block.id))
            else:
                #self.meta_backend.set_block(block.id, version_uid, block.uid, block.checksum, block.size, valid=1, _commit=False)
                logger.debug('Keeping block {}'.format(block.id))

        # now use the readers and write
        done_jobs = 0
        for i in range(read_jobs):
            block, data, data_checksum = self.reader.get()

            stats['blocks_read'] += 1
            stats['bytes_read'] += len(data)

            # dedup
            existing_block = self.meta_backend.get_block_by_checksum(data_checksum)
            if existing_block and existing_block.size == len(data):
                self.meta_backend.set_block(block.id, version_uid, existing_block.uid, data_checksum, len(data), valid=1, _commit=False)
                stats['blocks_found_dedup'] += 1
                stats['bytes_found_dedup'] += len(data)
                logger.debug('Found existing block for id {} with uid {})'.format
                        (block.id, existing_block.uid))
            else:
                block_uid = self.data_backend.save(data)
                self.meta_backend.set_block(block.id, version_uid, block_uid, data_checksum, len(data), valid=1, _commit=False)
                stats['blocks_written'] += 1
                stats['bytes_written'] += len(data)
                logger.debug('Wrote block {} (checksum {}...)'.format(block.id, data_checksum[:16]))
            done_jobs += 1

        self.reader.close()  # wait for all readers
        self.data_backend.close()  # wait for all writers
        if read_jobs != done_jobs:
            logger.error('backy broke somewhere. Backup is invalid.')
            sys.exit(3)

        self.meta_backend.set_version_valid(version_uid)
        self.meta_backend.set_stats(
            version_uid=version_uid,
            version_name=name,
            version_size_bytes=stats['version_size_bytes'],
            version_size_blocks=stats['version_size_blocks'],
            bytes_read=stats['bytes_read'],
            blocks_read=stats['blocks_read'],
            bytes_written=stats['bytes_written'],
            blocks_written=stats['blocks_written'],
            bytes_found_dedup=stats['bytes_found_dedup'],
            blocks_found_dedup=stats['blocks_found_dedup'],
            bytes_sparse=stats['bytes_sparse'],
            blocks_sparse=stats['blocks_sparse'],
            duration_seconds=int(time.time() - stats['start_time']),
            )
        logger.info('New version: {}'.format(version_uid))
        self.locking.unlock(version_uid)
        return version_uid
예제 #52
0
파일: backy.py 프로젝트: ckauhaus/backy2
def main():
    parser = argparse.ArgumentParser(
        description='Backup and restore for block devices.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument(
        '-v', '--verbose', action='store_true', help='verbose output')
    parser.add_argument(
        '-m', '--machine-output', action='store_true', default=False)
    parser.add_argument(
        '-V', '--version', action='store_true', help='Show version')

    subparsers = parser.add_subparsers()

    # BACKUP
    p = subparsers.add_parser(
        'backup',
        help="Perform a backup.")
    p.add_argument(
        'source',
        help='Source file')
    p.add_argument(
        'name',
        help='Backup name')
    p.add_argument('-r', '--rbd', default=None, help='Hints as rbd json format')
    p.add_argument('-f', '--from-version', default=None, help='Use this version-uid as base')
    p.set_defaults(func='backup')

    # RESTORE
    p = subparsers.add_parser(
        'restore',
        help="Restore a given backup with level to a given target.")
    p.add_argument('-s', '--sparse', action='store_true', help='Write restore file sparse (does not work with legacy devices)')
    p.add_argument('version_uid')
    p.add_argument('target')
    p.set_defaults(func='restore')

    # RM
    p = subparsers.add_parser(
        'rm',
        help="Remove a given backup version. This will only remove meta data and you will have to cleanup after this.")
    p.add_argument('version_uid')
    p.set_defaults(func='rm')

    # SCRUB
    p = subparsers.add_parser(
        'scrub',
        help="Scrub a given backup and check for consistency.")
    p.add_argument('-s', '--source', default=None,
        help="Source, optional. If given, check if source matches backup in addition to checksum tests.")
    p.add_argument('-p', '--percentile', default=100,
        help="Only check PERCENTILE percent of the blocks (value 0..100). Default: 100")
    p.add_argument('version_uid')
    p.set_defaults(func='scrub')

    # Export
    p = subparsers.add_parser(
        'export',
        help="Export the metadata of a backup uid into a file.")
    p.add_argument('version_uid')
    p.add_argument('filename', help="Export into this filename ('-' is for stdout)")
    p.set_defaults(func='export')

    # Import
    p = subparsers.add_parser(
        'import',
        help="Import the metadata of a backup from a file.")
    p.add_argument('filename', help="Read from this file ('-' is for stdin)")
    p.set_defaults(func='import_')

    # CLEANUP
    p = subparsers.add_parser(
        'cleanup',
        help="Clean unreferenced blobs.")
    p.add_argument(
        '-f', '--full', action='store_true', default=False,
        help='Do a full cleanup. This will read the full metadata from the data backend (i.e. backup storage) '
             'and compare it to the metadata in the meta backend. Unused data will then be deleted. '
             'This is a slow, but complete process. A full cleanup must not be run parallel to ANY other backy '
             'jobs.')
    p.add_argument(
        '-p', '--prefix', default=None,
        help='If you perform a full cleanup, you may add --prefix to only cleanup block uids starting '
             'with this prefix. This is for iterative cleanups. Example: '
             'cleanup --full --prefix=a')
    p.set_defaults(func='cleanup')

    # LS
    p = subparsers.add_parser(
        'ls',
        help="List existing backups.")
    p.add_argument('version_uid', nargs='?', default=None, help='Show verbose blocks for this version')
    p.set_defaults(func='ls')

    # STATS
    p = subparsers.add_parser(
        'stats',
        help="Show statistics")
    p.add_argument('version_uid', nargs='?', default=None, help='Show statistics for this version')
    p.add_argument('-l', '--limit', default=None,
            help="Limit output to this number (default: unlimited)")
    p.set_defaults(func='stats')

    # diff-meta
    p = subparsers.add_parser(
        'diff-meta',
        help="Output a diff between two versions")
    p.add_argument('version_uid1', help='Left version')
    p.add_argument('version_uid2', help='Right version')
    p.set_defaults(func='diff_meta')

    # NBD
    p = subparsers.add_parser(
        'nbd',
        help="Start an nbd server")
    p.add_argument('version_uid', nargs='?', default=None, help='Start an nbd server for this version')
    p.add_argument('-a', '--bind-address', default='127.0.0.1',
            help="Bind to this ip address (default: 127.0.0.1)")
    p.add_argument('-p', '--bind-port', default=10809,
            help="Bind to this port (default: 10809)")
    p.add_argument(
        '-r', '--read-only', action='store_true', default=False,
        help='Read only if set, otherwise a copy on write backup is created.')
    p.set_defaults(func='nbd')

    args = parser.parse_args()

    if args.version:
        print(__version__)
        sys.exit(0)

    if not hasattr(args, 'func'):
        parser.print_usage()
        sys.exit(1)

    if args.verbose:
        console_level = logging.DEBUG
    #elif args.func == 'scheduler':
        #console_level = logging.INFO
    else:
        console_level = logging.INFO

    Config = partial(_Config, conf_name='backy')
    config = Config(section='DEFAULTS')
    init_logging(config.get('logfile'), console_level)

    commands = Commands(args.machine_output, Config)
    func = getattr(commands, args.func)

    # Pass over to function
    func_args = dict(args._get_kwargs())
    del func_args['func']
    del func_args['verbose']
    del func_args['version']
    del func_args['machine_output']

    try:
        logger.debug('backup.{0}(**{1!r})'.format(args.func, func_args))
        func(**func_args)
        logger.info('Backy complete.\n')
        sys.exit(0)
    except Exception as e:
        logger.error('Unexpected exception')
        logger.exception(e)
        logger.info('Backy failed.\n')
        sys.exit(100)