Exemple #1
0
    def write_chunk(self, chunk, is_dummy=False):
        """
        Given a new chunk, write it to the storage.

        @type chunk: AbstractChunkWithContents

        @type is_dummy: bool
        """
        # For clarity only
        super(ChunkStorageFS, self).write_chunk(chunk, is_dummy)
        file_path = self.__get_chunk_file_path(chunk.uuid, is_dummy=is_dummy)
        logger.debug('Chunk body %s being written to %s',
                     chunk.uuid, file_path)

        with self.__chunk_op_lock:
            if os.path.exists(file_path):
                logger.warning('Attempting to write the chunk %s, '
                                   'but it exists already!',
                               chunk.uuid)
            else:
                bytes = chunk.get_bytes()
                logger.debug('Writing chunk: %r (%i/%i/%i: %s)',
                             chunk,
                             len(bytes),
                             chunk.size(),
                             chunk.phys_size(),
                             chunk.phys_size() == len(bytes))
                with open_wb(file_path) as fh:
                    fh.write(bytes)
                    fh.flush()
                    os.fsync(fh.fileno())
Exemple #2
0
def _test_create_permfile(path, variant=0, repeat=1):
    """
    Create a file with some artificial data,
    which is NOT removed automatically when it becomes unused.
    It is the caller duty to remove the file when it becomes unused.

    @param variant: Which variant of the text to use.
    @param repeat: How many times to repeat the text.
    """
    assert repeat >= 1

    with open_wb(path) as fh:
        fh.write(get_sample_data(variant) * repeat)
Exemple #3
0
def _test_create_permfile(path, variant=0, repeat=1):
    """
    Create a file with some artificial data,
    which is NOT removed automatically when it becomes unused.
    It is the caller duty to remove the file when it becomes unused.

    @param variant: Which variant of the text to use.
    @param repeat: How many times to repeat the text.
    """
    assert repeat >= 1

    with open_wb(path) as fh:
        fh.write(get_sample_data(variant) * repeat)
Exemple #4
0
    def __restore_single_file(self,
                              file_path, file_, file_blocks, cryptographer):
        """Actual restore procedure for a single file.

        @param file_: the file to restore.
        @type file_: LocalPhysicalFileState

        @param file_path: what is the real path for the file.
        @type file_path: basestring

        @param file_blocks: the iterable of blocks used to restore the file
            (if needed).
        @type file_blocks: col.Iterable

        @param cryptographer: C{Cryptographer} object to use for decrypting.
        @type cryptographer: Cryptographer

        @raises CannotRestoreFile: if checksum or fingerprint of restored file
            is not correct, and that was spotted before corrupting
            the actual file.
        @raises FileCorrupt: if checksum or fingerprint of restored file
            is not correct, and that caused the corruption of a file.
        """
        cls = self.__class__

        file_blocks = list(file_blocks)  # for reiterability

        # 1. Do a "dry run" to try to ensure nothing will be broken
        #    during the restore.
        try:
            consume(self.__file_blocks_generator(file_, file_blocks,
                                                 cryptographer))
        except Exception as e:
            logger.debug('Cannot restore: %s', traceback.format_exc())
            raise CannotRestoreFile('{:s} at {!r}'.format(file_.uuid,
                                                          file_path))
        else:
            # 2. Do a real run of restoration.

            # TODO: do not write if the existing file is newer,
            # but write to a "non-merged copy".
            # In more details:
            # * if the existing file is newer AND "hot" (i.e., not backed up,
            #   or yet buffered to be backed up), this should be treated
            #   as a name conflict.
            # * if the existing file is newer AND "cold" (i.e. in the cloud
            #   already), just think that we've received some too old file
            #   for some reason, and don't write anything.
            try:
                with open_wb(file_path) as fh:
                    for block in self.__file_blocks_generator(file_,
                                                              file_blocks,
                                                              cryptographer):
                        fh.write(block)
                    # Finished the loop over blocks and writing the file.
                    fh.flush()
                    fh.truncate()  # what if we were rewriting a larger file?
                    os.fsync(fh.fileno())
            except Exception as e:
                logger.debug('File corrupt: %s', traceback.format_exc())
                raise FileCorrupt('{:s} at {!r}'.format(file_.uuid,
                                                        file_path))
            else:
                _new_dt = file_.time_changed
                _new_dt_local = _new_dt.replace(tzinfo=TZ_UTC) \
                                       .astimezone(TZ_LOCAL)
                logger.debug('Converted update time from %s to %s',
                             _new_dt, _new_dt_local)
                _new_ts = time.mktime(_new_dt_local.timetuple())
                # _new_ts must be integer
                os.utime(file_path, (_new_ts, _new_ts))
                logger.debug("%r file's mtime force set to %s",
                             file_path, _new_dt)
Exemple #5
0
    def __restore_single_file(self, file_path, file_, file_blocks,
                              cryptographer):
        """Actual restore procedure for a single file.

        @param file_: the file to restore.
        @type file_: LocalPhysicalFileState

        @param file_path: what is the real path for the file.
        @type file_path: basestring

        @param file_blocks: the iterable of blocks used to restore the file
            (if needed).
        @type file_blocks: col.Iterable

        @param cryptographer: C{Cryptographer} object to use for decrypting.
        @type cryptographer: Cryptographer

        @raises CannotRestoreFile: if checksum or fingerprint of restored file
            is not correct, and that was spotted before corrupting
            the actual file.
        @raises FileCorrupt: if checksum or fingerprint of restored file
            is not correct, and that caused the corruption of a file.
        """
        cls = self.__class__

        file_blocks = list(file_blocks)  # for reiterability

        # 1. Do a "dry run" to try to ensure nothing will be broken
        #    during the restore.
        try:
            consume(
                self.__file_blocks_generator(file_, file_blocks,
                                             cryptographer))
        except Exception as e:
            logger.debug('Cannot restore: %s', traceback.format_exc())
            raise CannotRestoreFile('{:s} at {!r}'.format(
                file_.uuid, file_path))
        else:
            # 2. Do a real run of restoration.

            # TODO: do not write if the existing file is newer,
            # but write to a "non-merged copy".
            # In more details:
            # * if the existing file is newer AND "hot" (i.e., not backed up,
            #   or yet buffered to be backed up), this should be treated
            #   as a name conflict.
            # * if the existing file is newer AND "cold" (i.e. in the cloud
            #   already), just think that we've received some too old file
            #   for some reason, and don't write anything.
            try:
                with open_wb(file_path) as fh:
                    for block in self.__file_blocks_generator(
                            file_, file_blocks, cryptographer):
                        fh.write(block)
                    # Finished the loop over blocks and writing the file.
                    fh.flush()
                    fh.truncate()  # what if we were rewriting a larger file?
                    os.fsync(fh.fileno())
            except Exception as e:
                logger.debug('File corrupt: %s', traceback.format_exc())
                raise FileCorrupt('{:s} at {!r}'.format(file_.uuid, file_path))
            else:
                _new_dt = file_.time_changed
                _new_dt_local = _new_dt.replace(tzinfo=TZ_UTC) \
                                       .astimezone(TZ_LOCAL)
                logger.debug('Converted update time from %s to %s', _new_dt,
                             _new_dt_local)
                _new_ts = time.mktime(_new_dt_local.timetuple())
                # _new_ts must be integer
                os.utime(file_path, (_new_ts, _new_ts))
                logger.debug("%r file's mtime force set to %s", file_path,
                             _new_dt)
Exemple #6
0
    def migrate_chunks(self, old_path, new_path):
        """
        Migrate the chunks from their previous path to the new one.

        @note: Only non-dummy chunks are migrated;
               dummy chunks are removed from the old place and
               not regenerated at the new place,
               please call update_dummy_chunks_size() manually for that.
        """
        assert old_path != new_path, (old_path, new_path)

        # This two variables will be used to specify the progress of the task.
        num, of = 0, 0


        @exceptions_logged(logger)
        def timercb():
            """
            Callback function called on timer firing.
            """
            if (num, of) != (0, 0):
                logger_status_chunks_op.info(
                    'The chunk migration takes too long, completed %i/%i',
                    num, of,
                    extra={'_type': 'chunks_migration.progress',
                           'num': num,
                           'of': of})


        timer_service = TimerService(1.0, timercb)

        # If the task takes more than 3 seconds, start notifying
        # about the progress
        _callLater = reactor.callLater  # pylint:disable=E1101,C0103
        # Won't worry about deferToThread here, cause it is very fast.
        long_task_timer = _callLater(3.0, timer_service.startService)

        with self.__chunk_op_lock:
            try:
                # What chunk files are present on the FS,
                # and what are the chunk UUIDs?
                present_chunk_uuids_iter = self.__get_chunk_uuids_on_fs()

                with db.RDB() as rdbw:
                    dummy_chunks_in_db = \
                        frozenset(HostQueries.HostChunks
                                             .get_all_dummy_chunk_uuids(
                                                  rdbw=rdbw))

                # First, remove all the dummy chunks
                removed_dummy_chunks = []
                for dummy_chunk_uuid in dummy_chunks_in_db:
                    try:
                        assert self.__get_chunk_file_path(dummy_chunk_uuid,
                                                          is_dummy=True,
                                                          dir_path=old_path) \
                               == self.__get_chunk_file_path(dummy_chunk_uuid,
                                                             is_dummy=True)

                        _path = self.__get_chunk_file_path(dummy_chunk_uuid,
                                                           is_dummy=True)
                        if os.path.exists(_path):
                            os.unlink(_path)
                        # If we removed the file successfully, let's append it
                        # to the list of the chunks which are to be removed
                        # from the DB.
                        removed_dummy_chunks.append(dummy_chunk_uuid)
                    except Exception as e:
                        logger.error('Cannot remove dummy chunk %s: %s',
                                     dummy_chunk_uuid, e)
                HostQueries.HostChunks \
                           .delete_dummy_chunks(removed_dummy_chunks)

                # This dictionary maps the chunk UUID
                # to a tuple of the old filename and the new filename.
                #
                # Btw, no need to convert present_chunk_uuids_iter to set
                # and do the set difference, as it is the same complexity
                # as for ... if not in.
                uuid_to_filenames = \
                    {u: (self.__get_chunk_file_path(u,
                                                    is_dummy=False,
                                                    dir_path=old_path),
                         self.__get_chunk_file_path(u,
                                                    is_dummy=False,
                                                    dir_path=new_path))
                         for u in present_chunk_uuids_iter
                         if u not in dummy_chunks_in_db}

                # Now, move the files to the new directory.
                of = len(uuid_to_filenames)
                for u, (old_filename, new_filename) \
                        in uuid_to_filenames.iteritems():

                    logger.debug('Moving chunk %s from %s to %s',
                                 u, old_filename, new_filename)

                    try:
                        with open(old_filename, 'rb') as rfh:
                            with open_wb(new_filename) as wfh:
                                wfh.write(rfh.read())
                    except Exception:
                        logger.error('Cannot move chunk %s from %s to %s',
                                     u, old_filename, new_filename)
                    else:
                        try:
                            os.unlink(old_filename)
                        except Exception:
                            logger.error('Cannot remove chunk file %s',
                                         old_filename)

                    num += 1

            except Exception as e:
                logger_status_chunks_op_error.error(
                    'The chunks migration failed: %r',
                    e,
                    extra={'_type': 'chunks_migration.error',
                           '_exc': e,
                           '_tb': traceback.format_exc()})

            finally:
                if (not long_task_timer.called and
                    not long_task_timer.cancelled):
                    long_task_timer.cancel()

                if timer_service.running:
                    timer_service.stopService()