Exemple #1
0
    def migrate_to_next_version(self):
        """Migrate this version to its next version."""
        next_version = self.next_version
        ABUNDANT_LOGGER.debug('Migrating version %s to %s...' % (self.uuid, next_version.uuid))

        # copy all files from current version to another version
        # unless they already exists
        number_of_file_copied = 0
        for relative_path, absolute_path in self.exact_files:
            if not next_version.has_file(relative_path):
                absolute_path_in_another_version = next_version._get_full_path_of_file(relative_path)
                shutil.move(absolute_path, absolute_path_in_another_version)
                number_of_file_copied += 1
                ABUNDANT_LOGGER.debug('Copied %s' % absolute_path_in_another_version)
        ABUNDANT_LOGGER.info('Copied %s file(s)' % number_of_file_copied)

        # set base version
        if self.is_base_version:
            next_version.is_base_version = True

        # remove this version
        self.remove(base_version_pardon=True)

        # refresh status
        next_version.load_config()
        ABUNDANT_LOGGER.info('Migrated %s to %s' % (self.uuid, next_version.uuid))
Exemple #2
0
    def load_config(self):
        """Load archive configurations."""
        if not os.path.exists(self.archive_config_path):
            ABUNDANT_LOGGER.error('Archive config not found at %s' % self.archive_config_path)
            raise FileNotFoundError('Archive config not found at %s' % self.archive_config_path)

        with open(self.archive_config_path, mode='r', encoding='utf-8') as raw_archive_config:
            self.archive_config = json.load(raw_archive_config)
        ABUNDANT_LOGGER.debug('Loaded archive config')
Exemple #3
0
 def load_config(self):
     """Load configuration for this version."""
     with get_config(self.version_config_path) as version_config:
         for version in version_config['VersionRecords']:
             if version['UUID'] == self.uuid:
                 self.version_config = dict(version)
                 ABUNDANT_LOGGER.debug('Version record found: %s' % self.uuid)
                 return
     ABUNDANT_LOGGER.error('Cannot find config for version %s' % self.uuid)
     raise FileNotFoundError('Cannot find config for version %s' % self.uuid)
Exemple #4
0
 def load_versions(self):
     """Load all versions in this archive."""
     self.versions = get_versions(self)
     self.versions.sort(key=lambda x: x.time_of_creation)
     if self.on_creation_pardon:
         self.on_creation_pardon = False
     elif not self.validate_versions():
         self.fix_missing_base_version()
         assert self.validate_versions(), 'Fatal internal error: either more than one base version' \
                                          'is found or sorting function is not working'
     number_of_versions = len(self.versions)
     ABUNDANT_LOGGER.debug('Found %s version%s' % (number_of_versions, 's' if number_of_versions > 1 else ''))
Exemple #5
0
    def add_archive_record(self, source_dir: str, archive_dir: str):
        """Add an archive record."""
        archive_uuid = str(uuid.uuid4())
        while self.get_archive_record(archive_uuid):
            archive_uuid = str(uuid.uuid4())

        archive_record = dict(ARCHIVE_RECORD_TEMPLATE)
        archive_record.update({"SourceDirectory": source_dir, "ArchiveDirectory": archive_dir, "UUID": archive_uuid})
        self.archive_records.append(archive_record)
        self.save_config()

        ABUNDANT_LOGGER.info("Added archive record: %s" % archive_uuid)
        ABUNDANT_LOGGER.debug("From %s to %s" % (source_dir, archive_dir))
        return archive_record
Exemple #6
0
    def export(self, destination_dir: str, exact=False):
        """Export files in this version to destination directory."""
        ABUNDANT_LOGGER.debug('Exporting version %s to %s' % (self.uuid, destination_dir))

        if not os.path.exists(destination_dir):
            ABUNDANT_LOGGER.error('Cannot find destination directory: %s' % destination_dir)
            raise FileNotFoundError('Cannot find destination directory: %s' % destination_dir)

        file_source = self.files if not exact else self.exact_files
        for relative_path, absolute_path in file_source:
            destination_path = os.path.join(destination_dir, relative_path)
            os.makedirs(os.path.dirname(destination_path), exist_ok=True)
            shutil.copy(absolute_path, destination_path)
            ABUNDANT_LOGGER.debug('Copied %s' % destination_path)
        ABUNDANT_LOGGER.info('Exported version %s to %s' % (self.uuid, destination_dir))
Exemple #7
0
def create_archive(archive_record: dict, algorithm: str, max_number_of_versions: int) -> ArchiveAgent:
    """Create an archive according to the archive record.
    No validity check will be performed."""
    source_dir, archive_dir = archive_record['SourceDirectory'], archive_record['ArchiveDirectory']
    uuid = archive_record['UUID']
    archive_content_dir = os.path.join(archive_dir, 'archive')
    archive_meta_dir = os.path.join(archive_dir, 'meta')
    ABUNDANT_LOGGER.debug('Creating archive: %s' % uuid)

    try:
        # create archive and meta directories
        os.mkdir(archive_content_dir)
        os.mkdir(archive_meta_dir)

        # set archive config
        archive_config = dict(ARCHIVE_CONFIG_TEMPLATE)
        archive_config.update({
            'HashAlgorithm': algorithm,
            'SourceDirectory': source_dir,
            'MaxNumberOfVersions': max_number_of_versions,
            'UUID': uuid
        })
        with open(os.path.join(archive_meta_dir, 'archive_config.json'), mode='w', encoding='utf-8') \
                as raw_archive_config:
            json.dump(archive_config, raw_archive_config)
        ABUNDANT_LOGGER.debug('Created archive config: %s' % uuid)

    except OSError as e:
        # OSError on file operations usually indicates insufficient privilege or
        # incorrect configurations
        ABUNDANT_LOGGER.error('Cannot create archive %s, possibly caused by insufficient privilege' %
                              uuid)

        # undo previous change
        if os.path.exists(archive_content_dir):
            os.rmdir(archive_content_dir)
        if os.path.exists(archive_meta_dir):
            os.rmdir(archive_meta_dir)
        ABUNDANT_LOGGER.info('Previous change undone')

        # raise
        raise e
    else:
        ABUNDANT_LOGGER.info('Created archive: %s' % uuid)
        return ArchiveAgent(archive_dir, on_creation_pardon=True)
Exemple #8
0
    def create_archive(self, source_dir: str, archive_dir: str, algorithm: str, max_number_of_versions: int):
        """Create an archive."""
        # validity check
        if not os.path.exists(source_dir):
            ABUNDANT_LOGGER.error('Source directory does not exist: %s' % source_dir)
            raise FileNotFoundError('Source directory does not exist: %s' % source_dir)
        if not os.path.exists(archive_dir):
            ABUNDANT_LOGGER.error('Archive directory does not exist: %s' % archive_dir)
            raise FileNotFoundError('Archive directory does not exist: %s' % archive_dir)
        if self.master_config.get_archive_record(archive_dir=archive_dir):
            ABUNDANT_LOGGER.error('Archive directory has already been used: %s' % archive_dir)
            raise FileNotFoundError('Archive directory has already been used: %s' % archive_dir)
        algorithm = algorithm.lower()
        if algorithm not in VALID_ALGORITHMS:
            ABUNDANT_LOGGER.error('Invalid hash algorithm: %s' % algorithm)
            raise NotImplementedError('Requested algorithm is either invalid or has not been implemented yet: %s'
                                      % algorithm)
        if max_number_of_versions < 0:
            ABUNDANT_LOGGER.error('At least one version should be kept: %s' % max_number_of_versions)
            raise ValueError('At least one version should be kept: %s' % max_number_of_versions)

        # create archive record
        new_archive_record = self.master_config.add_archive_record(source_dir, archive_dir)

        # create archive
        try:
            archive = create_archive(new_archive_record, algorithm, max_number_of_versions)
        except OSError as e:
            # delete the archive record previously created
            self.master_config.remove_archive_record(uuid=new_archive_record['UUID'])
            ABUNDANT_LOGGER.debug('Archive record added removed')

            raise e

        # create base version
        archive.create_base()
        ABUNDANT_LOGGER.info('Created archive %s' % archive.uuid)
        return archive
Exemple #9
0
def create_version(is_base_version: bool, archive_agent) -> VersionAgent:
    """Create a version.
    :type archive_agent: ArchiveAgent"""
    # generate version uuid
    version_uuid = str(uuid.uuid4())
    while archive_agent.get_version(version_uuid):
        version_uuid = str(uuid.uuid4())

    # prepare version record
    version_record = dict(VERSION_RECORD_TEMPLATE)
    version_record.update({
        'TimeOfCreation': time.time(),
        'IsBaseVersion': is_base_version,
        'UUID': version_uuid
    })
    ABUNDANT_LOGGER.debug('Creating %s version: %s' % ('base' if is_base_version else 'non-base', version_uuid))

    # create version config if no version is present
    version_config_path = os.path.join(archive_agent.archive_dir, 'meta', 'version_config.json')
    if not os.path.exists(version_config_path):
        create_config(VERSION_CONFIG_TEMPLATE, version_config_path)
        ABUNDANT_LOGGER.debug('Created version config: %s' % archive_agent.uuid)

    # add version record
    with get_config(version_config_path, save_change=True) as version_config:
        version_config['VersionRecords'].append(version_record)
    archive_agent.load_versions()
    ABUNDANT_LOGGER.info('Added version record: %s' % version_uuid)

    # create version directory and copy files
    version = archive_agent.get_version(version_uuid)
    if not os.path.exists(version.version_dir):
        os.mkdir(version.version_dir)
    version.copy_files()

    ABUNDANT_LOGGER.info('Created %s version %s' % ('base' if is_base_version else 'non-base', version_uuid))
    return version
Exemple #10
0
    def copy_files(self):
        """Copy files from source directory to version directory."""
        ABUNDANT_LOGGER.debug('Copying files...')

        # copy new or modified files
        source_dir = self.archive_agent.source_dir
        number_of_file_copied = 0
        for root_dir, dirs, files in os.walk(source_dir):
            for dir in dirs:
                absolute_dir = os.path.join(root_dir, dir)
                relative_dir = get_relative_path(absolute_dir, source_dir)
                os.makedirs(self._get_full_path_of_file(relative_dir), exist_ok=True)
            for file in files:
                source_absolute_path = os.path.join(root_dir, file)
                relative_path = get_relative_path(source_absolute_path, source_dir)

                # find the previous version of this file
                previous_version = self._get_previous_version_of_file(relative_path)

                # under following circumstances file will be treated
                # as already existing in previous versions and will
                # not be copied
                # if this is not a base version
                # and if there is a previous version for this file
                # and if that previous version is identical to current one
                if not self.is_base_version \
                        and previous_version is not None \
                        and self.hasher.hash(previous_version._get_full_path_of_file(relative_path)) \
                                == self.hasher.hash(source_absolute_path):
                    ABUNDANT_LOGGER.debug('Skipping %s' % source_absolute_path)
                    continue

                # otherwise just copy the file
                shutil.copy(source_absolute_path, self._get_full_path_of_file(relative_path))
                number_of_file_copied += 1
                ABUNDANT_LOGGER.debug('Copied file %s' % relative_path)
        ABUNDANT_LOGGER.info('Copied %s file(s)' % number_of_file_copied)
Exemple #11
0
 def save_config(self, use_default_template=False):
     """Save the config."""
     with open(self.master_config_path, mode="w", encoding="utf-8") as raw_master_config:
         json.dump(self.master_config if not use_default_template else MASTER_CONFIG_TEMPLATE, raw_master_config)
     if use_default_template:
         ABUNDANT_LOGGER.debug("Create default master config")