def migrate_to_next_version(self):
"""Migrate this version to its next version."""
next_version = self.next_version
ABUNDANT_LOGGER.debug('Migrating version %s to %s...' % (self.uuid, next_version.uuid))
# copy all files from current version to another version
# unless they already exists
number_of_file_copied = 0
for relative_path, absolute_path in self.exact_files:
if not next_version.has_file(relative_path):
absolute_path_in_another_version = next_version._get_full_path_of_file(relative_path)
shutil.move(absolute_path, absolute_path_in_another_version)
number_of_file_copied += 1
ABUNDANT_LOGGER.debug('Copied %s' % absolute_path_in_another_version)
ABUNDANT_LOGGER.info('Copied %s file(s)' % number_of_file_copied)
# set base version
if self.is_base_version:
next_version.is_base_version = True
# remove this version
self.remove(base_version_pardon=True)
# refresh status
next_version.load_config()
ABUNDANT_LOGGER.info('Migrated %s to %s' % (self.uuid, next_version.uuid))
def load_config(self):
"""Load archive configurations."""
if not os.path.exists(self.archive_config_path):
ABUNDANT_LOGGER.error('Archive config not found at %s' % self.archive_config_path)
raise FileNotFoundError('Archive config not found at %s' % self.archive_config_path)
with open(self.archive_config_path, mode='r', encoding='utf-8') as raw_archive_config:
self.archive_config = json.load(raw_archive_config)
ABUNDANT_LOGGER.debug('Loaded archive config')
def load_config(self):
"""Load configuration for this version."""
with get_config(self.version_config_path) as version_config:
for version in version_config['VersionRecords']:
if version['UUID'] == self.uuid:
self.version_config = dict(version)
ABUNDANT_LOGGER.debug('Version record found: %s' % self.uuid)
return
ABUNDANT_LOGGER.error('Cannot find config for version %s' % self.uuid)
raise FileNotFoundError('Cannot find config for version %s' % self.uuid)
def load_versions(self):
"""Load all versions in this archive."""
self.versions = get_versions(self)
self.versions.sort(key=lambda x: x.time_of_creation)
if self.on_creation_pardon:
self.on_creation_pardon = False
elif not self.validate_versions():
self.fix_missing_base_version()
assert self.validate_versions(), 'Fatal internal error: either more than one base version' \
'is found or sorting function is not working'
number_of_versions = len(self.versions)
ABUNDANT_LOGGER.debug('Found %s version%s' % (number_of_versions, 's' if number_of_versions > 1 else ''))
def add_archive_record(self, source_dir: str, archive_dir: str):
"""Add an archive record."""
archive_uuid = str(uuid.uuid4())
while self.get_archive_record(archive_uuid):
archive_uuid = str(uuid.uuid4())
archive_record = dict(ARCHIVE_RECORD_TEMPLATE)
archive_record.update({"SourceDirectory": source_dir, "ArchiveDirectory": archive_dir, "UUID": archive_uuid})
self.archive_records.append(archive_record)
self.save_config()
ABUNDANT_LOGGER.info("Added archive record: %s" % archive_uuid)
ABUNDANT_LOGGER.debug("From %s to %s" % (source_dir, archive_dir))
return archive_record
def export(self, destination_dir: str, exact=False):
"""Export files in this version to destination directory."""
ABUNDANT_LOGGER.debug('Exporting version %s to %s' % (self.uuid, destination_dir))
if not os.path.exists(destination_dir):
ABUNDANT_LOGGER.error('Cannot find destination directory: %s' % destination_dir)
raise FileNotFoundError('Cannot find destination directory: %s' % destination_dir)
file_source = self.files if not exact else self.exact_files
for relative_path, absolute_path in file_source:
destination_path = os.path.join(destination_dir, relative_path)
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
shutil.copy(absolute_path, destination_path)
ABUNDANT_LOGGER.debug('Copied %s' % destination_path)
ABUNDANT_LOGGER.info('Exported version %s to %s' % (self.uuid, destination_dir))
def create_archive(archive_record: dict, algorithm: str, max_number_of_versions: int) -> ArchiveAgent:
"""Create an archive according to the archive record.
No validity check will be performed."""
source_dir, archive_dir = archive_record['SourceDirectory'], archive_record['ArchiveDirectory']
uuid = archive_record['UUID']
archive_content_dir = os.path.join(archive_dir, 'archive')
archive_meta_dir = os.path.join(archive_dir, 'meta')
ABUNDANT_LOGGER.debug('Creating archive: %s' % uuid)
try:
# create archive and meta directories
os.mkdir(archive_content_dir)
os.mkdir(archive_meta_dir)
# set archive config
archive_config = dict(ARCHIVE_CONFIG_TEMPLATE)
archive_config.update({
'HashAlgorithm': algorithm,
'SourceDirectory': source_dir,
'MaxNumberOfVersions': max_number_of_versions,
'UUID': uuid
})
with open(os.path.join(archive_meta_dir, 'archive_config.json'), mode='w', encoding='utf-8') \
as raw_archive_config:
json.dump(archive_config, raw_archive_config)
ABUNDANT_LOGGER.debug('Created archive config: %s' % uuid)
except OSError as e:
# OSError on file operations usually indicates insufficient privilege or
# incorrect configurations
ABUNDANT_LOGGER.error('Cannot create archive %s, possibly caused by insufficient privilege' %
uuid)
# undo previous change
if os.path.exists(archive_content_dir):
os.rmdir(archive_content_dir)
if os.path.exists(archive_meta_dir):
os.rmdir(archive_meta_dir)
ABUNDANT_LOGGER.info('Previous change undone')
# raise
raise e
else:
ABUNDANT_LOGGER.info('Created archive: %s' % uuid)
return ArchiveAgent(archive_dir, on_creation_pardon=True)
def create_archive(self, source_dir: str, archive_dir: str, algorithm: str, max_number_of_versions: int):
"""Create an archive."""
# validity check
if not os.path.exists(source_dir):
ABUNDANT_LOGGER.error('Source directory does not exist: %s' % source_dir)
raise FileNotFoundError('Source directory does not exist: %s' % source_dir)
if not os.path.exists(archive_dir):
ABUNDANT_LOGGER.error('Archive directory does not exist: %s' % archive_dir)
raise FileNotFoundError('Archive directory does not exist: %s' % archive_dir)
if self.master_config.get_archive_record(archive_dir=archive_dir):
ABUNDANT_LOGGER.error('Archive directory has already been used: %s' % archive_dir)
raise FileNotFoundError('Archive directory has already been used: %s' % archive_dir)
algorithm = algorithm.lower()
if algorithm not in VALID_ALGORITHMS:
ABUNDANT_LOGGER.error('Invalid hash algorithm: %s' % algorithm)
raise NotImplementedError('Requested algorithm is either invalid or has not been implemented yet: %s'
% algorithm)
if max_number_of_versions < 0:
ABUNDANT_LOGGER.error('At least one version should be kept: %s' % max_number_of_versions)
raise ValueError('At least one version should be kept: %s' % max_number_of_versions)
# create archive record
new_archive_record = self.master_config.add_archive_record(source_dir, archive_dir)
# create archive
try:
archive = create_archive(new_archive_record, algorithm, max_number_of_versions)
except OSError as e:
# delete the archive record previously created
self.master_config.remove_archive_record(uuid=new_archive_record['UUID'])
ABUNDANT_LOGGER.debug('Archive record added removed')
raise e
# create base version
archive.create_base()
ABUNDANT_LOGGER.info('Created archive %s' % archive.uuid)
return archive
def create_version(is_base_version: bool, archive_agent) -> VersionAgent:
"""Create a version.
:type archive_agent: ArchiveAgent"""
# generate version uuid
version_uuid = str(uuid.uuid4())
while archive_agent.get_version(version_uuid):
version_uuid = str(uuid.uuid4())
# prepare version record
version_record = dict(VERSION_RECORD_TEMPLATE)
version_record.update({
'TimeOfCreation': time.time(),
'IsBaseVersion': is_base_version,
'UUID': version_uuid
})
ABUNDANT_LOGGER.debug('Creating %s version: %s' % ('base' if is_base_version else 'non-base', version_uuid))
# create version config if no version is present
version_config_path = os.path.join(archive_agent.archive_dir, 'meta', 'version_config.json')
if not os.path.exists(version_config_path):
create_config(VERSION_CONFIG_TEMPLATE, version_config_path)
ABUNDANT_LOGGER.debug('Created version config: %s' % archive_agent.uuid)
# add version record
with get_config(version_config_path, save_change=True) as version_config:
version_config['VersionRecords'].append(version_record)
archive_agent.load_versions()
ABUNDANT_LOGGER.info('Added version record: %s' % version_uuid)
# create version directory and copy files
version = archive_agent.get_version(version_uuid)
if not os.path.exists(version.version_dir):
os.mkdir(version.version_dir)
version.copy_files()
ABUNDANT_LOGGER.info('Created %s version %s' % ('base' if is_base_version else 'non-base', version_uuid))
return version
def copy_files(self):
"""Copy files from source directory to version directory."""
ABUNDANT_LOGGER.debug('Copying files...')
# copy new or modified files
source_dir = self.archive_agent.source_dir
number_of_file_copied = 0
for root_dir, dirs, files in os.walk(source_dir):
for dir in dirs:
absolute_dir = os.path.join(root_dir, dir)
relative_dir = get_relative_path(absolute_dir, source_dir)
os.makedirs(self._get_full_path_of_file(relative_dir), exist_ok=True)
for file in files:
source_absolute_path = os.path.join(root_dir, file)
relative_path = get_relative_path(source_absolute_path, source_dir)
# find the previous version of this file
previous_version = self._get_previous_version_of_file(relative_path)
# under following circumstances file will be treated
# as already existing in previous versions and will
# not be copied
# if this is not a base version
# and if there is a previous version for this file
# and if that previous version is identical to current one
if not self.is_base_version \
and previous_version is not None \
and self.hasher.hash(previous_version._get_full_path_of_file(relative_path)) \
== self.hasher.hash(source_absolute_path):
ABUNDANT_LOGGER.debug('Skipping %s' % source_absolute_path)
continue
# otherwise just copy the file
shutil.copy(source_absolute_path, self._get_full_path_of_file(relative_path))
number_of_file_copied += 1
ABUNDANT_LOGGER.debug('Copied file %s' % relative_path)
ABUNDANT_LOGGER.info('Copied %s file(s)' % number_of_file_copied)
def save_config(self, use_default_template=False):
"""Save the config."""
with open(self.master_config_path, mode="w", encoding="utf-8") as raw_master_config:
json.dump(self.master_config if not use_default_template else MASTER_CONFIG_TEMPLATE, raw_master_config)
if use_default_template:
ABUNDANT_LOGGER.debug("Create default master config")