def fix_missing_base_version(self):
"""If there is no base version then set the oldest one as the base version."""
ABUNDANT_LOGGER.info('Fixing base version...')
if self.versions and self.base_version is None:
self.versions[0].is_base_version = True
self.versions = get_versions(self)
self.versions.sort(key=lambda x: x.time_of_creation)
def create_base(self) -> VersionAgent:
"""Create the base version."""
if self.base_version is None:
create_version(True, self)
else:
ABUNDANT_LOGGER.warning('Cannot create duplicate base versions')
self.load_versions()
return self.base_version
def load_config(self):
"""Load archive configurations."""
if not os.path.exists(self.archive_config_path):
ABUNDANT_LOGGER.error('Archive config not found at %s' % self.archive_config_path)
raise FileNotFoundError('Archive config not found at %s' % self.archive_config_path)
with open(self.archive_config_path, mode='r', encoding='utf-8') as raw_archive_config:
self.archive_config = json.load(raw_archive_config)
ABUNDANT_LOGGER.debug('Loaded archive config')
def load_config(self):
"""Load configuration for this version."""
with get_config(self.version_config_path) as version_config:
for version in version_config['VersionRecords']:
if version['UUID'] == self.uuid:
self.version_config = dict(version)
ABUNDANT_LOGGER.debug('Version record found: %s' % self.uuid)
return
ABUNDANT_LOGGER.error('Cannot find config for version %s' % self.uuid)
raise FileNotFoundError('Cannot find config for version %s' % self.uuid)
def get_versions(archive_agent) -> list:
"""Get all versions."""
version_config_path = os.path.join(archive_agent.archive_dir, 'meta', 'version_config.json')
if not os.path.exists(version_config_path):
ABUNDANT_LOGGER.warning('Version config missing')
return list()
with get_config(version_config_path) as version_config:
return [VersionAgent(version_record['UUID'], archive_agent) for version_record in
version_config['VersionRecords']]
def get_archive(self, uuid=None, source_dir=None, archive_dir=None) -> ArchiveAgent:
"""Get the archive that matches given restraints."""
if uuid is None and source_dir is None and archive_dir is None:
ABUNDANT_LOGGER.warning('Must provide at least one restraint')
raise ValueError('Must provide at least one restraint')
archive_record = self.master_config.get_archive_record(uuid, source_dir, archive_dir)
if archive_record is None:
return archive_record
return ArchiveAgent(archive_record['ArchiveDirectory'])
def is_base_version(self, is_base_version: bool):
"""Set if this version is a base version."""
with get_config(config_path=self.version_config_path, save_change=True) as version_config:
for version in version_config['VersionRecords']:
if version['UUID'] == self.uuid:
version['IsBaseVersion'] = is_base_version
break
if is_base_version:
ABUNDANT_LOGGER.info('Version %s is now base version' % self.uuid)
else:
ABUNDANT_LOGGER.info('Version %s is now non-base version' % self.uuid)
def migrate_oldest_version_to_base(self):
"""Migrate the oldest version to the base version.
But underneath it migrate the base version to the oldest version."""
assert self.base_version == self.versions[0]
if not self.versions:
ABUNDANT_LOGGER.warning('No base version found')
elif len(self.versions) == 1:
ABUNDANT_LOGGER.warning('Cannot migrate when only base version exists')
else:
self.base_version.migrate_to_next_version()
self.load_versions()
def load_versions(self):
"""Load all versions in this archive."""
self.versions = get_versions(self)
self.versions.sort(key=lambda x: x.time_of_creation)
if self.on_creation_pardon:
self.on_creation_pardon = False
elif not self.validate_versions():
self.fix_missing_base_version()
assert self.validate_versions(), 'Fatal internal error: either more than one base version' \
'is found or sorting function is not working'
number_of_versions = len(self.versions)
ABUNDANT_LOGGER.debug('Found %s version%s' % (number_of_versions, 's' if number_of_versions > 1 else ''))
def create_version(self) -> VersionAgent:
"""Add a new version."""
if self.max_number_of_versions == 1:
self.base_version.remove()
self.create_base()
else:
while len(self.versions) >= self.max_number_of_versions:
self.migrate_oldest_version_to_base()
if self.base_version is None:
ABUNDANT_LOGGER.warning('Cannot create non-base versions without a base version')
else:
create_version(False, self)
self.load_versions()
return self.versions[-1]
def migrate_to_next_version(self):
"""Migrate this version to its next version."""
next_version = self.next_version
ABUNDANT_LOGGER.debug('Migrating version %s to %s...' % (self.uuid, next_version.uuid))
# copy all files from current version to another version
# unless they already exists
number_of_file_copied = 0
for relative_path, absolute_path in self.exact_files:
if not next_version.has_file(relative_path):
absolute_path_in_another_version = next_version._get_full_path_of_file(relative_path)
shutil.move(absolute_path, absolute_path_in_another_version)
number_of_file_copied += 1
ABUNDANT_LOGGER.debug('Copied %s' % absolute_path_in_another_version)
ABUNDANT_LOGGER.info('Copied %s file(s)' % number_of_file_copied)
# set base version
if self.is_base_version:
next_version.is_base_version = True
# remove this version
self.remove(base_version_pardon=True)
# refresh status
next_version.load_config()
ABUNDANT_LOGGER.info('Migrated %s to %s' % (self.uuid, next_version.uuid))
def add_archive_record(self, source_dir: str, archive_dir: str):
"""Add an archive record."""
archive_uuid = str(uuid.uuid4())
while self.get_archive_record(archive_uuid):
archive_uuid = str(uuid.uuid4())
archive_record = dict(ARCHIVE_RECORD_TEMPLATE)
archive_record.update({"SourceDirectory": source_dir, "ArchiveDirectory": archive_dir, "UUID": archive_uuid})
self.archive_records.append(archive_record)
self.save_config()
ABUNDANT_LOGGER.info("Added archive record: %s" % archive_uuid)
ABUNDANT_LOGGER.debug("From %s to %s" % (source_dir, archive_dir))
return archive_record
def create_archive(archive_record: dict, algorithm: str, max_number_of_versions: int) -> ArchiveAgent:
"""Create an archive according to the archive record.
No validity check will be performed."""
source_dir, archive_dir = archive_record['SourceDirectory'], archive_record['ArchiveDirectory']
uuid = archive_record['UUID']
archive_content_dir = os.path.join(archive_dir, 'archive')
archive_meta_dir = os.path.join(archive_dir, 'meta')
ABUNDANT_LOGGER.debug('Creating archive: %s' % uuid)
try:
# create archive and meta directories
os.mkdir(archive_content_dir)
os.mkdir(archive_meta_dir)
# set archive config
archive_config = dict(ARCHIVE_CONFIG_TEMPLATE)
archive_config.update({
'HashAlgorithm': algorithm,
'SourceDirectory': source_dir,
'MaxNumberOfVersions': max_number_of_versions,
'UUID': uuid
})
with open(os.path.join(archive_meta_dir, 'archive_config.json'), mode='w', encoding='utf-8') \
as raw_archive_config:
json.dump(archive_config, raw_archive_config)
ABUNDANT_LOGGER.debug('Created archive config: %s' % uuid)
except OSError as e:
# OSError on file operations usually indicates insufficient privilege or
# incorrect configurations
ABUNDANT_LOGGER.error('Cannot create archive %s, possibly caused by insufficient privilege' %
uuid)
# undo previous change
if os.path.exists(archive_content_dir):
os.rmdir(archive_content_dir)
if os.path.exists(archive_meta_dir):
os.rmdir(archive_meta_dir)
ABUNDANT_LOGGER.info('Previous change undone')
# raise
raise e
else:
ABUNDANT_LOGGER.info('Created archive: %s' % uuid)
return ArchiveAgent(archive_dir, on_creation_pardon=True)
def remove(self, base_version_pardon=False):
"""Remove this version."""
if not base_version_pardon and self.is_base_version:
raise PermissionError('Base version cannot be removed')
# delete version record
with get_config(self.version_config_path, save_change=True) as version_config:
current_version_record = [version for version in version_config['VersionRecords']
if version['UUID'] == self.uuid][0]
version_config['VersionRecords'].remove(current_version_record)
# delete directory
shutil.rmtree(self.version_dir)
# update version records
self.archive_agent.load_versions()
ABUNDANT_LOGGER.info('Removed version %s' % self.uuid)
def export(self, destination_dir: str, exact=False):
"""Export files in this version to destination directory."""
ABUNDANT_LOGGER.debug('Exporting version %s to %s' % (self.uuid, destination_dir))
if not os.path.exists(destination_dir):
ABUNDANT_LOGGER.error('Cannot find destination directory: %s' % destination_dir)
raise FileNotFoundError('Cannot find destination directory: %s' % destination_dir)
file_source = self.files if not exact else self.exact_files
for relative_path, absolute_path in file_source:
destination_path = os.path.join(destination_dir, relative_path)
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
shutil.copy(absolute_path, destination_path)
ABUNDANT_LOGGER.debug('Copied %s' % destination_path)
ABUNDANT_LOGGER.info('Exported version %s to %s' % (self.uuid, destination_dir))
def load_config(self):
"""Load the config."""
with open("init_config.json", mode="r", encoding="utf-8") as raw_init_config:
self.init_config = json.load(raw_init_config)
ABUNDANT_LOGGER.info("Loaded initialisation config")
master_config_dir = self.init_config["MasterConfigDirectory"]
self.master_config_path = os.path.join(master_config_dir, "master_config.json")
if not os.path.exists(self.master_config_path):
ABUNDANT_LOGGER.warning("No master config found")
self.save_config(use_default_template=True)
with open(self.master_config_path, mode="r", encoding="utf-8") as raw_master_config:
self.master_config = json.load(raw_master_config)
ABUNDANT_LOGGER.info("Loaded master config")
def create_version(is_base_version: bool, archive_agent) -> VersionAgent:
"""Create a version.
:type archive_agent: ArchiveAgent"""
# generate version uuid
version_uuid = str(uuid.uuid4())
while archive_agent.get_version(version_uuid):
version_uuid = str(uuid.uuid4())
# prepare version record
version_record = dict(VERSION_RECORD_TEMPLATE)
version_record.update({
'TimeOfCreation': time.time(),
'IsBaseVersion': is_base_version,
'UUID': version_uuid
})
ABUNDANT_LOGGER.debug('Creating %s version: %s' % ('base' if is_base_version else 'non-base', version_uuid))
# create version config if no version is present
version_config_path = os.path.join(archive_agent.archive_dir, 'meta', 'version_config.json')
if not os.path.exists(version_config_path):
create_config(VERSION_CONFIG_TEMPLATE, version_config_path)
ABUNDANT_LOGGER.debug('Created version config: %s' % archive_agent.uuid)
# add version record
with get_config(version_config_path, save_change=True) as version_config:
version_config['VersionRecords'].append(version_record)
archive_agent.load_versions()
ABUNDANT_LOGGER.info('Added version record: %s' % version_uuid)
# create version directory and copy files
version = archive_agent.get_version(version_uuid)
if not os.path.exists(version.version_dir):
os.mkdir(version.version_dir)
version.copy_files()
ABUNDANT_LOGGER.info('Created %s version %s' % ('base' if is_base_version else 'non-base', version_uuid))
return version
def copy_files(self):
"""Copy files from source directory to version directory."""
ABUNDANT_LOGGER.debug('Copying files...')
# copy new or modified files
source_dir = self.archive_agent.source_dir
number_of_file_copied = 0
for root_dir, dirs, files in os.walk(source_dir):
for dir in dirs:
absolute_dir = os.path.join(root_dir, dir)
relative_dir = get_relative_path(absolute_dir, source_dir)
os.makedirs(self._get_full_path_of_file(relative_dir), exist_ok=True)
for file in files:
source_absolute_path = os.path.join(root_dir, file)
relative_path = get_relative_path(source_absolute_path, source_dir)
# find the previous version of this file
previous_version = self._get_previous_version_of_file(relative_path)
# under following circumstances file will be treated
# as already existing in previous versions and will
# not be copied
# if this is not a base version
# and if there is a previous version for this file
# and if that previous version is identical to current one
if not self.is_base_version \
and previous_version is not None \
and self.hasher.hash(previous_version._get_full_path_of_file(relative_path)) \
== self.hasher.hash(source_absolute_path):
ABUNDANT_LOGGER.debug('Skipping %s' % source_absolute_path)
continue
# otherwise just copy the file
shutil.copy(source_absolute_path, self._get_full_path_of_file(relative_path))
number_of_file_copied += 1
ABUNDANT_LOGGER.debug('Copied file %s' % relative_path)
ABUNDANT_LOGGER.info('Copied %s file(s)' % number_of_file_copied)
def remove_archive_record(self, uuid=None, source_dir=None, archive_dir=None):
"""Delete an archive record matching given restraints."""
archive = self.get_archive_record(uuid, source_dir, archive_dir)
if archive_dir is not None:
self.archive_records.remove(archive)
ABUNDANT_LOGGER.info("Deleted archive record: %s" % archive["UUID"])
def create_archive(self, source_dir: str, archive_dir: str, algorithm: str, max_number_of_versions: int):
"""Create an archive."""
# validity check
if not os.path.exists(source_dir):
ABUNDANT_LOGGER.error('Source directory does not exist: %s' % source_dir)
raise FileNotFoundError('Source directory does not exist: %s' % source_dir)
if not os.path.exists(archive_dir):
ABUNDANT_LOGGER.error('Archive directory does not exist: %s' % archive_dir)
raise FileNotFoundError('Archive directory does not exist: %s' % archive_dir)
if self.master_config.get_archive_record(archive_dir=archive_dir):
ABUNDANT_LOGGER.error('Archive directory has already been used: %s' % archive_dir)
raise FileNotFoundError('Archive directory has already been used: %s' % archive_dir)
algorithm = algorithm.lower()
if algorithm not in VALID_ALGORITHMS:
ABUNDANT_LOGGER.error('Invalid hash algorithm: %s' % algorithm)
raise NotImplementedError('Requested algorithm is either invalid or has not been implemented yet: %s'
% algorithm)
if max_number_of_versions < 0:
ABUNDANT_LOGGER.error('At least one version should be kept: %s' % max_number_of_versions)
raise ValueError('At least one version should be kept: %s' % max_number_of_versions)
# create archive record
new_archive_record = self.master_config.add_archive_record(source_dir, archive_dir)
# create archive
try:
archive = create_archive(new_archive_record, algorithm, max_number_of_versions)
except OSError as e:
# delete the archive record previously created
self.master_config.remove_archive_record(uuid=new_archive_record['UUID'])
ABUNDANT_LOGGER.debug('Archive record added removed')
raise e
# create base version
archive.create_base()
ABUNDANT_LOGGER.info('Created archive %s' % archive.uuid)
return archive
def save_config(self, use_default_template=False):
"""Save the config."""
with open(self.master_config_path, mode="w", encoding="utf-8") as raw_master_config:
json.dump(self.master_config if not use_default_template else MASTER_CONFIG_TEMPLATE, raw_master_config)
if use_default_template:
ABUNDANT_LOGGER.debug("Create default master config")
def __setitem__(self, key, value):
"""Update a master config item."""
self.master_config[key] = value
self.save_config()
ABUNDANT_LOGGER.info("Updated master config [%s] to [%s]" % (key, value))
#!/usr/env/bin python
# -*- encoding: utf-8 -*-
"""
Command line interface for abundant.
"""
from log import ABUNDANT_LOGGER, ABUNDANT_LOG_STD_OUT_HANDLER
ABUNDANT_LOGGER.info('Starting command line interface...')
ABUNDANT_LOGGER.removeHandler(ABUNDANT_LOG_STD_OUT_HANDLER)
from abundant import Abundant
__author__ = 'Kevin'
class CLICommandError(Exception):
"""CLI command error, either caused by incorrect parameter
or syntax."""
def __init__(self, message: str):
self.message = message
super(CLICommandError, self).__init__(message)
WELCOME_MESSAGE = '\nCommand line interface for Abundant, version 0.3'
LIST_ARCHIVE_FORMAT = '''
Number: {0}
UUID: {1}
Source directory: {2}
