def read_date(self, type_of_date):
"""
Reds the :py:attr:_creation_date or :py:attr:_last_backup_date attribute.
:param type_of_date: 'creation_date' or 'last_backup_date'.
:return: A datetime.datetime object or a datetime.date object in function of the ``type_of_date``.
"""
metadata_data = read_ini_file(self._metadata_file)
if type_of_date == 'creation_date':
logger.info(
"The creation date is read form the repository's metadata file."
)
return datetime.strptime(
'{} {} {} {} {} {} {}'.format(
metadata_data[type_of_date]['year'],
metadata_data[type_of_date]['month'],
metadata_data[type_of_date]['day'],
metadata_data[type_of_date]['hour'],
metadata_data[type_of_date]['minute'],
metadata_data[type_of_date]['second'],
metadata_data[type_of_date]['microsecond']),
'%Y %m %d %H %M %S %f')
elif type_of_date == 'last_backup_date':
logger.info(
"The last backup date is read form the repository's metadata file."
)
return datetime.strptime(
'{:0>4} {} {}'.format(metadata_data[type_of_date]['year'],
metadata_data[type_of_date]['month'],
metadata_data[type_of_date]['day']),
'%Y %m %d').date()
def load(self):
"""
Try to load a :py:class:Repository object. If the :py:attr:_location path already exists then it tries to load
it, but if not it will create a new :py:class:Repository object by calling the :py:meth:initialize method.
:exception ValueError is raised if the path is not a directory.
:return:
"""
if path.exists(self._location):
if path.isdir(self._location):
self._creation_date = self.read_date('creation_date')
self._last_backup_date = self.read_date('last_backup_date')
self.create_repo_metadata_file(self._creation_date,
self._last_backup_date)
if self.is_backup_needed():
self._last_backup_date = datetime.utcnow().date()
self.create_repo_metadata_file(self._creation_date,
self._last_backup_date)
self.create_backup(backup_file_name=self._name)
self._name = read_ini_file(
self._paths_file)['repository']['name']
else:
raise ValueError('The repository should be a directory!')
else:
self.initialize()
def __init__(self, repository_location, paths_file):
"""
Initialisation of a new :py:class:RoleManager object.
:param repository_location: The path of the users directory linked to the :py:class:Repository object.
:param paths_file: Tbe paths file path of the :py:class:Repository object.
"""
metadata_data = read_ini_file(paths_file)
self._location = path.join(repository_location, metadata_data['directories']['users'])
def __init__(self, repository_location, paths_file):
"""
Initialisation of a new :py:class:UserManager object.
:param repository_location: The path of the linked to :py:class:Repository object.
:param paths_file: The paths file of the :py:class:Repository object.
"""
self.paths_file = paths_file
self.repository_location = repository_location
metadata_data = read_ini_file(self.paths_file)
self._location = path.join(self.repository_location, metadata_data['directories']['users'])
def __init__(self, repository_location, paths_file=None):
"""
Initialisation of a new :py:class:DocumentManager object.
:param repository_location: The path of the repository for which is working.
:param paths_file: The path where the repositorie's paths_file is, this is a metadata file of the repository.
"""
if not paths_file:
self._location = repository_location
else:
metadata_data = read_ini_file(paths_file)
self._location = path.join(
repository_location, metadata_data['directories']['documents'])
def show_repository_info(self, name=''):
"""
Shows some information about the :py:class:Repository object in an index.html file.
The following information is getherd in the HTML file: :py:class:Repository object :py:attr:name,
:py:attr:_creation_date, :py:attr:_last_backup_date, the :py:attr:path_file meta data, number and all
:py:class:User objects, all :py:class:Roles object and the number and all :py:class:Document objects.
:param name: This attribute is not used to show information about an actual :py:class:Repository, because the
information is printed into the index.html file, but we can prin out information about an archived
:py:class:Repository too, and for those this parameter is the name of the backup file.
:return:
"""
paths = read_ini_file(self._paths_file)
users = dict()
documents = dict()
roles = dict()
for user_id in self._user_manager.find_all_users():
users[user_id] = self._user_manager.load_user(user_id)
for document_id in self._document_manager.find_all_documents():
documents[document_id] = self._document_manager.load_document(
document_id, self._user_manager)
for role_key, user_ids_value in self._user_manager.list_users_by_role(
).iteritems():
roles[role_key] = ', '.join([str(i) for i in user_ids_value])
abs_path = path.dirname(path.abspath(__file__))
logger.info("All data is collected to show in HTML.")
env = Environment(
loader=FileSystemLoader(path.join(abs_path, 'templates')))
template = env.get_template('rep_info.html')
output_from_parsed_template = template.render(
repository_name=self._name,
creation_date=self._creation_date,
backup_date=self._last_backup_date,
paths=paths,
users=users,
roles=roles,
documents=documents)
logger.info("The template is rendered.")
with open(path.join(abs_path, "index{}.html".format('_' + name)),
"wb") as fh:
fh.write(output_from_parsed_template)
logger.info("The template is written to {} file.".format(
"index{}.html".format('_' + name)))
webbrowser.open(path.join(abs_path, "index{}.html".format('_' + name)))
logger.info("The {} file is opened in the browser.".format(
"index{}.html".format('_' + name)))
def load_project(self, project_id):
"""
Loads a :py:class:Project object from the filesystem.
:param project_id: The ID of the :py:class:Project object to load.
:exception ValueError is raised if no :py:class:Project is found with the ``project_id``.
:return: :py:class:Project object loaded from the filesystem.
"""
project_path = path.join(self.location, str(project_id))
if path.exists(project_path):
project_metadata_path = path.join(project_path, PROJECT_METADATA_FILE_NAME_FORMAT.format(project_id))
project_data = read_ini_file(project_metadata_path)['project']
return Project(project_data['name'], project_data['description'], project_data['members'],
project_data['documents'])
else:
raise ValueError("The {} path doesn't exists!".format(project_path))
def test_empty_repository_creation(self):
Repository('Empty', '/tmp/test_repo')
self.assertTrue(os.path.isdir('/tmp/test_repo/documents'))
self.assertTrue(os.path.isdir('/tmp/test_repo/projects'))
self.assertTrue(os.path.isdir('/tmp/test_repo/logs'))
self.assertTrue(os.path.isdir('/tmp/test_repo/users'))
self.assertTrue(os.path.exists('/tmp/test_repo/paths.ini'))
self.assertTrue(os.path.exists('/tmp/test_repo/users/roles.txt'))
# THIS IS WRONG!!! You can't predict the order of a dictionary => you can't test an INI file by line!
# with open('/tmp/test_repo/paths.ini') as path_file:
# self.assertEqual(path_file.readline().rstrip('\n'), '[directories]')
# self.assertEqual(path_file.readline().rstrip('\n'), 'documents=documents')
# self.assertEqual(path_file.readline().rstrip('\n'), 'logs=logs')
# self.assertEqual(path_file.readline().rstrip('\n'), 'projects=projects')
# self.assertEqual(path_file.readline().rstrip('\n'), 'users=users')
metadata_file = read_ini_file('/tmp/test_repo/paths.ini')
self.assertEqual(metadata_file['directories']['documents'],
'documents')
self.assertEqual(metadata_file['directories']['logs'], 'logs')
self.assertEqual(metadata_file['directories']['projects'], 'projects')
self.assertEqual(metadata_file['directories']['users'], 'users')
shutil.rmtree('/tmp/test_repo')
def restore(self,
backup_file_name='backup',
backup_path='./Backups',
verbose=False,
date_format='%Y/%m/%d %H:%M:%S',
backup_documents=True,
backup_logs=True,
backup_projects=True,
backup_reports=True,
backup_users=True):
"""
Restores a :py:class:Repository object from the filesystem and deletes the old :py:class:Repository object.
:param backup_file_name: The backup files name of the :py:class:Repository object, the default value is 'backup'.
:param backup_path: The backup path from where the ``backup_file_name`` will be restored, the default value
is './Backups'
:param verbose: Bool, if it's True it will print out some information about the restore process, default value
is False.
:param date_format: The date format in which the date are printed out if the ``verbose`` parameter is True,
the default value is '%Y/%m/%d %H:%M:%S'.
:param backup_documents: Bool, determines if to restore the :py:class:Document objects of the
:py:class:Repository.
:param backup_logs: Bool, determines if to restpre the log files of the :py:class:Repository.
:param backup_projects: Bool, determines if to restore the :py:class:Project objects of the
:py:class:Repository.
:param backup_reports: Bool, determines if to restore the :py:class:Report objects of the :py:class:Repository.
:param backup_users: Bool, determines if to restore the :py:class:User objects of the :py:class:Repository.
"""
start_time = datetime.utcnow()
logger.info(
"The restore of the {} repository has started on UTC {}.".format(
self._name, start_time.strftime(date_format)))
if verbose:
print("The restore of the {} repository has started on UTC {}.".
format(self._name, start_time.strftime(date_format)))
rmtree(self._location)
logger.info("The old repository is deleted on {} path.".format(
self._location))
if verbose:
print("The old repository is deleted on {} path.".format(
self._location))
with ZipFile(path.join(backup_path, backup_file_name + '.zip'),
"r") as z:
z.extractall(self._location)
if not (backup_documents and backup_logs and backup_projects
and backup_reports and backup_users):
pats_file = read_ini_file(self._paths_file)
unimported = []
if not backup_documents:
rmtree(
path.join(self._location,
pats_file['directories']['documents']))
makedirs(
path.join(self._location,
pats_file['directories']['documents']))
logger.debug("The {} directory is removed.".format(
path.join(self._location,
pats_file['directories']['documents'])))
unimported.append('documents')
if not backup_logs:
rmtree(
path.join(self._location,
pats_file['directories']['logs']))
makedirs(
path.join(self._location,
pats_file['directories']['logs']))
logger.debug("The {} directory is removed.".format(
(path.join(self._location,
pats_file['directories']['logs']))))
unimported.append('logs')
if not backup_projects:
rmtree(
path.join(self._location,
pats_file['directories']['projects']))
makedirs(
path.join(self._location,
pats_file['directories']['projects']))
logger.debug("The {} directory is removed.".format(
(path.join(self._location,
pats_file['directories']['projects']))))
unimported.append('projects')
if not backup_reports:
rmtree(
path.join(self._location,
pats_file['directories']['reports']))
makedirs(
path.join(self._location,
pats_file['directories']['reports']))
logger.debug("The {} directory is removed.".format(
(path.join(self._location,
pats_file['directories']['reports']))))
unimported.append('reports')
if not backup_users:
rmtree(
path.join(self._location,
pats_file['directories']['users']))
makedirs(
path.join(self._location,
pats_file['directories']['users']))
logger.debug("The {} directory is removed.".format(
(path.join(self._location,
pats_file['directories']['users']))))
unimported.append('users')
if len(unimported) > 0:
print("The {} were not imported.".format(
', '.join(unimported)))
logger.debug("The {} were not imported.".format(
', '.join(unimported)))
end_time = datetime.utcnow()
if verbose:
print(
"The restore is completed on UTC {}, please check the {} repository"
.format(end_time.strftime(date_format), self._location))
print("The process lasted {} seconds.".format(
(end_time - start_time).total_seconds()))
logger.info(
"The restore is completed on UTC {}, please check the {} repository"
.format(end_time.strftime(date_format), self._location))
logger.info("The process lasted {} seconds.".format(
(end_time - start_time).total_seconds()))
def create_backup(self,
backup_file_name='backup',
backup_path='./Backups',
verbose=False,
date_format='%Y/%m/%d %H:%M:%S',
backup_documents=True,
backup_logs=True,
backup_projects=True,
backup_reports=True,
backup_users=True):
"""
Creates a backup of the :py:class:Repository object to the ``backup_path`` with ``backup_file_name``.
:param backup_file_name: The backup files name of the :py:class:Repository object, the default value is 'backup'.
:param backup_path: The backup path where the ``backup_file_name`` is saved, the default value is './Backups'
:param verbose: Bool, if it's True it will print out some information about the backup process, default value
is False.
:param date_format: The date format in which the date are printed out if the ``verbose`` parameter is True,
the default value is '%Y/%m/%d %H:%M:%S'.
:param backup_documents: Bool, determines if to back up the :py:class:Document objects of the
:py:class:Repository.
:param backup_logs: Bool, determines if to back up the log files of the :py:class:Repository.
:param backup_projects: Bool, determines if to back up the :py:class:Project objects of the
:py:class:Repository.
:param backup_reports: Bool, determines if to back up the :py:class:Report objects of the
:py:class:Repository.
:param backup_users: Bool, determines if to back up the :py:class:User objects of the
:py:class:Repository.
:return:
"""
start_time = datetime.utcnow()
logger.info(
"The backup of the {} repository has started on UTC {}.".format(
self._name, start_time.strftime(date_format)))
if verbose:
print("The backup of the {} repository has started on UTC {}.".
format(self._name, start_time.strftime(date_format)))
if not path.exists(backup_path):
makedirs(backup_path)
logger.info(
"The {} backup path structure is created.".format(backup_path))
if verbose:
print("The {} backup path structure is created.".format(
backup_path))
else:
logger.info("The {} backup path exists.".format(backup_path))
if verbose:
print("The {} backup path exists.".format(backup_path))
backup_file_name = self.determine_export_file_name(
backup_file_name, backup_path)
logger.info(
"The name of the backup file is: {}.zip.".format(backup_file_name))
if verbose:
print("The name of the backup file is: {}.zip.".format(
backup_file_name))
new_location = self._location
if not (backup_documents and backup_logs and backup_projects
and backup_reports and backup_users):
pats_file = read_ini_file(self._paths_file)
copytree(new_location, './{}'.format(backup_file_name))
logger.debug(
"The backup file is copied from to {} with {} name.".format(
new_location, './{}'.format(backup_file_name)))
new_location = './{}'.format(backup_file_name)
if not backup_documents:
rmtree(
path.join(self._location,
pats_file['directories']['documents']))
makedirs(
path.join(self._location,
pats_file['directories']['documents']))
logger.debug("The {} directory is removed.".format(
path.join(self._location,
pats_file['directories']['documents'])))
if not backup_logs:
rmtree(
path.join(self._location,
pats_file['directories']['logs']))
makedirs(
path.join(self._location,
pats_file['directories']['logs']))
logger.debug("The {} directory is removed.".format(
path.join(self._location,
pats_file['directories']['logs'])))
if not backup_projects:
rmtree(
path.join(self._location,
pats_file['directories']['projects']))
makedirs(
path.join(self._location,
pats_file['directories']['projects']))
logger.debug("The {} directory is removed.".format(
path.join(self._location,
pats_file['directories']['projects'])))
if not backup_reports:
rmtree(
path.join(self._location,
pats_file['directories']['reports']))
makedirs(
path.join(self._location,
pats_file['directories']['reports']))
logger.debug("The {} directory is removed.".format(
path.join(self._location,
pats_file['directories']['reports'])))
if not backup_users:
rmtree(
path.join(self._location,
pats_file['directories']['users']))
makedirs(
path.join(self._location,
pats_file['directories']['users']))
logger.debug("The {} directory is removed.".format(
path.join(self._location,
pats_file['directories']['users'])))
make_archive(path.join(backup_path, backup_file_name),
'zip',
new_location,
verbose=verbose,
logger=logger)
if new_location == './{}'.format(backup_file_name) and path.exists(
new_location):
rmtree(new_location)
end_time = datetime.utcnow()
if verbose:
print(
"The backup is completed on UTC {}, please check the {} file".
format(end_time.strftime(date_format),
path.join(backup_path, backup_file_name)))
print("The process lasted {} seconds.".format(
(end_time - start_time).total_seconds()))
logger.info(
"The backup is completed on UTC {}, please check the {} file".
format(end_time.strftime(date_format),
path.join(backup_path, backup_file_name)))
logger.info("The process lasted {} seconds.".format(
(end_time - start_time).total_seconds()))
def import_documents(self, from_path):
"""
Imports all :py:class:Document objects from a path to the :py:class:Repository.
:param from_path: The path where to search for :py:class:Document objects.
:exception RuntimeError is raised if the :py:class:Document object doesn't contains the referenced file.
:exception ValueError is raised if the :py:class:Document object has no author.
:exception ValueError is raised if there is no available :py:class:Document objects on the ``from_path`` path.
:exception ValueError is raised if the ``from_path`` doesn't exists.
:return:
"""
if path.exists(from_path):
all_documents = Repository.find_all_documents_in_path(from_path)
logger.debug(
"All documents are loaded form the {} path.".format(from_path))
metadata_data = read_ini_file(self._paths_file)
logger.debug("The content repositories metadata file is loaded.")
to_path = path.join(self._location,
metadata_data['directories']['documents'])
if len(all_documents) > 0:
for document_id in all_documents:
new_path = reduce(path.join, [to_path, str(document_id)])
old_path = path.join(from_path, str(document_id))
copytree(old_path, new_path)
logger.info(
"The {} directory's content is copied to {} path.".
format(old_path, new_path))
try:
document_files_existence = self._document_manager.document_files_exist(
document_id, user_manager=self._user_manager)
for file_name_key, exists_value in document_files_existence.iteritems(
):
if not exists_value:
logger.exception(
"The {} file doesn't exists in the {} ID document!"
.format(file_name_key, document_id))
raise RuntimeError(
"The {} file doesn't exists in the {} ID document!"
.format(file_name_key, document_id))
logger.info("All the directory's files exist.")
document = self._document_manager.load_document(
document_id, self._user_manager)
logger.debug(
"The document with {} ID is loaded into the memory"
.format(document_id))
if not isinstance(document.author, list):
doc_author = [document.author]
else:
doc_author = document.author
if len(doc_author) == 0:
logger.exception("No author related to document!")
raise ValueError("No author related to document!")
except Exception as e:
rmtree(new_path)
logger.exception(
"An {} exception is raised when importing the document with {} ID."
.format(e.__class__.__name__, document_id))
raise e
else:
logger.exception(
"No document to import from the '{}' path!".format(
from_path))
raise ValueError(
"No document to import from the '{}' path!".format(
from_path))
else:
logger.exception("The '{}' doesn't exists!".format(from_path))
raise ValueError("The '{}' doesn't exists!".format(from_path))
def load_document(self, document_id, user_manager=None):
"""
Loads a document to the memory.
:param document_id: The ID of the :py:class:Document.
:param user_manager: The :py:class:UserManager of the :py:class:Repository.
:exception DocumentDoesntExistsError is raised when the document is missing from the filesystem,
:return: :py:class:Document object.
"""
document_path = path.join(self._location, str(document_id))
if not path.exists(document_path):
raise DocumentDoesntExistsError(
"The {} path doesn't exists, so the document with {} id can't be loaded!"
.format(document_path, document_id))
else:
metadata_file = reduce(path.join, [
self._location,
str(document_id),
'{}_document_metadata.edd'.format(document_id)
])
meta_data = read_ini_file(metadata_file)
list_of_files = ([
str(file_name.strip("'")) for file_name in
meta_data['document']['files'][1:-1].split(', ')
])
if 'author' in meta_data['document']:
if '[' in meta_data['document']['author'] and ']' in meta_data[
'document']['author']:
list_of_authors = [
int(file_name.strip("'")) for file_name in
meta_data['document']['author'][1:-1].split(', ')
]
else:
list_of_authors = meta_data['document']['author']
else:
if '[' in meta_data['document'][
'author_name'] and ']' in meta_data['document'][
'author_name']:
list_of_authors_by_name = [
file_name.strip("'") for file_name in
meta_data['document']['author_name'][1:-1].split(', ')
]
else:
list_of_authors_by_name = meta_data['document'][
'author_name']
if not isinstance(list_of_authors_by_name, list):
list_of_authors_by_name = [list_of_authors_by_name]
list_of_authors = set()
for author_name in list_of_authors_by_name:
for user_id in user_manager.find_users_by_name(
author_name):
list_of_authors.add(int(user_id))
list_of_authors = list(list_of_authors)
document = Document(meta_data['document']['title'],
meta_data['document']['description'],
list_of_authors, list_of_files,
meta_data['document']['doc_format'])
document.creation_date = datetime.strptime(
meta_data['document']['creation_date'], '%Y/%m/%d %H:%M:%S %f')
document.modification_date = datetime.strptime(
meta_data['document']['modification_date'],
'%Y/%m/%d %H:%M:%S %f')
if 'author' in meta_data['document']:
document.state = meta_data['document']['state']
if meta_data['document']['is_public'] == 'True':
document.make_public()
else:
document.state = 'new'
document.make_private()
return document