def create_job(self, user, for_user, workspace, target, email):
"""
:return: ID of a new Job
:raises UserNotFoundException: if 'for_user' or 'user' doesn't exist in database
:raises SearchException: Search raises an exception. See exception's message for details
:raises StorageNotAcceptingException: if one of the referenced Storages doesn't accept jobs currently
:raises NoAdministratorRightsException: if 'user' can not create jobs for 'for_user'
:raises PermissionException: if 'for_user' reached his max. amount of jobs
:raises MasterPausedException: if Master is paused
"""
caller = User.get_user_by_username(user)
for_user = User.get_user_by_username(for_user) if for_user else caller
target_storj = Storage.get_storage_by_name(target)
src_workspace = Workspaces.get_by_full_name(workspace)
if src_workspace.username != caller.name and caller.get_user_type() != UserRole.Administrator:
raise PermissionError('Unauthorized to move workspaces of other users.')
src_storj = Storage.get_storage_by_name(src_workspace.storage)
source = src_workspace.full_path
addr = search_class.verify_job(source, target)
target_path = target_storj.mountpoint + addr['source_relative_path']
copy_options = determine_copytool(src_storj, target_storj)
job = Job(addr['source_alias'], target, source, email, for_user, target_path=target_path, copy_options=json.dumps(copy_options))
self.access_handler.check_create_job(job, caller)
self.master.add_to_queue(job)
return job.get_job_id()
def get_directory_list(usernames: [str], storage_alias: str = None):
"""Searches given Storage for all Data created by users with given usernames and returns a dictionary where keys are storage aliases and values are lists with relative adresses of directories.
:param storage_alias: the unique string Alias of the Storage to be searched (default = None).
:param usernames: a List of usernames.
:raises StorageAliasNotFoundException: if storage_alias is not found.
:raises StorageNotMountedException: if a Storage to be worked on is not currently mounted.
"""
storage_list = []
# We search all the Storages
if storage_alias is None:
storage_list = Storage.get_storages()
# We search only the specified Storage
else:
storage_list.append(Storage.get_storage_by_name(storage_alias))
# search for directories in selected storages
directories = dict()
for storage in storage_list:
# assure the storage is accessible
if not (check_storage_mount(storage)):
raise(StorageNotMountedException("The Storage with the alias >>" + storage.alias +
"<< is currently not mounted under " + storage.mountpoint))
abs_paths = search_for_directories(storage, usernames)
rel_paths = []
for abs_path in abs_paths:
rel_paths.append(resolve_abs_address(abs_path)[1])
# Eliminate storages without directories to show only if storage_alias is None
#if storage_alias is None and len(abs_paths) == 0:
# pass
#else:
directories[storage.alias] = rel_paths
return directories
def initialize_storages_from_config():
config_path = get_conf_directory() + 'application/storages.ini'
if not os.path.exists(config_path):
raise IncorrectConfigFileError('Configuration file not found')
if not (config_path[-4:] == ".ini"):
raise IncorrectConfigFileError(
'Configuration file is in the wrong format')
_config = ConfigParser()
_config.read(config_path)
storages_sections = _config.sections()
storages = dict()
for section in storages_sections:
alias = _config.get(section, 'alias')
mountpoint = _config.get(section, 'mountpoint')
has_home_path = _config.getboolean(section, 'has_home_path')
is_archive = _config.getboolean(section, 'is_archive')
storage_type = _config.get(section, 'type')
max_extensions = int(_config.get(section, 'max_extensions'))
max_extension_period = int(_config.get(section,
'max_extension_period'))
description = _config.get(section, 'description')
if not Storage.get_storage_by_name(alias):
Storage(alias, mountpoint, True, has_home_path, is_archive,
max_extensions, max_extension_period, description)
return storages
def create_workspace(self, username, ws_label, storage_alias=None):
if not storage_alias: storage_alias = self.config_handler.workspace_default_storage
storj = Storage.get_storage_by_name(storage_alias)
counter = Workspaces.calc_counter(ws_label, username)
filename = self.config_handler.assemble_workspace_full_name(username, ws_label, counter)
mnt = storj.mountpoint
path = mnt + filename + '/'
if os.path.exists(path): raise FileExistsError('workspace full_name naming collision, directory still exists')
Workspaces(
username=username,
label=ws_label,
storage=storage_alias,
max_extension_period=storj.max_extension_period,
max_extensions=storj.max_extensions,
counter=counter
)
os.mkdir(path)
uid = pwd.getpwnam(username).pw_uid
os.chmod(path, 0o755)
os.chown(path, uid, uid)
w = Workspaces.get_by_triple(username, ws_label, counter)
w.set_full_path(path)
return w.full_name
def get_storage_alias_list(self):
"""
:return: list of available storages
:raises NoResponseFromSearchException: if Search is not reachable
"""
storage_aliases = [s.alias for s in Storage.get_storages()]
return dict(storages=storage_aliases)
def resolve_abs_address(source_path: str):
"""Searches a Storage with same mountpoint as source_paths beginning and returns a List
[storage_alias, source_rel_path].
If there is a conflict (storage1.mountpoint being substring of storage2.mountpoint) the storage with longest
mountpoint will be selected.
:param source_path: The path to be split into storage_alias and relative_path.
:raises SourcePathNotValidException: if no matching Storage is found.
"""
if not os.path.exists(source_path):
raise(SourcePathNotValidException("The path " + source_path +
" does not exist on the system."))
match_list = []
for storage in Storage.get_storages():
# test if both start with same path
if source_path.startswith(storage.mountpoint):
match_list.append(storage)
if len(match_list) == 0:
# executed only if we found no matching storage
raise(SourcePathNotValidException("The path " + source_path +
" does not match any registered Storages mountpath."))
else:
# select storage with longest mountpoint
maxlen = 0
match_storage = None
for storage in match_list:
if len(storage.mountpoint) > maxlen:
maxlen = len(storage.mountpoint)
match_storage = storage
# strip the mount point
source_rel_path = source_path[len(match_storage.mountpoint):]
return [match_storage.alias, source_rel_path]
def __init__(self, name, status, current_job, address, mountpoints):
"""
Constructor for the Worker class, saves class attributes and commits to the database
:param name: name of the worker
:param status: status of the Worker
:param current_job_id: id of the active job
:param address: the base url
:param storages_aliases: List of mounted storages
"""
self.name = name
self.status = status
self.address = address
if mountpoints is not None:
mountpoints = list(set(mountpoints))
for mountpoint in mountpoints:
storage = Storage.get_storage_by_mountpoint(mountpoint)
if storage is not None:
meta.get_session(storage).close()
self.storages.append(storage)
else:
pass
if current_job is not None:
self.current_job_id = current_job.id
else:
self.current_job_id = None
session = meta.get_session(self)
session.add(self)
session.commit()
def _get_home_storages():
"""Returns a List of all mounted Storages with home directories."""
home_storages = Storage.get_home_storages()
for storage in home_storages:
# assure the storages we want to work on actually function ;)
if not check_storage_mount(storage):
raise(StorageNotMountedException("The Storage with the alias >>" + storage.alias +
"<< is currently not mounted under " + storage.path))
return home_storages
def verify_job(source_absolute_path: str, target_alias: str):
"""If the job is valid, returns a List [source_alias, source_rel_path, target_alias].
Else throws an Exception.
:param source_absolute_path: The path of the directory to be moved.
:param target_alias: The alias of the Storage the directory is to be moved to.
:raises SourcePathNotValidException: if the path does not exist in the system.
:raises SourcePathNotValidException: if no Storage mounted at source_absolute_paths beginning.
:raises StorageAliasNotFoundException: if no Storage with target_alias is found.
:raises StorageNotMountedException: if either one of the involved Storages is not currently mounted.
:raises StorageTypeCompatibilityException: if the Storage types of source and target are incompatible.
:raises StorageNotAcceptingException: if one of the referenced Storages doesn't currently accept jobs.
"""
if not os.path.isdir(source_absolute_path):
raise(SourcePathNotValidException("The path " + source_absolute_path + " is not a directory."))
src_alias, source_rel_path = resolve_abs_address(source_absolute_path)
target_storage = Storage.get_storage_by_name(target_alias)
source_storage = Storage.get_storage_by_name(src_alias)
if target_storage is None: raise AttributeError('Storage with alias ' + target_alias + ' not found.')
verify_target(source_storage)
verify_target(target_storage)
if not check_storage_type_compatibility(source_storage, target_storage):
raise(StorageTypeCompatibilityException("The storage Types are not compatible."))
if not source_storage.accept_jobs:
raise(StorageNotAcceptingException("The Storage " + source_storage.alias +
" does not currently accept jobs."))
elif not target_storage.accept_jobs:
raise(StorageNotAcceptingException("The Storage " + target_storage.alias +
" does not currently accept jobs."))
else:
return {
'source_alias' : source_storage.alias,
'source_relative_path': source_rel_path,
'target_alias' : target_storage.alias
}
def get_storage(self, alias):
return build_storages([Storage.get_storage_by_name(alias)])
def get_storages(self):
return build_storages(Storage.get_storages())
def initialize_storage_db():
initialize_storages_from_config()
check_storages_mount(Storage.get_storages())