def create_job(self, user, for_user, workspace, target, email): """ :return: ID of a new Job :raises UserNotFoundException: if 'for_user' or 'user' doesn't exist in database :raises SearchException: Search raises an exception. See exception's message for details :raises StorageNotAcceptingException: if one of the referenced Storages doesn't accept jobs currently :raises NoAdministratorRightsException: if 'user' can not create jobs for 'for_user' :raises PermissionException: if 'for_user' reached his max. amount of jobs :raises MasterPausedException: if Master is paused """ caller = User.get_user_by_username(user) for_user = User.get_user_by_username(for_user) if for_user else caller target_storj = Storage.get_storage_by_name(target) src_workspace = Workspaces.get_by_full_name(workspace) if src_workspace.username != caller.name and caller.get_user_type() != UserRole.Administrator: raise PermissionError('Unauthorized to move workspaces of other users.') src_storj = Storage.get_storage_by_name(src_workspace.storage) source = src_workspace.full_path addr = search_class.verify_job(source, target) target_path = target_storj.mountpoint + addr['source_relative_path'] copy_options = determine_copytool(src_storj, target_storj) job = Job(addr['source_alias'], target, source, email, for_user, target_path=target_path, copy_options=json.dumps(copy_options)) self.access_handler.check_create_job(job, caller) self.master.add_to_queue(job) return job.get_job_id()
def get_directory_list(usernames: [str], storage_alias: str = None): """Searches given Storage for all Data created by users with given usernames and returns a dictionary where keys are storage aliases and values are lists with relative adresses of directories. :param storage_alias: the unique string Alias of the Storage to be searched (default = None). :param usernames: a List of usernames. :raises StorageAliasNotFoundException: if storage_alias is not found. :raises StorageNotMountedException: if a Storage to be worked on is not currently mounted. """ storage_list = [] # We search all the Storages if storage_alias is None: storage_list = Storage.get_storages() # We search only the specified Storage else: storage_list.append(Storage.get_storage_by_name(storage_alias)) # search for directories in selected storages directories = dict() for storage in storage_list: # assure the storage is accessible if not (check_storage_mount(storage)): raise(StorageNotMountedException("The Storage with the alias >>" + storage.alias + "<< is currently not mounted under " + storage.mountpoint)) abs_paths = search_for_directories(storage, usernames) rel_paths = [] for abs_path in abs_paths: rel_paths.append(resolve_abs_address(abs_path)[1]) # Eliminate storages without directories to show only if storage_alias is None #if storage_alias is None and len(abs_paths) == 0: # pass #else: directories[storage.alias] = rel_paths return directories
def initialize_storages_from_config(): config_path = get_conf_directory() + 'application/storages.ini' if not os.path.exists(config_path): raise IncorrectConfigFileError('Configuration file not found') if not (config_path[-4:] == ".ini"): raise IncorrectConfigFileError( 'Configuration file is in the wrong format') _config = ConfigParser() _config.read(config_path) storages_sections = _config.sections() storages = dict() for section in storages_sections: alias = _config.get(section, 'alias') mountpoint = _config.get(section, 'mountpoint') has_home_path = _config.getboolean(section, 'has_home_path') is_archive = _config.getboolean(section, 'is_archive') storage_type = _config.get(section, 'type') max_extensions = int(_config.get(section, 'max_extensions')) max_extension_period = int(_config.get(section, 'max_extension_period')) description = _config.get(section, 'description') if not Storage.get_storage_by_name(alias): Storage(alias, mountpoint, True, has_home_path, is_archive, max_extensions, max_extension_period, description) return storages
def create_workspace(self, username, ws_label, storage_alias=None): if not storage_alias: storage_alias = self.config_handler.workspace_default_storage storj = Storage.get_storage_by_name(storage_alias) counter = Workspaces.calc_counter(ws_label, username) filename = self.config_handler.assemble_workspace_full_name(username, ws_label, counter) mnt = storj.mountpoint path = mnt + filename + '/' if os.path.exists(path): raise FileExistsError('workspace full_name naming collision, directory still exists') Workspaces( username=username, label=ws_label, storage=storage_alias, max_extension_period=storj.max_extension_period, max_extensions=storj.max_extensions, counter=counter ) os.mkdir(path) uid = pwd.getpwnam(username).pw_uid os.chmod(path, 0o755) os.chown(path, uid, uid) w = Workspaces.get_by_triple(username, ws_label, counter) w.set_full_path(path) return w.full_name
def get_storage_alias_list(self): """ :return: list of available storages :raises NoResponseFromSearchException: if Search is not reachable """ storage_aliases = [s.alias for s in Storage.get_storages()] return dict(storages=storage_aliases)
def resolve_abs_address(source_path: str): """Searches a Storage with same mountpoint as source_paths beginning and returns a List [storage_alias, source_rel_path]. If there is a conflict (storage1.mountpoint being substring of storage2.mountpoint) the storage with longest mountpoint will be selected. :param source_path: The path to be split into storage_alias and relative_path. :raises SourcePathNotValidException: if no matching Storage is found. """ if not os.path.exists(source_path): raise(SourcePathNotValidException("The path " + source_path + " does not exist on the system.")) match_list = [] for storage in Storage.get_storages(): # test if both start with same path if source_path.startswith(storage.mountpoint): match_list.append(storage) if len(match_list) == 0: # executed only if we found no matching storage raise(SourcePathNotValidException("The path " + source_path + " does not match any registered Storages mountpath.")) else: # select storage with longest mountpoint maxlen = 0 match_storage = None for storage in match_list: if len(storage.mountpoint) > maxlen: maxlen = len(storage.mountpoint) match_storage = storage # strip the mount point source_rel_path = source_path[len(match_storage.mountpoint):] return [match_storage.alias, source_rel_path]
def __init__(self, name, status, current_job, address, mountpoints): """ Constructor for the Worker class, saves class attributes and commits to the database :param name: name of the worker :param status: status of the Worker :param current_job_id: id of the active job :param address: the base url :param storages_aliases: List of mounted storages """ self.name = name self.status = status self.address = address if mountpoints is not None: mountpoints = list(set(mountpoints)) for mountpoint in mountpoints: storage = Storage.get_storage_by_mountpoint(mountpoint) if storage is not None: meta.get_session(storage).close() self.storages.append(storage) else: pass if current_job is not None: self.current_job_id = current_job.id else: self.current_job_id = None session = meta.get_session(self) session.add(self) session.commit()
def _get_home_storages(): """Returns a List of all mounted Storages with home directories.""" home_storages = Storage.get_home_storages() for storage in home_storages: # assure the storages we want to work on actually function ;) if not check_storage_mount(storage): raise(StorageNotMountedException("The Storage with the alias >>" + storage.alias + "<< is currently not mounted under " + storage.path)) return home_storages
def verify_job(source_absolute_path: str, target_alias: str): """If the job is valid, returns a List [source_alias, source_rel_path, target_alias]. Else throws an Exception. :param source_absolute_path: The path of the directory to be moved. :param target_alias: The alias of the Storage the directory is to be moved to. :raises SourcePathNotValidException: if the path does not exist in the system. :raises SourcePathNotValidException: if no Storage mounted at source_absolute_paths beginning. :raises StorageAliasNotFoundException: if no Storage with target_alias is found. :raises StorageNotMountedException: if either one of the involved Storages is not currently mounted. :raises StorageTypeCompatibilityException: if the Storage types of source and target are incompatible. :raises StorageNotAcceptingException: if one of the referenced Storages doesn't currently accept jobs. """ if not os.path.isdir(source_absolute_path): raise(SourcePathNotValidException("The path " + source_absolute_path + " is not a directory.")) src_alias, source_rel_path = resolve_abs_address(source_absolute_path) target_storage = Storage.get_storage_by_name(target_alias) source_storage = Storage.get_storage_by_name(src_alias) if target_storage is None: raise AttributeError('Storage with alias ' + target_alias + ' not found.') verify_target(source_storage) verify_target(target_storage) if not check_storage_type_compatibility(source_storage, target_storage): raise(StorageTypeCompatibilityException("The storage Types are not compatible.")) if not source_storage.accept_jobs: raise(StorageNotAcceptingException("The Storage " + source_storage.alias + " does not currently accept jobs.")) elif not target_storage.accept_jobs: raise(StorageNotAcceptingException("The Storage " + target_storage.alias + " does not currently accept jobs.")) else: return { 'source_alias' : source_storage.alias, 'source_relative_path': source_rel_path, 'target_alias' : target_storage.alias }
def get_storage(self, alias): return build_storages([Storage.get_storage_by_name(alias)])
def get_storages(self): return build_storages(Storage.get_storages())
def initialize_storage_db(): initialize_storages_from_config() check_storages_mount(Storage.get_storages())