def validate_server_directory_upload(trans, server_dir): if server_dir in [None, 'None', '']: raise RequestParameterInvalidException("Invalid or unspecified server_dir parameter") if trans.user_is_admin: import_dir = trans.app.config.library_import_dir import_dir_desc = 'library_import_dir' if not import_dir: raise ConfigDoesNotAllowException('"library_import_dir" is not set in the Galaxy configuration') else: import_dir = trans.app.config.user_library_import_dir if not import_dir: raise ConfigDoesNotAllowException('"user_library_import_dir" is not set in the Galaxy configuration') if server_dir != trans.user.email: import_dir = os.path.join(import_dir, trans.user.email) import_dir_desc = 'user_library_import_dir' full_dir = os.path.join(import_dir, server_dir) unsafe = None if safe_relpath(server_dir): username = trans.user.username if trans.app.config.user_library_import_check_permissions else None if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, whitelist=trans.app.config.user_library_import_symlink_whitelist): for unsafe in unsafe_walk(full_dir, whitelist=[import_dir] + trans.app.config.user_library_import_symlink_whitelist, username=username): log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe)) else: log.error('User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s', server_dir, os.path.realpath(full_dir)) unsafe = True if unsafe: raise RequestParameterInvalidException("Invalid server_dir specified") return full_dir, import_dir_desc
def index(self, trans, shed, owner, repo, tool, version, image_file): """ Open an image file that is contained in an installed tool shed repository or that is referenced by a URL for display. The image can be defined in either a README.rst file contained in the repository or the help section of a Galaxy tool config that is contained in the repository. The following image definitions are all supported. The former $PATH_TO_IMAGES is no longer required, and is now ignored. .. image:: https://raw.github.com/galaxy/some_image.png .. image:: $PATH_TO_IMAGES/some_image.png .. image:: /static/images/some_image.gif .. image:: some_image.jpg .. image:: /deep/some_image.png """ guid = '/'.join([shed, 'repos', owner, repo, tool, version]) tool = trans.app.toolbox.get_tool(guid) repo_path = tool._repository_dir if 'static/images' not in image_file: path = join(repo_path, 'static', 'images', image_file) else: path = join(repo_path, image_file) if not safe_contains(os.path.abspath(repo_path), os.path.abspath(path)): raise RequestParameterInvalidException() ext = os.path.splitext(image_file)[-1].lstrip('.') if ext: mime = trans.app.datatypes_registry.get_mimetype_by_extension(ext) if mime: trans.response.set_content_type(mime) if os.path.exists(path): return open(path, 'rb')
def validate_server_directory_upload(trans, server_dir): if server_dir in [None, 'None', '']: raise RequestParameterInvalidException("Invalid or unspecified server_dir parameter") if trans.user_is_admin(): import_dir = trans.app.config.library_import_dir import_dir_desc = 'library_import_dir' if not import_dir: raise ConfigDoesNotAllowException('"library_import_dir" is not set in the Galaxy configuration') else: import_dir = trans.app.config.user_library_import_dir if not import_dir: raise ConfigDoesNotAllowException('"user_library_import_dir" is not set in the Galaxy configuration') if server_dir != trans.user.email: import_dir = os.path.join(import_dir, trans.user.email) import_dir_desc = 'user_library_import_dir' full_dir = os.path.join(import_dir, server_dir) unsafe = None if safe_relpath(server_dir): username = trans.user.username if trans.app.config.user_library_import_check_permissions else None if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, whitelist=trans.app.config.user_library_import_symlink_whitelist, username=username): for unsafe in unsafe_walk(full_dir, whitelist=[import_dir] + trans.app.config.user_library_import_symlink_whitelist): log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe)) else: log.error('User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s', server_dir, os.path.realpath(full_dir)) unsafe = True if unsafe: raise RequestParameterInvalidException("Invalid server_dir specified") return full_dir, import_dir_desc
def _realize_to(self, source_path, native_path, user_context=None): effective_root = self._effective_root(user_context) source_native_path = self._to_native_path(source_path, user_context=user_context) if self.enforce_symlink_security: if not safe_contains(effective_root, source_native_path, allowlist=self._allowlist): raise Exception("Operation not allowed.") else: source_native_path = os.path.normpath(source_native_path) assert source_native_path.startswith(os.path.normpath(effective_root)) if not self.delete_on_realize: shutil.copyfile(source_native_path, native_path) else: shutil.move(source_native_path, native_path)
def _write_from(self, target_path, native_path, user_context=None): effective_root = self._effective_root(user_context) target_native_path = self._to_native_path(target_path, user_context=user_context) if self.enforce_symlink_security: if not safe_contains(effective_root, target_native_path, allowlist=self._allowlist): raise Exception("Operation not allowed.") else: target_native_path = os.path.normpath(target_native_path) assert target_native_path.startswith(os.path.normpath(effective_root)) target_native_path_parent = os.path.dirname(target_native_path) if not os.path.exists(target_native_path_parent): if self.allow_subdir_creation: os.makedirs(target_native_path_parent) else: raise Exception("Parent directory does not exist.") shutil.copyfile(native_path, target_native_path)
def _asset_exists_and_is_safe(repo_path, asset_path): if not safe_contains(repo_path, asset_path): raise RequestParameterInvalidException() return os.path.exists(asset_path)
def load(self, trans, payload=None, **kwd): """ Load dataset(s) from the given source into the library. * POST /api/libraries/datasets :param payload: dictionary structure containing: :param encoded_folder_id: the encoded id of the folder to import dataset(s) to :type encoded_folder_id: an encoded id string :param source: source the datasets should be loaded from Source can be: user directory - root folder specified in galaxy.ini as "$user_library_import_dir" example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here} the folder with the user login has to be created beforehand (admin)import directory - root folder specified in galaxy ini as "$library_import_dir" example path: path/to/galaxy/$library_import_dir/{admin can browse everything here} (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini :type source: str :param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy while linking is set to True all symlinks will be resolved _once_ :type link_data: bool :param preserve_dirs: flag whether to preserve the directory structure when importing dir if False only datasets will be imported :type preserve_dirs: bool :param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect) :type file_type: str :param dbkey: dbkey of the loaded genome, defaults to '?' (unknown) :type dbkey: str :param tag_using_filenames: flag whether to generate dataset tags from filenames :type tag_using_filenames: bool :type dictionary :returns: dict containing information about the created upload job :rtype: dictionary :raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException InsufficientPermissionsException, ObjectNotFound """ if payload: kwd.update(payload) kwd['space_to_tab'] = False kwd['to_posix_lines'] = True kwd['dbkey'] = kwd.get('dbkey', '?') kwd['file_type'] = kwd.get('file_type', 'auto') kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(kwd.get('link_data', False)) else 'copy_files' kwd['tag_using_filenames'] = util.string_as_bool(kwd.get('tag_using_filenames', None)) encoded_folder_id = kwd.get('encoded_folder_id', None) if encoded_folder_id is not None: folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id) else: raise exceptions.RequestParameterMissingException('The required attribute encoded_folder_id is missing.') path = kwd.get('path', None) if path is None: raise exceptions.RequestParameterMissingException('The required attribute path is missing.') folder = self.folder_manager.get(trans, folder_id) source = kwd.get('source', None) if source not in ['userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path']: raise exceptions.RequestParameterMissingException('You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ') elif source in ['importdir_file', 'importdir_folder']: if not trans.user_is_admin(): raise exceptions.AdminRequiredException('Only admins can import from importdir.') if not trans.app.config.library_import_dir: raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from importdir.') import_base_dir = trans.app.config.library_import_dir if not safe_relpath(path): # admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed. # the reasoning here is that galaxy admins may not have direct filesystem access or can only access # library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the # import directory. if they have filesystem access, all bets are off. raise exceptions.RequestParameterInvalidException('The given path is invalid.') path = os.path.join(import_base_dir, path) elif source in ['userdir_file', 'userdir_folder']: unsafe = None user_login = trans.user.email user_base_dir = trans.app.config.user_library_import_dir if user_base_dir is None: raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow upload from user directories.') full_dir = os.path.join(user_base_dir, user_login) if not safe_contains(full_dir, path, whitelist=trans.app.config.user_library_import_symlink_whitelist): # the path is a symlink outside the user dir path = os.path.join(full_dir, path) log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path)) raise exceptions.RequestParameterInvalidException('The given path is invalid.') path = os.path.join(full_dir, path) for unsafe in unsafe_walk(path, whitelist=[full_dir] + trans.app.config.user_library_import_symlink_whitelist): # the path is a dir and contains files that symlink outside the user dir log.error('User attempted to import a directory containing a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe)) if unsafe: raise exceptions.RequestParameterInvalidException('The given path is invalid.') if not os.path.exists(path): raise exceptions.RequestParameterInvalidException('Given path does not exist on the host.') if not self.folder_manager.can_add_item(trans, folder): raise exceptions.InsufficientPermissionsException('You do not have proper permission to add items to the given folder.') elif source == 'admin_path': if not trans.app.config.allow_library_path_paste: raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from path.') if not trans.user_is_admin(): raise exceptions.AdminRequiredException('Only admins can import from path.') # Set up the traditional tool state/params tool_id = 'upload1' tool = trans.app.toolbox.get_tool(tool_id) state = tool.new_state(trans) populate_state(trans, tool.inputs, kwd, state.inputs) tool_params = state.inputs dataset_upload_inputs = [] for input in tool.inputs.itervalues(): if input.type == "upload_dataset": dataset_upload_inputs.append(input) library_bunch = upload_common.handle_library_params(trans, {}, trans.security.encode_id(folder.id)) abspath_datasets = [] kwd['filesystem_paths'] = path if source in ['importdir_folder']: kwd['filesystem_paths'] = os.path.join(import_base_dir, path) # user wants to import one file only elif source in ["userdir_file", "importdir_file"]: file = os.path.abspath(path) abspath_datasets.append(trans.webapp.controllers['library_common'].make_library_uploaded_dataset( trans, 'api', kwd, os.path.basename(file), file, 'server_dir', library_bunch)) # user wants to import whole folder elif source == "userdir_folder": uploaded_datasets_bunch = trans.webapp.controllers['library_common'].get_path_paste_uploaded_datasets( trans, 'api', kwd, library_bunch, 200, '') uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound('Given folder does not contain any datasets.') for ud in uploaded_datasets: ud.path = os.path.abspath(ud.path) abspath_datasets.append(ud) # user wants to import from path if source in ["admin_path", "importdir_folder"]: # validate the path is within root uploaded_datasets_bunch = trans.webapp.controllers['library_common'].get_path_paste_uploaded_datasets( trans, 'api', kwd, library_bunch, 200, '') uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound('Given folder does not contain any datasets.') for ud in uploaded_datasets: ud.path = os.path.abspath(ud.path) abspath_datasets.append(ud) json_file_path = upload_common.create_paramfile(trans, abspath_datasets) data_list = [ud.data for ud in abspath_datasets] job_params = {} job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files')) job_params['uuid'] = dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params) trans.sa_session.add(job) trans.sa_session.flush() job_dict = job.to_dict() job_dict['id'] = trans.security.encode_id(job_dict['id']) return job_dict
def load(self, trans, payload=None, **kwd): """ POST /api/libraries/datasets Load dataset(s) from the given source into the library. :param payload: dictionary structure containing: :param encoded_folder_id: the encoded id of the folder to import dataset(s) to :type encoded_folder_id: an encoded id string :param source: source the datasets should be loaded from. Source can be: - user directory root folder specified in galaxy.ini as "$user_library_import_dir" example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here} the folder with the user login has to be created beforehand - (admin)import directory root folder specified in galaxy ini as "$library_import_dir" example path: path/to/galaxy/$library_import_dir/{admin can browse everything here} - (admin)any absolute or relative path option allowed with "allow_library_path_paste" in galaxy.ini :type source: str :param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy while linking is set to True all symlinks will be resolved _once_ :type link_data: bool :param preserve_dirs: flag whether to preserve the directory structure when importing dir if False only datasets will be imported :type preserve_dirs: bool :param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect) :type file_type: str :param dbkey: dbkey of the loaded genome, defaults to '?' (unknown) :type dbkey: str :param tag_using_filenames: flag whether to generate dataset tags from filenames :type tag_using_filenames: bool :type dictionary :returns: dict containing information about the created upload job :rtype: dictionary :raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException InsufficientPermissionsException, ObjectNotFound """ if payload: kwd.update(payload) kwd['space_to_tab'] = False kwd['to_posix_lines'] = True kwd['dbkey'] = kwd.get('dbkey', '?') kwd['file_type'] = kwd.get('file_type', 'auto') kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(kwd.get('link_data', False)) else 'copy_files' kwd['tag_using_filenames'] = util.string_as_bool(kwd.get('tag_using_filenames', None)) encoded_folder_id = kwd.get('encoded_folder_id', None) if encoded_folder_id is not None: folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id) else: raise exceptions.RequestParameterMissingException('The required attribute encoded_folder_id is missing.') path = kwd.get('path', None) if path is None: raise exceptions.RequestParameterMissingException('The required attribute path is missing.') if not isinstance(path, str): raise exceptions.RequestParameterInvalidException('The required attribute path is not String.') folder = self.folder_manager.get(trans, folder_id) source = kwd.get('source', None) if source not in ['userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path']: raise exceptions.RequestParameterMissingException('You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ') elif source in ['importdir_file', 'importdir_folder']: if not trans.user_is_admin: raise exceptions.AdminRequiredException('Only admins can import from importdir.') if not trans.app.config.library_import_dir: raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from importdir.') import_base_dir = trans.app.config.library_import_dir if not safe_relpath(path): # admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed. # the reasoning here is that galaxy admins may not have direct filesystem access or can only access # library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the # import directory. if they have filesystem access, all bets are off. raise exceptions.RequestParameterInvalidException('The given path is invalid.') path = os.path.join(import_base_dir, path) elif source in ['userdir_file', 'userdir_folder']: username = trans.user.username if trans.app.config.user_library_import_check_permissions else None user_login = trans.user.email user_base_dir = trans.app.config.user_library_import_dir if user_base_dir is None: raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow upload from user directories.') full_dir = os.path.join(user_base_dir, user_login) if not safe_contains(full_dir, path, allowlist=trans.app.config.user_library_import_symlink_allowlist): # the path is a symlink outside the user dir path = os.path.join(full_dir, path) log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path)) raise exceptions.RequestParameterInvalidException('The given path is invalid.') if trans.app.config.user_library_import_check_permissions and not full_path_permission_for_user(full_dir, path, username): log.error('User attempted to import a path that resolves to a path outside of their import dir: ' '%s -> %s and cannot be read by them.', path, os.path.realpath(path)) raise exceptions.RequestParameterInvalidException('The given path is invalid.') path = os.path.join(full_dir, path) if unsafe_walk(path, allowlist=[full_dir] + trans.app.config.user_library_import_symlink_allowlist, username=username): # the path is a dir and contains files that symlink outside the user dir error = 'User attempted to import a path that resolves to a path outside of their import dir: {} -> {}'.format( path, os.path.realpath(path) ) if trans.app.config.user_library_import_check_permissions: error += ' or is not readable for them.' log.error(error) raise exceptions.RequestParameterInvalidException('The given path is invalid.') if not os.path.exists(path): raise exceptions.RequestParameterInvalidException('Given path does not exist on the host.') if not self.folder_manager.can_add_item(trans, folder): raise exceptions.InsufficientPermissionsException('You do not have proper permission to add items to the given folder.') elif source == 'admin_path': if not trans.app.config.allow_library_path_paste: raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from path.') if not trans.user_is_admin: raise exceptions.AdminRequiredException('Only admins can import from path.') # Set up the traditional tool state/params tool_id = 'upload1' tool = trans.app.toolbox.get_tool(tool_id) state = tool.new_state(trans) populate_state(trans, tool.inputs, kwd, state.inputs) tool_params = state.inputs dataset_upload_inputs = [] for input in tool.inputs.values(): if input.type == "upload_dataset": dataset_upload_inputs.append(input) library_bunch = upload_common.handle_library_params(trans, {}, trans.security.encode_id(folder.id)) abspath_datasets = [] kwd['filesystem_paths'] = path if source in ['importdir_folder']: kwd['filesystem_paths'] = os.path.join(import_base_dir, path) # user wants to import one file only elif source in ["userdir_file", "importdir_file"]: file = os.path.abspath(path) abspath_datasets.append(self._make_library_uploaded_dataset( trans, kwd, os.path.basename(file), file, 'server_dir', library_bunch)) # user wants to import whole folder elif source == "userdir_folder": uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets( trans, kwd, library_bunch, 200, '') uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound('Given folder does not contain any datasets.') for ud in uploaded_datasets: ud.path = os.path.abspath(ud.path) abspath_datasets.append(ud) # user wants to import from path if source in ["admin_path", "importdir_folder"]: # validate the path is within root uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets( trans, kwd, library_bunch, 200, '') uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound('Given folder does not contain any datasets.') for ud in uploaded_datasets: ud.path = os.path.abspath(ud.path) abspath_datasets.append(ud) json_file_path = upload_common.create_paramfile(trans, abspath_datasets) data_list = [ud.data for ud in abspath_datasets] job_params = {} job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files')) job_params['uuid'] = dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params) trans.app.job_manager.enqueue(job, tool=tool) job_dict = job.to_dict() job_dict['id'] = trans.security.encode_id(job_dict['id']) return job_dict