def build_module(self, trans, payload={}):
"""
POST /api/workflows/build_module
Builds module models for the workflow editor.
"""
inputs = payload.get('inputs', {})
trans.workflow_building_mode = workflow_building_modes.ENABLED
module = module_factory.from_dict(trans, payload)
if 'tool_state' not in payload:
module_state = {}
populate_state(trans,
module.get_inputs(),
inputs,
module_state,
check=False)
module.recover_state(module_state)
return {
'label': inputs.get('__label', ''),
'annotation': inputs.get('__annotation', ''),
'name': module.get_name(),
'tool_state': module.get_state(),
'inputs': module.get_all_inputs(connectable_only=True),
'outputs': module.get_all_outputs(),
'config_form': module.get_config_form(),
'post_job_actions': module.get_post_job_actions(inputs)
}
def __get_tool_extensions(self, trans, tool_id):
"""
Get the input and output extensions of a tool
"""
payload = {'type': 'tool', 'tool_id': tool_id, '_': 'true'}
inputs = payload.get('inputs', {})
trans.workflow_building_mode = workflow_building_modes.ENABLED
module = module_factory.from_dict(trans, payload)
if 'tool_state' not in payload:
module_state = {}
populate_state(trans,
module.get_inputs(),
inputs,
module_state,
check=False)
module.recover_state(module_state)
inputs = module.get_all_inputs(connectable_only=True)
outputs = module.get_all_outputs()
input_extensions = list()
output_extensions = list()
for i_ext in inputs:
input_extensions.extend(i_ext['extensions'])
for o_ext in outputs:
output_extensions.extend(o_ext['extensions'])
return input_extensions, output_extensions
def _upload_dataset(self, trans, library_id, folder_id, replace_dataset=None, **kwd):
# Set up the traditional tool state/params
cntrller = 'api'
tool_id = 'upload1'
message = None
file_type = kwd.get('file_type')
try:
upload_common.validate_datatype_extension(datatypes_registry=trans.app.datatypes_registry, ext=file_type)
except RequestParameterInvalidException as e:
return (400, util.unicodify(e))
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.items():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
# Library-specific params
server_dir = kwd.get('server_dir', '')
upload_option = kwd.get('upload_option', 'upload_file')
response_code = 200
if upload_option == 'upload_directory':
full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir)
message = 'Select a directory'
elif upload_option == 'upload_paths':
# Library API already checked this - following check isn't actually needed.
validate_path_upload(trans)
# Some error handling should be added to this method.
try:
# FIXME: instead of passing params here ( which have been processed by util.Params(), the original kwd
# should be passed so that complex objects that may have been included in the initial request remain.
library_bunch = upload_common.handle_library_params(trans, kwd, folder_id, replace_dataset)
except Exception:
response_code = 500
message = "Unable to parse upload parameters, please report this error."
# Proceed with (mostly) regular upload processing if we're still errorless
if response_code == 200:
if upload_option == 'upload_file':
tool_params = upload_common.persist_uploads(tool_params, trans)
uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, dataset_upload_inputs, library_bunch=library_bunch)
elif upload_option == 'upload_directory':
uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)
elif upload_option == 'upload_paths':
uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(trans, kwd, library_bunch, response_code, message)
if upload_option == 'upload_file' and not uploaded_datasets:
response_code = 400
message = 'Select a file, enter a URL or enter text'
if response_code != 200:
return (response_code, message)
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
job_params = {}
job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = json.dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
return output
def _upload_dataset(self, trans, library_id, folder_id, replace_dataset=None, **kwd):
# Set up the traditional tool state/params
cntrller = 'api'
tool_id = 'upload1'
message = None
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.items():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
# Library-specific params
server_dir = kwd.get('server_dir', '')
upload_option = kwd.get('upload_option', 'upload_file')
response_code = 200
if upload_option == 'upload_directory':
full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir)
message = 'Select a directory'
elif upload_option == 'upload_paths':
# Library API already checked this - following check isn't actually needed.
validate_path_upload(trans)
# Some error handling should be added to this method.
try:
# FIXME: instead of passing params here ( which have been processed by util.Params(), the original kwd
# should be passed so that complex objects that may have been included in the initial request remain.
library_bunch = upload_common.handle_library_params(trans, kwd, folder_id, replace_dataset)
except Exception:
response_code = 500
message = "Unable to parse upload parameters, please report this error."
# Proceed with (mostly) regular upload processing if we're still errorless
if response_code == 200:
precreated_datasets = upload_common.get_precreated_datasets(trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller)
if upload_option == 'upload_file':
tool_params = upload_common.persist_uploads(tool_params, trans)
uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch)
elif upload_option == 'upload_directory':
uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)
elif upload_option == 'upload_paths':
uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(trans, kwd, library_bunch, response_code, message)
upload_common.cleanup_unused_precreated_datasets(precreated_datasets)
if upload_option == 'upload_file' and not uploaded_datasets:
response_code = 400
message = 'Select a file, enter a URL or enter text'
if response_code != 200:
return (response_code, message)
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
job_params = {}
job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = json.dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
return output
def build_module(self, trans, payload={}):
"""
POST /api/workflows/build_module
Builds module models for the workflow editor.
"""
inputs = payload.get('inputs', {})
module = module_factory.from_dict(trans, payload)
module_state = {}
populate_state(trans, module.get_inputs(), inputs, module_state, check=False)
module.recover_state(module_state)
return {
'label' : inputs.get('__label', ''),
'annotation' : inputs.get('__annotation', ''),
'name' : module.get_name(),
'tool_state' : module.get_state(),
'data_inputs' : module.get_data_inputs(),
'data_outputs' : module.get_data_outputs(),
'config_form' : module.get_config_form(),
'post_job_actions' : module.get_post_job_actions(inputs)
}
def build_module( self, trans, payload={} ):
"""
POST /api/workflows/build_module
Builds module models for the workflow editor.
"""
inputs = payload.get( 'inputs', {} )
module = module_factory.from_dict( trans, payload )
module_state = {}
populate_state( trans, module.get_inputs(), inputs, module_state, check=False )
module.recover_state( module_state )
return {
'label' : inputs.get( '__label', '' ),
'annotation' : inputs.get( '__annotation', '' ),
'name' : module.get_name(),
'tool_state' : module.get_state(),
'data_inputs' : module.get_data_inputs(),
'data_outputs' : module.get_data_outputs(),
'config_form' : module.get_config_form(),
'post_job_actions' : module.get_post_job_actions( inputs )
}
def build_module(self, trans, payload={}):
"""
POST /api/workflows/build_module
Builds module models for the workflow editor.
"""
inputs = payload.get('inputs', {})
trans.workflow_building_mode = workflow_building_modes.ENABLED
module = module_factory.from_dict(trans, payload)
if 'tool_state' not in payload:
module_state = {}
populate_state(trans, module.get_inputs(), inputs, module_state, check=False)
module.recover_state(module_state)
return {
'label' : inputs.get('__label', ''),
'annotation' : inputs.get('__annotation', ''),
'name' : module.get_name(),
'tool_state' : module.get_state(),
'inputs' : module.get_all_inputs(connectable_only=True),
'outputs' : module.get_all_outputs(),
'config_form' : module.get_config_form(),
'post_job_actions' : module.get_post_job_actions(inputs)
}
def load(self, trans, payload=None, **kwd):
"""
Load dataset(s) from the given source into the library.
* POST /api/libraries/datasets
:param payload: dictionary structure containing:
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source: source the datasets should be loaded from
Source can be:
user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
(admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
(admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs: flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:param tag_using_filenames: flag whether to generate dataset tags from filenames
:type tag_using_filenames: bool
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
:raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
InsufficientPermissionsException, ObjectNotFound
"""
if payload:
kwd.update(payload)
kwd['space_to_tab'] = False
kwd['to_posix_lines'] = True
kwd['dbkey'] = kwd.get('dbkey', '?')
kwd['file_type'] = kwd.get('file_type', 'auto')
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(kwd.get('link_data', False)) else 'copy_files'
kwd['tag_using_filenames'] = util.string_as_bool(kwd.get('tag_using_filenames', None))
encoded_folder_id = kwd.get('encoded_folder_id', None)
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
else:
raise exceptions.RequestParameterMissingException('The required attribute encoded_folder_id is missing.')
path = kwd.get('path', None)
if path is None:
raise exceptions.RequestParameterMissingException('The required attribute path is missing.')
folder = self.folder_manager.get(trans, folder_id)
source = kwd.get('source', None)
if source not in ['userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path']:
raise exceptions.RequestParameterMissingException('You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
elif source in ['importdir_file', 'importdir_folder']:
if not trans.user_is_admin():
raise exceptions.AdminRequiredException('Only admins can import from importdir.')
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from importdir.')
import_base_dir = trans.app.config.library_import_dir
if not safe_relpath(path):
# admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed.
# the reasoning here is that galaxy admins may not have direct filesystem access or can only access
# library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the
# import directory. if they have filesystem access, all bets are off.
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(import_base_dir, path)
elif source in ['userdir_file', 'userdir_folder']:
unsafe = None
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow upload from user directories.')
full_dir = os.path.join(user_base_dir, user_login)
if not safe_contains(full_dir, path, whitelist=trans.app.config.user_library_import_symlink_whitelist):
# the path is a symlink outside the user dir
path = os.path.join(full_dir, path)
log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path))
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(full_dir, path)
for unsafe in unsafe_walk(path, whitelist=[full_dir] + trans.app.config.user_library_import_symlink_whitelist):
# the path is a dir and contains files that symlink outside the user dir
log.error('User attempted to import a directory containing a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe))
if unsafe:
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
if not os.path.exists(path):
raise exceptions.RequestParameterInvalidException('Given path does not exist on the host.')
if not self.folder_manager.can_add_item(trans, folder):
raise exceptions.InsufficientPermissionsException('You do not have proper permission to add items to the given folder.')
elif source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from path.')
if not trans.user_is_admin():
raise exceptions.AdminRequiredException('Only admins can import from path.')
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.itervalues():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
library_bunch = upload_common.handle_library_params(trans, {}, trans.security.encode_id(folder.id))
abspath_datasets = []
kwd['filesystem_paths'] = path
if source in ['importdir_folder']:
kwd['filesystem_paths'] = os.path.join(import_base_dir, path)
# user wants to import one file only
elif source in ["userdir_file", "importdir_file"]:
file = os.path.abspath(path)
abspath_datasets.append(trans.webapp.controllers['library_common'].make_library_uploaded_dataset(
trans, 'api', kwd, os.path.basename(file), file, 'server_dir', library_bunch))
# user wants to import whole folder
elif source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers['library_common'].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
# user wants to import from path
if source in ["admin_path", "importdir_folder"]:
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers['library_common'].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
json_file_path = upload_common.create_paramfile(trans, abspath_datasets)
data_list = [ud.data for ud in abspath_datasets]
job_params = {}
job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict['id'] = trans.security.encode_id(job_dict['id'])
return job_dict
def load(self, trans, payload=None, **kwd):
"""
POST /api/libraries/datasets
Load dataset(s) from the given source into the library.
:param payload: dictionary structure containing:
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source:
source the datasets should be loaded from. Source can be:
- user directory
root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
- (admin)import directory
root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
- (admin)any absolute or relative path
option allowed with "allow_library_path_paste" in galaxy.ini
:type source: str
:param link_data:
flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs:
flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:param tag_using_filenames: flag whether to generate dataset tags from filenames
:type tag_using_filenames: bool
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
:raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
InsufficientPermissionsException, ObjectNotFound
"""
if payload:
kwd.update(payload)
kwd['space_to_tab'] = False
kwd['to_posix_lines'] = True
kwd['dbkey'] = kwd.get('dbkey', '?')
kwd['file_type'] = kwd.get('file_type', 'auto')
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(kwd.get('link_data', False)) else 'copy_files'
kwd['tag_using_filenames'] = util.string_as_bool(kwd.get('tag_using_filenames', None))
encoded_folder_id = kwd.get('encoded_folder_id', None)
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
else:
raise exceptions.RequestParameterMissingException('The required attribute encoded_folder_id is missing.')
path = kwd.get('path', None)
if path is None:
raise exceptions.RequestParameterMissingException('The required attribute path is missing.')
if not isinstance(path, str):
raise exceptions.RequestParameterInvalidException('The required attribute path is not String.')
folder = self.folder_manager.get(trans, folder_id)
source = kwd.get('source', None)
if source not in ['userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path']:
raise exceptions.RequestParameterMissingException('You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
elif source in ['importdir_file', 'importdir_folder']:
if not trans.user_is_admin:
raise exceptions.AdminRequiredException('Only admins can import from importdir.')
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from importdir.')
import_base_dir = trans.app.config.library_import_dir
if not safe_relpath(path):
# admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed.
# the reasoning here is that galaxy admins may not have direct filesystem access or can only access
# library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the
# import directory. if they have filesystem access, all bets are off.
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(import_base_dir, path)
elif source in ['userdir_file', 'userdir_folder']:
username = trans.user.username if trans.app.config.user_library_import_check_permissions else None
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow upload from user directories.')
full_dir = os.path.join(user_base_dir, user_login)
if not safe_contains(full_dir, path, allowlist=trans.app.config.user_library_import_symlink_allowlist):
# the path is a symlink outside the user dir
path = os.path.join(full_dir, path)
log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path))
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
if trans.app.config.user_library_import_check_permissions and not full_path_permission_for_user(full_dir, path, username):
log.error('User attempted to import a path that resolves to a path outside of their import dir: '
'%s -> %s and cannot be read by them.', path, os.path.realpath(path))
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(full_dir, path)
if unsafe_walk(path, allowlist=[full_dir] + trans.app.config.user_library_import_symlink_allowlist, username=username):
# the path is a dir and contains files that symlink outside the user dir
error = 'User attempted to import a path that resolves to a path outside of their import dir: {} -> {}'.format(
path, os.path.realpath(path)
)
if trans.app.config.user_library_import_check_permissions:
error += ' or is not readable for them.'
log.error(error)
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
if not os.path.exists(path):
raise exceptions.RequestParameterInvalidException('Given path does not exist on the host.')
if not self.folder_manager.can_add_item(trans, folder):
raise exceptions.InsufficientPermissionsException('You do not have proper permission to add items to the given folder.')
elif source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from path.')
if not trans.user_is_admin:
raise exceptions.AdminRequiredException('Only admins can import from path.')
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.values():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
library_bunch = upload_common.handle_library_params(trans, {}, trans.security.encode_id(folder.id))
abspath_datasets = []
kwd['filesystem_paths'] = path
if source in ['importdir_folder']:
kwd['filesystem_paths'] = os.path.join(import_base_dir, path)
# user wants to import one file only
elif source in ["userdir_file", "importdir_file"]:
file = os.path.abspath(path)
abspath_datasets.append(self._make_library_uploaded_dataset(
trans, kwd, os.path.basename(file), file, 'server_dir', library_bunch))
# user wants to import whole folder
elif source == "userdir_folder":
uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets(
trans, kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
# user wants to import from path
if source in ["admin_path", "importdir_folder"]:
# validate the path is within root
uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets(
trans, kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
json_file_path = upload_common.create_paramfile(trans, abspath_datasets)
data_list = [ud.data for ud in abspath_datasets]
job_params = {}
job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params)
trans.app.job_manager.enqueue(job, tool=tool)
job_dict = job.to_dict()
job_dict['id'] = trans.security.encode_id(job_dict['id'])
return job_dict
def load( self, trans, payload=None, **kwd ):
"""
* POST /api/libraries/datasets
Load dataset from the given source into the library.
Source can be:
user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
(admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
(admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
:param payload: dictionary structure containing:
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source: source the datasets should be loaded from
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs: flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
:raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
InsufficientPermissionsException, ObjectNotFound
"""
if payload:
kwd.update(payload)
kwd['space_to_tab'] = False
kwd['to_posix_lines'] = True
kwd[ 'dbkey' ] = kwd.get( 'dbkey', '?' )
kwd[ 'file_type' ] = kwd.get( 'file_type', 'auto' )
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool( kwd.get( 'link_data', False ) ) else 'copy_files'
encoded_folder_id = kwd.get( 'encoded_folder_id', None )
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id )
else:
raise exceptions.RequestParameterMissingException( 'The required atribute encoded_folder_id is missing.' )
path = kwd.get( 'path', None)
if path is None:
raise exceptions.RequestParameterMissingException( 'The required atribute path is missing.' )
folder = self.folder_manager.get( trans, folder_id )
source = kwd.get( 'source', None )
if source not in [ 'userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path' ]:
raise exceptions.RequestParameterMissingException( 'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
if source in [ 'importdir_file', 'importdir_folder' ]:
if not trans.user_is_admin:
raise exceptions.AdminRequiredException( 'Only admins can import from importdir.' )
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from importdir.' )
import_base_dir = trans.app.config.library_import_dir
path = os.path.join( import_base_dir, path )
if source in [ 'userdir_file', 'userdir_folder' ]:
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from user directories.' )
full_dir = os.path.join( user_base_dir, user_login )
if not path.lower().startswith( full_dir.lower() ):
path = os.path.join( full_dir, path )
if not os.path.exists( path ):
raise exceptions.RequestParameterInvalidException( 'Given path does not exist on the host.' )
if not self.folder_manager.can_add_item( trans, folder ):
raise exceptions.InsufficientPermissionsException( 'You do not have proper permission to add items to the given folder.' )
if source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from path.' )
if not trans.user_is_admin:
raise exceptions.AdminRequiredException( 'Only admins can import from path.' )
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool( tool_id )
state = tool.new_state( trans )
populate_state( trans, tool.inputs, kwd, state.inputs )
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.itervalues():
if input.type == "upload_dataset":
dataset_upload_inputs.append( input )
library_bunch = upload_common.handle_library_params( trans, {}, trans.security.encode_id( folder.id ) )
abspath_datasets = []
kwd[ 'filesystem_paths' ] = path
if source in [ 'importdir_folder' ]:
kwd[ 'filesystem_paths' ] = os.path.join( import_base_dir, path )
# user wants to import one file only
if source in [ "userdir_file", "importdir_file" ]:
file = os.path.abspath( path )
abspath_datasets.append( trans.webapp.controllers[ 'library_common' ].make_library_uploaded_dataset(
trans, 'api', kwd, os.path.basename( file ), file, 'server_dir', library_bunch ) )
# user wants to import whole folder
if source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '' )
uploaded_datasets = uploaded_datasets_bunch[ 0 ]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
for ud in uploaded_datasets:
ud.path = os.path.abspath( ud.path )
abspath_datasets.append( ud )
# user wants to import from path
if source in [ "admin_path", "importdir_folder" ]:
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '' )
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
for ud in uploaded_datasets:
ud.path = os.path.abspath( ud.path )
abspath_datasets.append( ud )
json_file_path = upload_common.create_paramfile( trans, abspath_datasets )
data_list = [ ud.data for ud in abspath_datasets ]
job_params = {}
job_params['link_data_only'] = dumps( kwd.get( 'link_data_only', 'copy_files' ) )
job_params['uuid'] = dumps( kwd.get( 'uuid', None ) )
job, output = upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params )
trans.sa_session.add( job )
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict[ 'id' ] = trans.security.encode_id( job_dict[ 'id' ] )
return job_dict
