def execute(self, tool, trans, incoming={}, set_output_hid=True, history=None, **kwargs):
dataset_upload_inputs = []
for input_name, input in tool.inputs.items():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
assert dataset_upload_inputs, Exception("No dataset upload groups were found.")
persisting_uploads_timer = ExecutionTimer()
precreated_datasets = upload_common.get_precreated_datasets(trans, incoming, trans.app.model.HistoryDatasetAssociation)
incoming = upload_common.persist_uploads(incoming, trans)
log.debug("Persisted uploads %s" % persisting_uploads_timer)
# We can pass an empty string as the cntrller here since it is used to check whether we
# are in an admin view, and this tool is currently not used there.
check_and_cleanup_timer = ExecutionTimer()
uploaded_datasets = upload_common.get_uploaded_datasets(trans, '', incoming, precreated_datasets, dataset_upload_inputs, history=history)
upload_common.cleanup_unused_precreated_datasets(precreated_datasets)
if not uploaded_datasets:
return None, 'No data was entered in the upload form, please go back and choose data to upload.'
log.debug("Checked and cleaned uploads %s" % check_and_cleanup_timer)
create_job_timer = ExecutionTimer()
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
rval = upload_common.create_job(trans, incoming, tool, json_file_path, data_list, history=history)
log.debug("Created upload job %s" % create_job_timer)
return rval
def execute( self, tool, trans, incoming={}, set_output_hid=True, history=None, **kwargs ):
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
dataset_upload_inputs.append( input )
assert dataset_upload_inputs, Exception( "No dataset upload groups were found." )
persisting_uploads_timer = ExecutionTimer()
precreated_datasets = upload_common.get_precreated_datasets( trans, incoming, trans.app.model.HistoryDatasetAssociation )
incoming = upload_common.persist_uploads( incoming )
log.debug("Persisted uploads %s" % persisting_uploads_timer)
# We can pass an empty string as the cntrller here since it is used to check whether we
# are in an admin view, and this tool is currently not used there.
check_and_cleanup_timer = ExecutionTimer()
uploaded_datasets = upload_common.get_uploaded_datasets( trans, '', incoming, precreated_datasets, dataset_upload_inputs, history=history )
upload_common.cleanup_unused_precreated_datasets( precreated_datasets )
if not uploaded_datasets:
return None, 'No data was entered in the upload form, please go back and choose data to upload.'
log.debug("Checked and cleaned uploads %s" % check_and_cleanup_timer)
create_job_timer = ExecutionTimer()
json_file_path = upload_common.create_paramfile( trans, uploaded_datasets )
data_list = [ ud.data for ud in uploaded_datasets ]
rval = upload_common.create_job( trans, incoming, tool, json_file_path, data_list, history=history )
log.debug("Created upload job %s" % create_job_timer)
return rval
def _upload_dataset(self, trans, library_id, folder_id, replace_dataset=None, **kwd):
# Set up the traditional tool state/params
cntrller = 'api'
tool_id = 'upload1'
message = None
file_type = kwd.get('file_type')
try:
upload_common.validate_datatype_extension(datatypes_registry=trans.app.datatypes_registry, ext=file_type)
except RequestParameterInvalidException as e:
return (400, util.unicodify(e))
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.items():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
# Library-specific params
server_dir = kwd.get('server_dir', '')
upload_option = kwd.get('upload_option', 'upload_file')
response_code = 200
if upload_option == 'upload_directory':
full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir)
message = 'Select a directory'
elif upload_option == 'upload_paths':
# Library API already checked this - following check isn't actually needed.
validate_path_upload(trans)
# Some error handling should be added to this method.
try:
# FIXME: instead of passing params here ( which have been processed by util.Params(), the original kwd
# should be passed so that complex objects that may have been included in the initial request remain.
library_bunch = upload_common.handle_library_params(trans, kwd, folder_id, replace_dataset)
except Exception:
response_code = 500
message = "Unable to parse upload parameters, please report this error."
# Proceed with (mostly) regular upload processing if we're still errorless
if response_code == 200:
if upload_option == 'upload_file':
tool_params = upload_common.persist_uploads(tool_params, trans)
uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, dataset_upload_inputs, library_bunch=library_bunch)
elif upload_option == 'upload_directory':
uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)
elif upload_option == 'upload_paths':
uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(trans, kwd, library_bunch, response_code, message)
if upload_option == 'upload_file' and not uploaded_datasets:
response_code = 400
message = 'Select a file, enter a URL or enter text'
if response_code != 200:
return (response_code, message)
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
job_params = {}
job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = json.dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
return output
def _upload_dataset(self, trans, library_id, folder_id, replace_dataset=None, **kwd):
# Set up the traditional tool state/params
cntrller = 'api'
tool_id = 'upload1'
message = None
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.items():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
# Library-specific params
server_dir = kwd.get('server_dir', '')
upload_option = kwd.get('upload_option', 'upload_file')
response_code = 200
if upload_option == 'upload_directory':
full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir)
message = 'Select a directory'
elif upload_option == 'upload_paths':
# Library API already checked this - following check isn't actually needed.
validate_path_upload(trans)
# Some error handling should be added to this method.
try:
# FIXME: instead of passing params here ( which have been processed by util.Params(), the original kwd
# should be passed so that complex objects that may have been included in the initial request remain.
library_bunch = upload_common.handle_library_params(trans, kwd, folder_id, replace_dataset)
except Exception:
response_code = 500
message = "Unable to parse upload parameters, please report this error."
# Proceed with (mostly) regular upload processing if we're still errorless
if response_code == 200:
precreated_datasets = upload_common.get_precreated_datasets(trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller)
if upload_option == 'upload_file':
tool_params = upload_common.persist_uploads(tool_params, trans)
uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch)
elif upload_option == 'upload_directory':
uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)
elif upload_option == 'upload_paths':
uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(trans, kwd, library_bunch, response_code, message)
upload_common.cleanup_unused_precreated_datasets(precreated_datasets)
if upload_option == 'upload_file' and not uploaded_datasets:
response_code = 400
message = 'Select a file, enter a URL or enter text'
if response_code != 200:
return (response_code, message)
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
job_params = {}
job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = json.dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
return output
def execute(self, tool, trans, incoming={}, set_output_hid=True):
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
assert dataset_upload_inputs, Exception(
"No dataset upload groups were found.")
precreated_datasets = upload_common.get_precreated_datasets(
trans, incoming, trans.app.model.HistoryDatasetAssociation)
incoming = upload_common.persist_uploads(incoming)
uploaded_datasets = upload_common.get_uploaded_datasets(
trans, incoming, precreated_datasets, dataset_upload_inputs)
upload_common.cleanup_unused_precreated_datasets(precreated_datasets)
if not uploaded_datasets:
return 'No data was entered in the upload form, please go back and choose data to upload.'
json_file_path = upload_common.create_paramfile(
trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
return upload_common.create_job(trans, incoming, tool, json_file_path,
data_list)
def load( self, trans, **kwd ):
"""
load( self, trans, **kwd ):
* POST /api/libraries/datasets
Load dataset from the given source into the library.
Source can be:
user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
(admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
(admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source: source the datasets should be loaded form
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs: flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:returns: dict containing information about the created upload job
:rtype: dictionary
"""
kwd[ 'space_to_tab' ] = 'False'
kwd[ 'to_posix_lines' ] = 'True'
kwd[ 'dbkey' ] = kwd.get( 'dbkey', '?' )
kwd[ 'file_type' ] = kwd.get( 'file_type', 'auto' )
kwd[' link_data_only' ] = 'link_to_files' if util.string_as_bool( kwd.get( 'link_data', False ) ) else 'copy_files'
encoded_folder_id = kwd.get( 'encoded_folder_id', None )
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id )
else:
raise exceptions.RequestParameterMissingException( 'The required atribute encoded_folder_id is missing.' )
path = kwd.get( 'path', None)
if path is None:
raise exceptions.RequestParameterMissingException( 'The required atribute path is missing.' )
folder = self.folder_manager.get( trans, folder_id )
source = kwd.get( 'source', None )
if source not in [ 'userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path' ]:
raise exceptions.RequestParameterMissingException( 'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
if source in [ 'importdir_file', 'importdir_folder' ]:
if not trans.user_is_admin:
raise exceptions.AdminRequiredException( 'Only admins can import from importdir.' )
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from importdir.' )
import_base_dir = trans.app.config.library_import_dir
path = os.path.join( import_base_dir, path )
if source in [ 'userdir_file', 'userdir_folder' ]:
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from user directories.' )
full_dir = os.path.join( user_base_dir, user_login )
if not path.lower().startswith( full_dir.lower() ):
path = os.path.join( full_dir, path )
if not os.path.exists( path ):
raise exceptions.RequestParameterInvalidException( 'Given path does not exist on the host.' )
if not self.folder_manager.can_add_item( trans, folder ):
raise exceptions.InsufficientPermissionsException( 'You do not have proper permission to add items to the given folder.' )
if source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from path.' )
if not trans.user_is_admin:
raise exceptions.AdminRequiredException( 'Only admins can import from path.' )
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool( tool_id )
state = tool.new_state( trans )
tool.update_state( trans, tool.inputs_by_page[ 0 ], state.inputs, kwd )
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.itervalues():
if input.type == "upload_dataset":
dataset_upload_inputs.append( input )
library_bunch = upload_common.handle_library_params( trans, {}, trans.security.encode_id( folder.id ) )
abspath_datasets = []
kwd[ 'filesystem_paths' ] = path
if source in [ 'importdir_folder' ]:
kwd[ 'filesystem_paths' ] = os.path.join( import_base_dir, path )
params = util.Params( kwd )
# user wants to import one file only
if source in [ "userdir_file", "importdir_file" ]:
file = os.path.abspath( path )
abspath_datasets.append( trans.webapp.controllers[ 'library_common' ].make_library_uploaded_dataset(
trans, 'api', params, os.path.basename( file ), file, 'server_dir', library_bunch ) )
# user wants to import whole folder
if source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
trans, 'api', params, library_bunch, 200, '' )
uploaded_datasets = uploaded_datasets_bunch[ 0 ]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
for ud in uploaded_datasets:
ud.path = os.path.abspath( ud.path )
abspath_datasets.append( ud )
# user wants to import from path
if source in [ "admin_path", "importdir_folder" ]:
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
trans, 'api', params, library_bunch, 200, '' )
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
for ud in uploaded_datasets:
ud.path = os.path.abspath( ud.path )
abspath_datasets.append( ud )
json_file_path = upload_common.create_paramfile( trans, abspath_datasets )
data_list = [ ud.data for ud in abspath_datasets ]
job, output = upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=folder )
# HACK: Prevent outputs_to_working_directory from overwriting inputs when "linking"
job.add_parameter( 'link_data_only', dumps( kwd.get( 'link_data_only', 'copy_files' ) ) )
job.add_parameter( 'uuid', dumps( kwd.get( 'uuid', None ) ) )
trans.sa_session.add( job )
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict[ 'id' ] = trans.security.encode_id( job_dict[ 'id' ] )
return job_dict
def load(self, trans, **kwd):
"""
Load dataset from the given source into the library.
:param encoded_folder_id: the encoded id of the folder to import dataset to
:type encoded_folder_id: an encoded id string
:param source: source of the dataset to be loaded
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy
:type link_data: bool
:param preserve_dirs: flag whether to preserver directory structure when importing dir
:type preserve_dirs: bool
"""
kwd['space_to_tab'] = 'False'
kwd['to_posix_lines'] = 'True'
kwd['dbkey'] = kwd.get('dbkey', '?')
kwd['file_type'] = kwd.get('file_type', 'auto')
kwd[' link_data_only'] = 'link_to_files' if util.string_as_bool(
kwd.get('link_data', False)) else 'copy_files'
encoded_folder_id = kwd.get('encoded_folder_id', None)
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode(
trans, encoded_folder_id)
else:
raise exceptions.RequestParameterMissingException(
'The required atribute encoded_folder_id is missing.')
path = kwd.get('path', None)
if path is None:
raise exceptions.RequestParameterMissingException(
'The required atribute path is missing.')
folder = self.folder_manager.get(trans, folder_id)
source = kwd.get('source', None)
if source not in [
'userdir_file', 'userdir_folder', 'importdir_file',
'importdir_folder', 'admin_path'
]:
raise exceptions.RequestParameterMissingException(
'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". '
)
if source in ['importdir_file', 'importdir_folder']:
if not trans.user_is_admin:
raise exceptions.AdminRequiredException(
'Only admins can import from importdir.')
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException(
'The configuration of this Galaxy instance does not allow admins to import into library from importdir.'
)
user_base_dir = trans.app.config.library_import_dir
if source in ['userdir_file', 'userdir_folder']:
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException(
'The configuration of this Galaxy instance does not allow upload from user directories.'
)
full_dir = os.path.join(user_base_dir, user_login)
# path_to_root_import_folder = None
if not path.lower().startswith(full_dir.lower()):
# path_to_root_import_folder = path
path = os.path.join(full_dir, path)
if not os.path.exists(path):
raise exceptions.RequestParameterInvalidException(
'Given path does not exist on the host.')
if not self.folder_manager.can_add_item(trans, folder):
raise exceptions.InsufficientPermissionsException(
'You do not have proper permission to add items to the given folder.'
)
if source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException(
'The configuration of this Galaxy instance does not allow admins to import into library from path.'
)
if not trans.user_is_admin:
raise exceptions.AdminRequiredException(
'Only admins can import from path.')
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
tool.update_state(trans, tool.inputs_by_page[0], state.inputs, kwd)
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
library_bunch = upload_common.handle_library_params(
trans, {}, trans.security.encode_id(folder.id))
abspath_datasets = []
kwd['filesystem_paths'] = path
params = util.Params(kwd)
# user wants to import one file only
if source == "userdir_file":
file = os.path.abspath(path)
abspath_datasets.append(trans.webapp.controllers['library_common'].
make_library_uploaded_dataset(
trans, 'api', params,
os.path.basename(file), file,
'server_dir', library_bunch))
# user wants to import whole folder
if source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers[
'library_common'].get_path_paste_uploaded_datasets(
trans, 'api', params, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound(
'Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
# user wants to import from path (admins only)
if source == "admin_path":
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers[
'library_common'].get_path_paste_uploaded_datasets(
trans, 'api', params, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound(
'Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
json_file_path = upload_common.create_paramfile(
trans, abspath_datasets)
data_list = [ud.data for ud in abspath_datasets]
job, output = upload_common.create_job(trans,
tool_params,
tool,
json_file_path,
data_list,
folder=folder)
# HACK: Prevent outputs_to_working_directory from overwriting inputs when "linking"
job.add_parameter('link_data_only',
dumps(kwd.get('link_data_only', 'copy_files')))
job.add_parameter('uuid', dumps(kwd.get('uuid', None)))
trans.sa_session.add(job)
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict['id'] = trans.security.encode_id(job_dict['id'])
return job_dict
def load( self, trans, **kwd ):
"""
Load dataset from the given source into the library.
:param encoded_folder_id: the encoded id of the folder to import dataset to
:type encoded_folder_id: an encoded id string
:param source: source of the dataset to be loaded
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy
:type link_data: bool
:param preserve_dirs: flag whether to preserver directory structure when importing dir
:type preserve_dirs: bool
"""
kwd[ 'space_to_tab' ] = 'False'
kwd[ 'to_posix_lines' ] = 'True'
kwd[ 'dbkey' ] = kwd.get( 'dbkey', '?' )
kwd[ 'file_type' ] = kwd.get( 'file_type', 'auto' )
kwd[' link_data_only' ] = 'link_to_files' if util.string_as_bool( kwd.get( 'link_data', False ) ) else 'copy_files'
encoded_folder_id = kwd.get( 'encoded_folder_id', None )
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id )
else:
raise exceptions.RequestParameterMissingException( 'The required atribute encoded_folder_id is missing.' )
path = kwd.get( 'path', None)
if path is None:
raise exceptions.RequestParameterMissingException( 'The required atribute path is missing.' )
folder = self.folder_manager.get( trans, folder_id )
source = kwd.get( 'source', None )
if source not in [ 'userdir_file', 'userdir_folder', 'admin_path' ]:
raise exceptions.RequestParameterMissingException( 'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder" and "admin_path". ')
if source in [ 'userdir_file', 'userdir_folder' ]:
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from user directories.' )
full_dir = os.path.join( user_base_dir, user_login )
# path_to_root_import_folder = None
if not path.lower().startswith( full_dir.lower() ):
# path_to_root_import_folder = path
path = os.path.join( full_dir, path )
if not os.path.exists( path ):
raise exceptions.RequestParameterInvalidException( 'Given path does not exist on the host.' )
if not self.folder_manager.can_add_item( trans, folder ):
raise exceptions.InsufficientPermissionsException( 'You do not have proper permission to add items to the given folder.' )
if source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from path.' )
if not trans.user_is_admin:
raise exceptions.AdminRequiredException( 'Only admins can import from path.' )
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool( tool_id )
state = tool.new_state( trans )
tool.update_state( trans, tool.inputs_by_page[ 0 ], state.inputs, kwd )
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
dataset_upload_inputs.append( input )
library_bunch = upload_common.handle_library_params( trans, {}, trans.security.encode_id( folder.id ) )
abspath_datasets = []
kwd[ 'filesystem_paths' ] = path
params = util.Params( kwd )
# user wants to import one file only
if source == "userdir_file":
file = os.path.abspath( path )
abspath_datasets.append( trans.webapp.controllers[ 'library_common' ].make_library_uploaded_dataset(
trans, 'api', params, os.path.basename( file ), file, 'server_dir', library_bunch ) )
# user wants to import whole folder
if source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
trans, 'api', params, library_bunch, 200, '' )
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
for ud in uploaded_datasets:
ud.path = os.path.abspath( ud.path )
abspath_datasets.append( ud )
# user wants to import from path (admins only)
if source == "admin_path":
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
trans, 'api', params, library_bunch, 200, '' )
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
for ud in uploaded_datasets:
ud.path = os.path.abspath( ud.path )
abspath_datasets.append( ud )
json_file_path = upload_common.create_paramfile( trans, abspath_datasets )
data_list = [ ud.data for ud in abspath_datasets ]
job, output = upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=folder )
# HACK: Prevent outputs_to_working_directory from overwriting inputs when "linking"
job.add_parameter( 'link_data_only', dumps( kwd.get( 'link_data_only', 'copy_files' ) ) )
job.add_parameter( 'uuid', dumps( kwd.get( 'uuid', None ) ) )
trans.sa_session.add( job )
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict[ 'id' ] = trans.security.encode_id( job_dict[ 'id' ] )
return job_dict
def load(self, trans, payload=None, **kwd):
"""
Load dataset(s) from the given source into the library.
* POST /api/libraries/datasets
:param payload: dictionary structure containing:
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source: source the datasets should be loaded from
Source can be:
user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
(admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
(admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs: flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:param tag_using_filenames: flag whether to generate dataset tags from filenames
:type tag_using_filenames: bool
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
:raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
InsufficientPermissionsException, ObjectNotFound
"""
if payload:
kwd.update(payload)
kwd['space_to_tab'] = False
kwd['to_posix_lines'] = True
kwd['dbkey'] = kwd.get('dbkey', '?')
kwd['file_type'] = kwd.get('file_type', 'auto')
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(kwd.get('link_data', False)) else 'copy_files'
kwd['tag_using_filenames'] = util.string_as_bool(kwd.get('tag_using_filenames', None))
encoded_folder_id = kwd.get('encoded_folder_id', None)
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
else:
raise exceptions.RequestParameterMissingException('The required attribute encoded_folder_id is missing.')
path = kwd.get('path', None)
if path is None:
raise exceptions.RequestParameterMissingException('The required attribute path is missing.')
folder = self.folder_manager.get(trans, folder_id)
source = kwd.get('source', None)
if source not in ['userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path']:
raise exceptions.RequestParameterMissingException('You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
elif source in ['importdir_file', 'importdir_folder']:
if not trans.user_is_admin():
raise exceptions.AdminRequiredException('Only admins can import from importdir.')
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from importdir.')
import_base_dir = trans.app.config.library_import_dir
if not safe_relpath(path):
# admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed.
# the reasoning here is that galaxy admins may not have direct filesystem access or can only access
# library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the
# import directory. if they have filesystem access, all bets are off.
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(import_base_dir, path)
elif source in ['userdir_file', 'userdir_folder']:
unsafe = None
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow upload from user directories.')
full_dir = os.path.join(user_base_dir, user_login)
if not safe_contains(full_dir, path, whitelist=trans.app.config.user_library_import_symlink_whitelist):
# the path is a symlink outside the user dir
path = os.path.join(full_dir, path)
log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path))
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(full_dir, path)
for unsafe in unsafe_walk(path, whitelist=[full_dir] + trans.app.config.user_library_import_symlink_whitelist):
# the path is a dir and contains files that symlink outside the user dir
log.error('User attempted to import a directory containing a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe))
if unsafe:
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
if not os.path.exists(path):
raise exceptions.RequestParameterInvalidException('Given path does not exist on the host.')
if not self.folder_manager.can_add_item(trans, folder):
raise exceptions.InsufficientPermissionsException('You do not have proper permission to add items to the given folder.')
elif source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from path.')
if not trans.user_is_admin():
raise exceptions.AdminRequiredException('Only admins can import from path.')
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.itervalues():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
library_bunch = upload_common.handle_library_params(trans, {}, trans.security.encode_id(folder.id))
abspath_datasets = []
kwd['filesystem_paths'] = path
if source in ['importdir_folder']:
kwd['filesystem_paths'] = os.path.join(import_base_dir, path)
# user wants to import one file only
elif source in ["userdir_file", "importdir_file"]:
file = os.path.abspath(path)
abspath_datasets.append(trans.webapp.controllers['library_common'].make_library_uploaded_dataset(
trans, 'api', kwd, os.path.basename(file), file, 'server_dir', library_bunch))
# user wants to import whole folder
elif source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers['library_common'].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
# user wants to import from path
if source in ["admin_path", "importdir_folder"]:
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers['library_common'].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
json_file_path = upload_common.create_paramfile(trans, abspath_datasets)
data_list = [ud.data for ud in abspath_datasets]
job_params = {}
job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict['id'] = trans.security.encode_id(job_dict['id'])
return job_dict
def load(self, trans, payload=None, **kwd):
"""
POST /api/libraries/datasets
Load dataset(s) from the given source into the library.
:param payload: dictionary structure containing:
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source:
source the datasets should be loaded from. Source can be:
- user directory
root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
- (admin)import directory
root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
- (admin)any absolute or relative path
option allowed with "allow_library_path_paste" in galaxy.ini
:type source: str
:param link_data:
flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs:
flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:param tag_using_filenames: flag whether to generate dataset tags from filenames
:type tag_using_filenames: bool
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
:raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
InsufficientPermissionsException, ObjectNotFound
"""
if payload:
kwd.update(payload)
kwd['space_to_tab'] = False
kwd['to_posix_lines'] = True
kwd['dbkey'] = kwd.get('dbkey', '?')
kwd['file_type'] = kwd.get('file_type', 'auto')
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(kwd.get('link_data', False)) else 'copy_files'
kwd['tag_using_filenames'] = util.string_as_bool(kwd.get('tag_using_filenames', None))
encoded_folder_id = kwd.get('encoded_folder_id', None)
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
else:
raise exceptions.RequestParameterMissingException('The required attribute encoded_folder_id is missing.')
path = kwd.get('path', None)
if path is None:
raise exceptions.RequestParameterMissingException('The required attribute path is missing.')
if not isinstance(path, str):
raise exceptions.RequestParameterInvalidException('The required attribute path is not String.')
folder = self.folder_manager.get(trans, folder_id)
source = kwd.get('source', None)
if source not in ['userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path']:
raise exceptions.RequestParameterMissingException('You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
elif source in ['importdir_file', 'importdir_folder']:
if not trans.user_is_admin:
raise exceptions.AdminRequiredException('Only admins can import from importdir.')
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from importdir.')
import_base_dir = trans.app.config.library_import_dir
if not safe_relpath(path):
# admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed.
# the reasoning here is that galaxy admins may not have direct filesystem access or can only access
# library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the
# import directory. if they have filesystem access, all bets are off.
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(import_base_dir, path)
elif source in ['userdir_file', 'userdir_folder']:
username = trans.user.username if trans.app.config.user_library_import_check_permissions else None
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow upload from user directories.')
full_dir = os.path.join(user_base_dir, user_login)
if not safe_contains(full_dir, path, allowlist=trans.app.config.user_library_import_symlink_allowlist):
# the path is a symlink outside the user dir
path = os.path.join(full_dir, path)
log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path))
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
if trans.app.config.user_library_import_check_permissions and not full_path_permission_for_user(full_dir, path, username):
log.error('User attempted to import a path that resolves to a path outside of their import dir: '
'%s -> %s and cannot be read by them.', path, os.path.realpath(path))
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
path = os.path.join(full_dir, path)
if unsafe_walk(path, allowlist=[full_dir] + trans.app.config.user_library_import_symlink_allowlist, username=username):
# the path is a dir and contains files that symlink outside the user dir
error = 'User attempted to import a path that resolves to a path outside of their import dir: {} -> {}'.format(
path, os.path.realpath(path)
)
if trans.app.config.user_library_import_check_permissions:
error += ' or is not readable for them.'
log.error(error)
raise exceptions.RequestParameterInvalidException('The given path is invalid.')
if not os.path.exists(path):
raise exceptions.RequestParameterInvalidException('Given path does not exist on the host.')
if not self.folder_manager.can_add_item(trans, folder):
raise exceptions.InsufficientPermissionsException('You do not have proper permission to add items to the given folder.')
elif source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to import into library from path.')
if not trans.user_is_admin:
raise exceptions.AdminRequiredException('Only admins can import from path.')
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.values():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
library_bunch = upload_common.handle_library_params(trans, {}, trans.security.encode_id(folder.id))
abspath_datasets = []
kwd['filesystem_paths'] = path
if source in ['importdir_folder']:
kwd['filesystem_paths'] = os.path.join(import_base_dir, path)
# user wants to import one file only
elif source in ["userdir_file", "importdir_file"]:
file = os.path.abspath(path)
abspath_datasets.append(self._make_library_uploaded_dataset(
trans, kwd, os.path.basename(file), file, 'server_dir', library_bunch))
# user wants to import whole folder
elif source == "userdir_folder":
uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets(
trans, kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
# user wants to import from path
if source in ["admin_path", "importdir_folder"]:
# validate the path is within root
uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets(
trans, kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound('Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
json_file_path = upload_common.create_paramfile(trans, abspath_datasets)
data_list = [ud.data for ud in abspath_datasets]
job_params = {}
job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params)
trans.app.job_manager.enqueue(job, tool=tool)
job_dict = job.to_dict()
job_dict['id'] = trans.security.encode_id(job_dict['id'])
return job_dict
def _create_job(self, *args, **kwds):
"""Wrapper around upload_common.create_job with a timer."""
create_job_timer = ExecutionTimer()
rval = upload_common.create_job(*args, **kwds)
log.debug("Created upload job %s" % create_job_timer)
return rval
def _create_job(self, *args, **kwds):
"""Wrapper around upload_common.create_job with a timer."""
create_job_timer = ExecutionTimer()
rval = upload_common.create_job(*args, **kwds)
log.debug("Created upload job %s" % create_job_timer)
return rval
def upload_dataset( self, trans, controller, library_id, folder_id, replace_dataset=None, **kwd ):
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.tools_by_id[ tool_id ]
state = tool.new_state( trans )
errors = tool.update_state( trans, tool.inputs_by_page[0], state.inputs, kwd )
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
dataset_upload_inputs.append( input )
# Library-specific params
params = util.Params( kwd ) # is this filetoolparam safe?
library_bunch = upload_common.handle_library_params( trans, params, folder_id, replace_dataset )
msg = util.restore_text( params.get( 'msg', '' ) )
messagetype = params.get( 'messagetype', 'done' )
server_dir = util.restore_text( params.get( 'server_dir', '' ) )
if replace_dataset not in [ None, 'None' ]:
replace_id = replace_dataset.id
else:
replace_id = None
upload_option = params.get( 'upload_option', 'upload_file' )
err_redirect = False
if upload_option == 'upload_directory':
if server_dir in [ None, 'None', '' ]:
err_redirect = True
if controller == 'library_admin':
import_dir = trans.app.config.library_import_dir
import_dir_desc = 'library_import_dir'
full_dir = os.path.join( import_dir, server_dir )
else:
import_dir = trans.app.config.user_library_import_dir
import_dir_desc = 'user_library_import_dir'
if server_dir == trans.user.email:
full_dir = os.path.join( import_dir, server_dir )
else:
full_dir = os.path.join( import_dir, trans.user.email, server_dir )
if import_dir:
msg = 'Select a directory'
else:
msg = '"%s" is not defined in the Galaxy configuration file' % import_dir_desc
# Proceed with (mostly) regular upload processing
precreated_datasets = upload_common.get_precreated_datasets( trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=controller )
if upload_option == 'upload_file':
tool_params = upload_common.persist_uploads( tool_params )
uploaded_datasets = upload_common.get_uploaded_datasets( trans, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch )
elif upload_option == 'upload_directory':
uploaded_datasets, err_redirect, msg = self.get_server_dir_uploaded_datasets( trans, params, full_dir, import_dir_desc, library_bunch, err_redirect, msg )
elif upload_option == 'upload_paths':
uploaded_datasets, err_redirect, msg = self.get_path_paste_uploaded_datasets( trans, params, library_bunch, err_redirect, msg )
upload_common.cleanup_unused_precreated_datasets( precreated_datasets )
if upload_option == 'upload_file' and not uploaded_datasets:
msg = 'Select a file, enter a URL or enter text'
err_redirect = True
if err_redirect:
trans.response.send_redirect( web.url_for( controller=controller,
action='upload_library_dataset',
library_id=library_id,
folder_id=folder_id,
replace_id=replace_id,
upload_option=upload_option,
msg=util.sanitize_text( msg ),
messagetype='error' ) )
json_file_path = upload_common.create_paramfile( trans, uploaded_datasets )
data_list = [ ud.data for ud in uploaded_datasets ]
return upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder )
def load(self, trans, payload=None, **kwd):
"""
* POST /api/libraries/datasets
Load dataset from the given source into the library.
Source can be:
user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
the folder with the user login has to be created beforehand
(admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
(admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
:param payload: dictionary structure containing:
:param encoded_folder_id: the encoded id of the folder to import dataset(s) to
:type encoded_folder_id: an encoded id string
:param source: source the datasets should be loaded from
:type source: str
:param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
while linking is set to True all symlinks will be resolved _once_
:type link_data: bool
:param preserve_dirs: flag whether to preserve the directory structure when importing dir
if False only datasets will be imported
:type preserve_dirs: bool
:param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect)
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
:raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
InsufficientPermissionsException, ObjectNotFound
"""
if payload:
kwd.update(payload)
kwd['space_to_tab'] = False
kwd['to_posix_lines'] = True
kwd['dbkey'] = kwd.get('dbkey', '?')
kwd['file_type'] = kwd.get('file_type', 'auto')
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(
kwd.get('link_data', False)) else 'copy_files'
encoded_folder_id = kwd.get('encoded_folder_id', None)
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode(
trans, encoded_folder_id)
else:
raise exceptions.RequestParameterMissingException(
'The required atribute encoded_folder_id is missing.')
path = kwd.get('path', None)
if path is None:
raise exceptions.RequestParameterMissingException(
'The required atribute path is missing.')
folder = self.folder_manager.get(trans, folder_id)
source = kwd.get('source', None)
if source not in [
'userdir_file', 'userdir_folder', 'importdir_file',
'importdir_folder', 'admin_path'
]:
raise exceptions.RequestParameterMissingException(
'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". '
)
if source in ['importdir_file', 'importdir_folder']:
if not trans.user_is_admin:
raise exceptions.AdminRequiredException(
'Only admins can import from importdir.')
if not trans.app.config.library_import_dir:
raise exceptions.ConfigDoesNotAllowException(
'The configuration of this Galaxy instance does not allow admins to import into library from importdir.'
)
import_base_dir = trans.app.config.library_import_dir
path = os.path.join(import_base_dir, path)
if source in ['userdir_file', 'userdir_folder']:
user_login = trans.user.email
user_base_dir = trans.app.config.user_library_import_dir
if user_base_dir is None:
raise exceptions.ConfigDoesNotAllowException(
'The configuration of this Galaxy instance does not allow upload from user directories.'
)
full_dir = os.path.join(user_base_dir, user_login)
if not path.lower().startswith(full_dir.lower()):
path = os.path.join(full_dir, path)
if not os.path.exists(path):
raise exceptions.RequestParameterInvalidException(
'Given path does not exist on the host.')
if not self.folder_manager.can_add_item(trans, folder):
raise exceptions.InsufficientPermissionsException(
'You do not have proper permission to add items to the given folder.'
)
if source == 'admin_path':
if not trans.app.config.allow_library_path_paste:
raise exceptions.ConfigDoesNotAllowException(
'The configuration of this Galaxy instance does not allow admins to import into library from path.'
)
if not trans.user_is_admin:
raise exceptions.AdminRequiredException(
'Only admins can import from path.')
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.get_tool(tool_id)
state = tool.new_state(trans)
tool.populate_state(trans, tool.inputs, kwd, state.inputs)
tool_params = state.inputs
dataset_upload_inputs = []
for input in tool.inputs.itervalues():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
library_bunch = upload_common.handle_library_params(
trans, {}, trans.security.encode_id(folder.id))
abspath_datasets = []
kwd['filesystem_paths'] = path
if source in ['importdir_folder']:
kwd['filesystem_paths'] = os.path.join(import_base_dir, path)
# user wants to import one file only
if source in ["userdir_file", "importdir_file"]:
file = os.path.abspath(path)
abspath_datasets.append(trans.webapp.controllers['library_common'].
make_library_uploaded_dataset(
trans, 'api', kwd,
os.path.basename(file), file,
'server_dir', library_bunch))
# user wants to import whole folder
if source == "userdir_folder":
uploaded_datasets_bunch = trans.webapp.controllers[
'library_common'].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound(
'Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
# user wants to import from path
if source in ["admin_path", "importdir_folder"]:
# validate the path is within root
uploaded_datasets_bunch = trans.webapp.controllers[
'library_common'].get_path_paste_uploaded_datasets(
trans, 'api', kwd, library_bunch, 200, '')
uploaded_datasets = uploaded_datasets_bunch[0]
if uploaded_datasets is None:
raise exceptions.ObjectNotFound(
'Given folder does not contain any datasets.')
for ud in uploaded_datasets:
ud.path = os.path.abspath(ud.path)
abspath_datasets.append(ud)
json_file_path = upload_common.create_paramfile(
trans, abspath_datasets)
data_list = [ud.data for ud in abspath_datasets]
job_params = {}
job_params['link_data_only'] = dumps(
kwd.get('link_data_only', 'copy_files'))
job_params['uuid'] = dumps(kwd.get('uuid', None))
job, output = upload_common.create_job(trans,
tool_params,
tool,
json_file_path,
data_list,
folder=folder,
job_params=job_params)
trans.sa_session.add(job)
trans.sa_session.flush()
job_dict = job.to_dict()
job_dict['id'] = trans.security.encode_id(job_dict['id'])
return job_dict
def upload_dataset(self,
trans,
controller,
library_id,
folder_id,
replace_dataset=None,
**kwd):
# Set up the traditional tool state/params
tool_id = 'upload1'
tool = trans.app.toolbox.tools_by_id[tool_id]
state = tool.new_state(trans)
errors = tool.update_state(trans, tool.inputs_by_page[0], state.inputs,
kwd)
tool_params = state.inputs
dataset_upload_inputs = []
for input_name, input in tool.inputs.iteritems():
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
# Library-specific params
params = util.Params(kwd) # is this filetoolparam safe?
library_bunch = upload_common.handle_library_params(
trans, params, folder_id, replace_dataset)
msg = util.restore_text(params.get('msg', ''))
messagetype = params.get('messagetype', 'done')
server_dir = util.restore_text(params.get('server_dir', ''))
if replace_dataset not in [None, 'None']:
replace_id = replace_dataset.id
else:
replace_id = None
upload_option = params.get('upload_option', 'upload_file')
err_redirect = False
if upload_option == 'upload_directory':
if server_dir in [None, 'None', '']:
err_redirect = True
if controller == 'library_admin':
import_dir = trans.app.config.library_import_dir
import_dir_desc = 'library_import_dir'
full_dir = os.path.join(import_dir, server_dir)
else:
import_dir = trans.app.config.user_library_import_dir
import_dir_desc = 'user_library_import_dir'
if server_dir == trans.user.email:
full_dir = os.path.join(import_dir, server_dir)
else:
full_dir = os.path.join(import_dir, trans.user.email,
server_dir)
if import_dir:
msg = 'Select a directory'
else:
msg = '"%s" is not defined in the Galaxy configuration file' % import_dir_desc
# Proceed with (mostly) regular upload processing
precreated_datasets = upload_common.get_precreated_datasets(
trans,
tool_params,
trans.app.model.LibraryDatasetDatasetAssociation,
controller=controller)
if upload_option == 'upload_file':
tool_params = upload_common.persist_uploads(tool_params)
uploaded_datasets = upload_common.get_uploaded_datasets(
trans,
tool_params,
precreated_datasets,
dataset_upload_inputs,
library_bunch=library_bunch)
elif upload_option == 'upload_directory':
uploaded_datasets, err_redirect, msg = self.get_server_dir_uploaded_datasets(
trans, params, full_dir, import_dir_desc, library_bunch,
err_redirect, msg)
elif upload_option == 'upload_paths':
uploaded_datasets, err_redirect, msg = self.get_path_paste_uploaded_datasets(
trans, params, library_bunch, err_redirect, msg)
upload_common.cleanup_unused_precreated_datasets(precreated_datasets)
if upload_option == 'upload_file' and not uploaded_datasets:
msg = 'Select a file, enter a URL or enter text'
err_redirect = True
if err_redirect:
trans.response.send_redirect(
web.url_for(controller=controller,
action='upload_library_dataset',
library_id=library_id,
folder_id=folder_id,
replace_id=replace_id,
upload_option=upload_option,
msg=util.sanitize_text(msg),
messagetype='error'))
json_file_path = upload_common.create_paramfile(
trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
return upload_common.create_job(trans,
tool_params,
tool,
json_file_path,
data_list,
folder=library_bunch.folder)