def __delete_dataset( self, trans, history_id, id, purge, **kwd ): # get purge from the query or from the request body payload (a request body is optional here) purge = util.string_as_bool( purge ) if kwd.get( 'payload', None ): # payload takes priority purge = util.string_as_bool( kwd['payload'].get( 'purge', purge ) ) hda = self.mgrs.hdas.get( trans, self._decode_id( trans, id ), check_ownership=True, check_accessible=True ) self.mgrs.hdas.err_if_uploading( trans, hda ) rval = { 'id' : id } hda.deleted = True if purge: if not trans.app.config.allow_user_dataset_purge: raise exceptions.ConfigDoesNotAllowException( 'This instance does not allow user dataset purging' ) hda.purged = True trans.sa_session.add( hda ) trans.sa_session.flush() if hda.dataset.user_can_purge: try: hda.dataset.full_delete() trans.sa_session.add( hda.dataset ) except: pass # flush now to preserve deleted state in case of later interruption trans.sa_session.flush() rval[ 'purged' ] = True self.stop_hda_creating_job( hda ) trans.sa_session.flush() rval[ 'deleted' ] = True return rval
def __init__(self, **kwargs): self.app = kwargs['app'] self.verbose = string_as_bool(kwargs.get('verbose', False)) self.user_submission = string_as_bool(kwargs.get('user_submission', False)) self.report_directory = kwargs.get("directory", tempfile.gettempdir()) if not os.path.exists(self.report_directory): os.makedirs(self.report_directory)
def _parse_output( self, data_elem, tool, default_format="data", default_format_source=None, default_metadata_source="", ): output = ToolOutput( data_elem.get("name") ) output_format = data_elem.get("format", default_format) auto_format = string_as_bool( data_elem.get( "auto_format", "false" ) ) if auto_format and output_format != "data": raise ValueError("Setting format and auto_format is not supported at this time.") elif auto_format: output_format = "_sniff_" output.format = output_format output.change_format = data_elem.findall("change_format") output.format_source = data_elem.get("format_source", default_format_source) output.metadata_source = data_elem.get("metadata_source", default_metadata_source) output.parent = data_elem.get("parent", None) output.label = xml_text( data_elem, "label" ) output.count = int( data_elem.get("count", 1) ) output.filters = data_elem.findall( 'filter' ) output.tool = tool output.from_work_dir = data_elem.get("from_work_dir", None) output.hidden = string_as_bool( data_elem.get("hidden", "") ) output.actions = ToolOutputActionGroup( output, data_elem.find( 'actions' ) ) output.dataset_collector_descriptions = dataset_collector_descriptions_from_elem( data_elem, legacy=self.legacy_defaults ) return output
def __api_import_new_workflow(self, trans, payload, **kwd): data = payload["workflow"] publish = util.string_as_bool(payload.get("publish", False)) # If 'publish' set, default to importable. importable = util.string_as_bool(payload.get("importable", publish)) if publish and not importable: raise exceptions.RequestParameterInvalidException("Published workflow must be importable.") from_dict_kwds = dict(source="API", publish=publish) workflow, missing_tool_tups = self._workflow_from_dict(trans, data, **from_dict_kwds) if importable: self._make_item_accessible(trans.sa_session, workflow) trans.sa_session.flush() # galaxy workflow newly created id workflow_id = workflow.id # api encoded, id encoded_id = trans.security.encode_id(workflow_id) # return list rval = [] item = workflow.to_dict(value_mapper={"id": trans.security.encode_id}) item["url"] = url_for("workflow", id=encoded_id) rval.append(item) return item
def from_elem( cls, xml_root ): provider_elem = xml_root provider_id = provider_elem.get( 'id', None ) provider_name = provider_elem.get( 'name', provider_id ) op_endpoint_url = provider_elem.find( 'op_endpoint_url' ) if op_endpoint_url is not None: op_endpoint_url = op_endpoint_url.text never_associate_with_user = string_as_bool( provider_elem.get( 'never_associate_with_user', 'False' ) ) assert (provider_id and provider_name and op_endpoint_url), Exception( "OpenID Provider improperly configured" ) assert provider_id not in RESERVED_PROVIDER_IDS, Exception( 'Specified OpenID Provider uses a reserved id: %s' % ( provider_id ) ) sreg_required = [] sreg_optional = [] use_for = {} store_user_preference = {} use_default_sreg = True for elem in provider_elem.findall( 'sreg' ): use_default_sreg = False for field_elem in elem.findall( 'field' ): sreg_name = field_elem.get( 'name' ) assert sreg_name, Exception( 'A name is required for a sreg element' ) if string_as_bool( field_elem.get( 'required' ) ): sreg_required.append( sreg_name ) else: sreg_optional.append( sreg_name ) for use_elem in field_elem.findall( 'use_for' ): use_for[ use_elem.get( 'name' ) ] = sreg_name for store_user_preference_elem in field_elem.findall( 'store_user_preference' ): store_user_preference[ store_user_preference_elem.get( 'name' ) ] = sreg_name if use_default_sreg: sreg_required = None sreg_optional = None use_for = None return cls( provider_id, provider_name, op_endpoint_url, sreg_required=sreg_required, sreg_optional=sreg_optional, use_for=use_for, store_user_preference=store_user_preference, never_associate_with_user=never_associate_with_user )
def _get_attribute_as_bool( self, attribute, default, elem=None ): if elem is None: elem = self.root return string_as_bool( elem.get( attribute, default ) )
def _create(self, trans: GalaxyWebTransaction, payload, **kwd): action = payload.get('action') if action == 'rerun': raise Exception("'rerun' action has been deprecated") # Get tool. tool_version = payload.get('tool_version') tool_id = payload.get('tool_id') tool_uuid = payload.get('tool_uuid') get_kwds = dict( tool_id=tool_id, tool_uuid=tool_uuid, tool_version=tool_version, ) if tool_id is None and tool_uuid is None: raise exceptions.RequestParameterMissingException("Must specify either a tool_id or a tool_uuid.") tool = trans.app.toolbox.get_tool(**get_kwds) if not tool: log.debug(f"Not found tool with kwds [{get_kwds}]") raise exceptions.ToolMissingException('Tool not found.') if not tool.allow_user_access(trans.user): raise exceptions.ItemAccessibilityException('Tool not accessible.') if trans.app.config.user_activation_on: if not trans.user: log.warning("Anonymous user attempts to execute tool, but account activation is turned on.") elif not trans.user.active: log.warning(f"User \"{trans.user.email}\" attempts to execute tool, but account activation is turned on and user account is not active.") # Set running history from payload parameters. # History not set correctly as part of this API call for # dataset upload. history_id = payload.get('history_id') if history_id: decoded_id = self.decode_id(history_id) target_history = self.history_manager.get_owned(decoded_id, trans.user, current_history=trans.history) else: target_history = None # Set up inputs. inputs = payload.get('inputs', {}) if not isinstance(inputs, dict): raise exceptions.RequestParameterInvalidException(f"inputs invalid {inputs}") # Find files coming in as multipart file data and add to inputs. for k, v in payload.items(): if k.startswith('files_') or k.startswith('__files_'): inputs[k] = v # for inputs that are coming from the Library, copy them into the history self._patch_library_inputs(trans, inputs, target_history) # TODO: encode data ids and decode ids. # TODO: handle dbkeys params = util.Params(inputs, sanitize=False) incoming = params.__dict__ # use_cached_job can be passed in via the top-level payload or among the tool inputs. # I think it should be a top-level parameter, but because the selector is implemented # as a regular tool parameter we accept both. use_cached_job = payload.get('use_cached_job', False) or util.string_as_bool(inputs.get('use_cached_job', 'false')) input_format = str(payload.get('input_format', 'legacy')) vars = tool.handle_input(trans, incoming, history=target_history, use_cached_job=use_cached_job, input_format=input_format) # TODO: check for errors and ensure that output dataset(s) are available. output_datasets = vars.get('out_data', []) rval: Dict[str, Any] = {'outputs': [], 'output_collections': [], 'jobs': [], 'implicit_collections': []} rval['produces_entry_points'] = tool.produces_entry_points job_errors = vars.get('job_errors', []) if job_errors: # If we are here - some jobs were successfully executed but some failed. rval['errors'] = job_errors outputs = rval['outputs'] # TODO:?? poss. only return ids? for output_name, output in output_datasets: output_dict = output.to_dict() # add the output name back into the output data structure # so it's possible to figure out which newly created elements # correspond with which tool file outputs output_dict['output_name'] = output_name outputs.append(trans.security.encode_dict_ids(output_dict, skip_startswith="metadata_")) new_pja_flush = False for job in vars.get('jobs', []): rval['jobs'].append(self.encode_all_ids(trans, job.to_dict(view='collection'), recursive=True)) if inputs.get('send_email_notification', False): # Unless an anonymous user is invoking this via the API it # should never be an option, but check and enforce that here if trans.user is None: raise exceptions.ToolExecutionError("Anonymously run jobs cannot send an email notification.") else: job_email_action = PostJobAction('EmailAction') job.add_post_job_action(job_email_action) new_pja_flush = True if new_pja_flush: trans.sa_session.flush() for output_name, collection_instance in vars.get('output_collections', []): history = target_history or trans.history output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history) output_dict['output_name'] = output_name rval['output_collections'].append(output_dict) for output_name, collection_instance in vars.get('implicit_collections', {}).items(): history = target_history or trans.history output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history) output_dict['output_name'] = output_name rval['implicit_collections'].append(output_dict) return rval
def parse_hidden(self): hidden = xml_text(self.root, "hidden") if hidden: hidden = string_as_bool(hidden) return hidden
from galaxy.tools.util import maf_utilities from galaxy.util import string_as_bool assert sys.version_info[:2] >= ( 2, 4 ) def __main__(): try: maf_reader = maf.Reader( open( sys.argv[1] ) ) except Exception, e: maf_utilities.tool_fail( "Error opening MAF: %s" % e ) try: out = maf.Writer( open( sys.argv[2], "w") ) except Exception, e: maf_utilities.tool_fail( "Error opening file for output: %s" % e ) try: collapse_columns = string_as_bool( sys.argv[3] ) except Exception, e: maf_utilities.tool_fail( "Error determining collapse columns value: %s" % e ) start_count = 0 end_count = 0 for start_count, start_block in enumerate( maf_reader ): for block in maf_utilities.iter_blocks_split_by_species( start_block ): if collapse_columns: block.remove_all_gap_columns() out.write( block ) end_count += 1 out.close() if end_count: print "%i alignment blocks created from %i original blocks." % ( end_count, start_count + 1 )
def create(self, trans, cntrller='user', redirect_url='', refresh_frames=None, **kwd): refresh_frames = refresh_frames or [] params = util.Params(kwd) # If the honeypot field is not empty we are dealing with a bot. honeypot_field = params.get('bear_field', '') if honeypot_field != '': return trans.show_error_message("You've been flagged as a possible bot. If you are not, please try registering again and fill the form out carefully. <a target=\"_top\" href=\"%s\">Go to the home page</a>.") % url_for('/') message = util.restore_text(params.get('message', '')) status = params.get('status', 'done') use_panels = util.string_as_bool(kwd.get('use_panels', True)) email = util.restore_text(params.get('email', '')) # Do not sanitize passwords, so take from kwd # instead of params ( which were sanitized ) password = kwd.get('password', '') confirm = kwd.get('confirm', '') username = util.restore_text(params.get('username', '')) subscribe = params.get('subscribe', '') subscribe_checked = CheckboxField.is_checked(subscribe) referer = trans.request.referer or '' redirect = kwd.get('redirect', referer).strip() is_admin = trans.user_is_admin success = False show_user_prepopulate_form = False if not trans.app.config.allow_user_creation and not trans.user_is_admin: message = 'User registration is disabled. Please contact your local Galaxy administrator for an account.' if trans.app.config.error_email_to is not None: message += f' Contact: {trans.app.config.error_email_to}' status = 'error' else: # check user is allowed to register message, status = trans.app.auth_manager.check_registration_allowed(email, username, password) if not message: # Create the user, save all the user info and login to Galaxy if params.get('create_user_button', False): # Check email and password validity message = self.__validate(trans, email, password, confirm, username) if not message: # All the values are valid message, status, user, success = self.__register(trans, subscribe_checked=subscribe_checked, **kwd) if success and not is_admin: # The handle_user_login() method has a call to the history_set_default_permissions() method # (needed when logging in with a history), user needs to have default permissions set before logging in trans.handle_user_login(user) trans.log_event("User created a new account") trans.log_event("User logged in") else: status = 'error' registration_warning_message = trans.app.config.registration_warning_message if success: if is_admin: redirect_url = web.url_for('/admin/users?status=success&message=Created new user account.') else: redirect_url = web.url_for('/') return trans.fill_template('/webapps/tool_shed/user/register.mako', cntrller=cntrller, email=email, username=username, subscribe_checked=subscribe_checked, show_user_prepopulate_form=show_user_prepopulate_form, use_panels=use_panels, redirect=redirect, redirect_url=redirect_url, refresh_frames=refresh_frames, registration_warning_message=registration_warning_message, message=message, status=status)
def get_bool(self, key, default): if key in self.config_dict: return string_as_bool(self.config_dict[key]) else: return default
def __init__(self, spec): MetadataParameter.__init__(self, spec) self.values = self.spec.get("values") self.multiple = string_as_bool(self.spec.get("multiple"))
def __api_import_new_workflow(self, trans, payload, **kwd): data = payload['workflow'] import_tools = util.string_as_bool(payload.get("import_tools", False)) if import_tools and not trans.user_is_admin(): raise exceptions.AdminRequiredException() publish = util.string_as_bool(payload.get("publish", False)) # If 'publish' set, default to importable. importable = util.string_as_bool(payload.get("importable", publish)) # Galaxy will try to upgrade tool versions that don't match exactly during import, # this prevents that. exact_tools = util.string_as_bool(payload.get("exact_tools", False)) if publish and not importable: raise exceptions.RequestParameterInvalidException( "Published workflow must be importable.") from_dict_kwds = dict( source="API", publish=publish, exact_tools=exact_tools, ) workflow, missing_tool_tups = self._workflow_from_dict( trans, data, **from_dict_kwds) if importable: self._make_item_accessible(trans.sa_session, workflow) trans.sa_session.flush() # galaxy workflow newly created id workflow_id = workflow.id # api encoded, id encoded_id = trans.security.encode_id(workflow_id) # return list rval = [] item = workflow.to_dict(value_mapper={'id': trans.security.encode_id}) item['url'] = url_for('workflow', id=encoded_id) item['owner'] = workflow.user.username item['number_of_steps'] = len(workflow.latest_workflow.steps) rval.append(item) # if import_tools: tools = {} for key in data['steps']: item = data['steps'][key] if item is not None: if 'tool_shed_repository' in item: tool_shed_repository = item['tool_shed_repository'] if 'owner' in tool_shed_repository and 'changeset_revision' in tool_shed_repository and 'name' in tool_shed_repository and 'tool_shed' in tool_shed_repository: toolstr = tool_shed_repository['owner'] \ + tool_shed_repository['changeset_revision'] \ + tool_shed_repository['name'] \ + tool_shed_repository['tool_shed'] tools[toolstr] = tool_shed_repository irm = InstallRepositoryManager(self.app) for k in tools: item = tools[k] tool_shed_url = 'https://' + item['tool_shed'] + '/' name = item['name'] owner = item['owner'] changeset_revision = item['changeset_revision'] irm.install(tool_shed_url, name, owner, changeset_revision, payload) return item
def get_workflows_list(self, trans, kwd): """ Displays a collection of workflows. :param show_published: if True, show also published workflows :type show_published: boolean :param missing_tools: if True, include a list of missing tools per workflow :type missing_tools: boolean """ show_published = util.string_as_bool(kwd.get('show_published', 'False')) missing_tools = util.string_as_bool(kwd.get('missing_tools', 'False')) rval = [] filter1 = (trans.app.model.StoredWorkflow.user == trans.user) user = trans.get_user() if show_published: filter1 = or_(filter1, (trans.app.model.StoredWorkflow.published == true())) for wf in trans.sa_session.query( trans.app.model.StoredWorkflow).options( eagerload("latest_workflow").undefer( "step_count").lazyload("steps")).filter( filter1, trans.app.model.StoredWorkflow.table.c.deleted == false()).order_by( desc(trans.app.model.StoredWorkflow.table.c. update_time)).all(): item = wf.to_dict(value_mapper={'id': trans.security.encode_id}) encoded_id = trans.security.encode_id(wf.id) item['url'] = url_for('workflow', id=encoded_id) item['owner'] = wf.user.username item['number_of_steps'] = wf.latest_workflow.step_count item['show_in_tool_panel'] = False for x in user.stored_workflow_menu_entries: if x.stored_workflow_id == wf.id: item['show_in_tool_panel'] = True break rval.append(item) for wf_sa in trans.sa_session.query( trans.app.model.StoredWorkflowUserShareAssociation ).options( eagerload("stored_workflow").joinedload( "latest_workflow").undefer("step_count").lazyload("steps") ).filter_by(user=trans.user).filter( trans.app.model.StoredWorkflow.deleted == false()).order_by( desc(trans.app.model.StoredWorkflow.update_time)).all(): item = wf_sa.stored_workflow.to_dict( value_mapper={'id': trans.security.encode_id}) encoded_id = trans.security.encode_id(wf_sa.stored_workflow.id) item['url'] = url_for('workflow', id=encoded_id) item['slug'] = wf_sa.stored_workflow.slug item['owner'] = wf_sa.stored_workflow.user.username item[ 'number_of_steps'] = wf_sa.stored_workflow.latest_workflow.step_count item['show_in_tool_panel'] = False for x in user.stored_workflow_menu_entries: if x.stored_workflow_id == wf_sa.id: item['show_in_tool_panel'] = True break rval.append(item) if missing_tools: workflows_missing_tools = [] workflows = [] workflows_by_toolshed = dict() for key, value in enumerate(rval): tool_ids = [] workflow_details = self.workflow_contents_manager.workflow_to_dict( trans, self.__get_stored_workflow(trans, value['id']), style='instance') if 'steps' in workflow_details: for step in workflow_details['steps']: tool_id = workflow_details['steps'][step].get( 'tool_id') if tool_id and tool_id not in tool_ids and self.app.toolbox.is_missing_shed_tool( tool_id): tool_ids.append(tool_id) if len(tool_ids) > 0: value['missing_tools'] = tool_ids workflows_missing_tools.append(value) for workflow in workflows_missing_tools: for tool_id in workflow['missing_tools']: toolshed, _, owner, name, tool, version = tool_id.split( '/') shed_url = self.__get_full_shed_url(toolshed) repo_identifier = '/'.join([toolshed, owner, name]) if repo_identifier not in workflows_by_toolshed: workflows_by_toolshed[repo_identifier] = dict( shed=shed_url.rstrip('/'), repository=name, owner=owner, tools=[tool_id], workflows=[workflow['name']]) else: if tool_id not in workflows_by_toolshed[ repo_identifier]['tools']: workflows_by_toolshed[repo_identifier][ 'tools'].append(tool_id) if workflow['name'] not in workflows_by_toolshed[ repo_identifier]['workflows']: workflows_by_toolshed[repo_identifier][ 'workflows'].append(workflow['name']) for repo_tag in workflows_by_toolshed: workflows.append(workflows_by_toolshed[repo_tag]) return workflows return rval
def install(self, trans, **kwd): """ POST /api/tool_shed_repositories/install Initiate the installation of a repository. :param install_resolver_dependencies: True to install resolvable dependencies. :param install_tool_dependencies: True to install tool dependencies. :param install_repository_dependencies: True to install repository dependencies. :param tool_panel_section_id: The unique identifier for an existing tool panel section :param new_tool_panel_section_label: Create a new tool panel section with this label :param shed_tool_conf: The shed tool config file to use for this installation :param tool_shed_url: The URL for the toolshed whence this repository is being installed :param changeset: The changeset to update to after cloning the repository """ irm = InstallRepositoryManager(self.app) tool_shed_url = kwd.get('tool_shed_url', None) repositories = json.loads(kwd.get('repositories', '[]')) repo_info_dict = self.__get_repo_info_dict(trans, repositories, tool_shed_url) includes_tools = False includes_tools_for_display_in_tool_panel = False has_repository_dependencies = False includes_tool_dependencies = False install_resolver_dependencies = util.asbool( kwd.get('install_resolver_dependencies', False)) for encoded_repo_info_dict in repo_info_dict.get( 'repo_info_dicts', []): decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict) if not includes_tools: includes_tools = util.string_as_bool( decoded_repo_info_dict.get('includes_tools', False)) if not includes_tools_for_display_in_tool_panel: includes_tools_for_display_in_tool_panel = \ util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) ) if not has_repository_dependencies: has_repository_dependencies = util.string_as_bool( repo_info_dict.get('has_repository_dependencies', False)) if not includes_tool_dependencies: includes_tool_dependencies = util.string_as_bool( repo_info_dict.get('includes_tool_dependencies', False)) encoded_repo_info_dicts = util.listify( repo_info_dict.get('repo_info_dicts', [])) repo_info_dicts = [ encoding_util.tool_shed_decode(encoded_repo_info_dict) for encoded_repo_info_dict in encoded_repo_info_dicts ] tool_panel_section_id = kwd.get('tool_panel_section_id', None) new_tool_panel_section_label = kwd.get('new_tool_panel_section', None) tool_panel_section_mapping = json.loads( kwd.get('tool_panel_section', '{}')) install_tool_dependencies = util.asbool( kwd.get('install_tool_dependencies', False)) install_repository_dependencies = util.asbool( kwd.get('install_repository_dependencies', False)) shed_tool_conf = kwd.get('shed_tool_conf', None) tool_path = suc.get_tool_path_by_shed_tool_conf_filename( self.app, shed_tool_conf) installation_dict = dict( install_repository_dependencies=install_repository_dependencies, new_tool_panel_section_label=new_tool_panel_section_label, no_changes_checked=False, repo_info_dicts=repo_info_dicts, tool_panel_section_id=tool_panel_section_id, tool_path=tool_path, tool_shed_url=tool_shed_url) new_repositories, tool_panel_keys, repo_info_dicts, filtered_repos = irm.handle_tool_shed_repositories( installation_dict) if new_repositories: installation_dict = dict( created_or_updated_tool_shed_repositories=new_repositories, filtered_repo_info_dicts=filtered_repos, has_repository_dependencies=has_repository_dependencies, includes_tool_dependencies=includes_tool_dependencies, includes_tools=includes_tools, includes_tools_for_display_in_tool_panel= includes_tools_for_display_in_tool_panel, install_repository_dependencies=install_repository_dependencies, install_tool_dependencies=install_tool_dependencies, message='', new_tool_panel_section_label=new_tool_panel_section_label, tool_panel_section_mapping=tool_panel_section_mapping, install_resolver_dependencies=install_resolver_dependencies, shed_tool_conf=shed_tool_conf, status='ok', tool_panel_section_id=tool_panel_section_id, tool_panel_section_keys=tool_panel_keys, tool_path=tool_path, tool_shed_url=tool_shed_url) encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = \ irm.initiate_repository_installation( installation_dict ) return json.dumps( dict( operation='install', api=True, install_resolver_dependencies=install_resolver_dependencies, install_tool_dependencies=install_tool_dependencies, encoded_kwd=encoded_kwd, reinstalling=False, tool_shed_repository_ids=json.dumps( [repo[0] for repo in repositories]), repositories=[ trans.security.encode_id(repo.id) for repo in new_repositories ]))
def display_data(self, trans, data, preview=False, filename=None, to_ext=None, **kwd): """ Displays data in central pane if preview is `True`, else handles download. Datatypes should be very careful if overridding this method and this interface between datatypes and Galaxy will likely change. TOOD: Document alternatives to overridding this method (data providers?). """ headers = kwd.get("headers", {}) # Relocate all composite datatype display to a common location. composite_extensions = trans.app.datatypes_registry.get_composite_extensions() composite_extensions.append('html') # for archiving composite datatypes # Prevent IE8 from sniffing content type since we're explicit about it. This prevents intentionally text/plain # content from being rendered in the browser headers['X-Content-Type-Options'] = 'nosniff' if isinstance(data, str): return smart_str(data), headers if filename and filename != "index": # For files in extra_files_path extra_dir = data.dataset.extra_files_path_name file_path = trans.app.object_store.get_filename(data.dataset, extra_dir=extra_dir, alt_name=filename) if os.path.exists(file_path): if os.path.isdir(file_path): with tempfile.NamedTemporaryFile(mode='w', delete=False, dir=trans.app.config.new_file_path, prefix='gx_html_autocreate_') as tmp_fh: tmp_file_name = tmp_fh.name dir_items = sorted(os.listdir(file_path)) base_path, item_name = os.path.split(file_path) tmp_fh.write('<html><head><h3>Directory %s contents: %d items</h3></head>\n' % (escape(item_name), len(dir_items))) tmp_fh.write('<body><p/><table cellpadding="2">\n') for index, fname in enumerate(dir_items): if index % 2 == 0: bgcolor = '#D8D8D8' else: bgcolor = '#FFFFFF' # Can't have an href link here because there is no route # defined for files contained within multiple subdirectory # levels of the primary dataset. Something like this is # close, but not quite correct: # href = url_for(controller='dataset', action='display', # dataset_id=trans.security.encode_id(data.dataset.id), # preview=preview, filename=fname, to_ext=to_ext) tmp_fh.write(f'<tr bgcolor="{bgcolor}"><td>{escape(fname)}</td></tr>\n') tmp_fh.write('</table></body></html>\n') return self._yield_user_file_content(trans, data, tmp_file_name, headers), headers mime = mimetypes.guess_type(file_path)[0] if not mime: try: mime = trans.app.datatypes_registry.get_mimetype_by_extension(".".split(file_path)[-1]) except Exception: mime = "text/plain" self._clean_and_set_mime_type(trans, mime, headers) return self._yield_user_file_content(trans, data, file_path, headers), headers else: raise ObjectNotFound(f"Could not find '{filename}' on the extra files path {file_path}.") self._clean_and_set_mime_type(trans, data.get_mime(), headers) trans.log_event(f"Display dataset id: {str(data.id)}") from galaxy.datatypes import ( # DBTODO REMOVE THIS AT REFACTOR binary, images, text, ) if to_ext or isinstance( data.datatype, binary.Binary ): # Saving the file, or binary file if data.extension in composite_extensions: return self._archive_composite_dataset(trans, data, headers, do_action=kwd.get('do_action', 'zip')) else: headers['Content-Length'] = str(os.stat(data.file_name).st_size) filename = self._download_filename(data, to_ext, hdca=kwd.get("hdca"), element_identifier=kwd.get("element_identifier"), filename_pattern=kwd.get("filename_pattern")) headers['content-type'] = "application/octet-stream" # force octet-stream so Safari doesn't append mime extensions to filename headers["Content-Disposition"] = f'attachment; filename="{filename}"' return open(data.file_name, 'rb'), headers if not os.path.exists(data.file_name): raise ObjectNotFound(f"File Not Found ({data.file_name}).") max_peek_size = DEFAULT_MAX_PEEK_SIZE # 1 MB if isinstance(data.datatype, text.Html): max_peek_size = 10000000 # 10 MB for html preview = util.string_as_bool(preview) if ( not preview or isinstance(data.datatype, images.Image) or os.stat(data.file_name).st_size < max_peek_size ): return self._yield_user_file_content(trans, data, data.file_name, headers), headers else: headers["content-type"] = "text/html" return trans.fill_template_mako("/dataset/large_file.mako", truncated_data=open(data.file_name, 'rb').read(max_peek_size), data=data), headers
def __parse_test_attributes(output_elem, attrib, parse_elements=False, parse_discovered_datasets=False, profile=None): assert_list = __parse_assert_list(output_elem) # Allow either file or value to specify a target file to compare result with # file was traditionally used by outputs and value by extra files. file = attrib.pop('file', attrib.pop('value', None)) # File no longer required if an list of assertions was present. attributes = {} # Method of comparison attributes['compare'] = attrib.pop('compare', 'diff').lower() # Number of lines to allow to vary in logs (for dates, etc) attributes['lines_diff'] = int(attrib.pop('lines_diff', '0')) # Allow a file size to vary if sim_size compare attributes['delta'] = int(attrib.pop('delta', DEFAULT_DELTA)) attributes['delta_frac'] = float( attrib['delta_frac']) if 'delta_frac' in attrib else DEFAULT_DELTA_FRAC attributes['sort'] = string_as_bool(attrib.pop('sort', False)) attributes['decompress'] = string_as_bool(attrib.pop('decompress', False)) extra_files = [] if 'ftype' in attrib: attributes['ftype'] = attrib['ftype'] for extra in output_elem.findall('extra_files'): extra_files.append(__parse_extra_files_elem(extra)) metadata = {} for metadata_elem in output_elem.findall('metadata'): metadata[metadata_elem.get('name')] = metadata_elem.get('value') md5sum = attrib.get("md5", None) checksum = attrib.get("checksum", None) element_tests = {} if parse_elements: element_tests = __parse_element_tests(output_elem, profile=profile) primary_datasets = {} if parse_discovered_datasets: for primary_elem in (output_elem.findall("discovered_dataset") or []): primary_attrib = dict(primary_elem.attrib) designation = primary_attrib.pop('designation', None) if designation is None: raise Exception( "Test primary dataset does not have a 'designation'") primary_datasets[designation] = __parse_test_attributes( primary_elem, primary_attrib) has_checksum = md5sum or checksum has_nested_tests = extra_files or element_tests or primary_datasets if not (assert_list or file or metadata or has_checksum or has_nested_tests): raise Exception( "Test output defines nothing to check (e.g. must have a 'file' check against, assertions to check, metadata or checksum tests, etc...)" ) attributes['assert_list'] = assert_list attributes['extra_files'] = extra_files attributes['metadata'] = metadata attributes['md5'] = md5sum attributes['checksum'] = checksum attributes['elements'] = element_tests attributes['primary_datasets'] = primary_datasets return file, attributes
def _get_option_value(self, key, default): root = self.root for option_elem in root.findall("options"): if key in option_elem.attrib: return string_as_bool(option_elem.get(key)) return default
def parse_outputs(self, tool): out_elem = self.root.find("outputs") outputs = odict() output_collections = odict() if out_elem is None: return outputs, output_collections data_dict = odict() def _parse(data_elem, **kwds): output_def = self._parse_output(data_elem, tool, **kwds) data_dict[output_def.name] = output_def return output_def map(_parse, out_elem.findall("data")) for collection_elem in out_elem.findall("collection"): name = collection_elem.get( "name" ) default_format = collection_elem.get( "format", "data" ) collection_type = collection_elem.get( "type", None ) structured_like = collection_elem.get( "structured_like", None ) inherit_format = False inherit_metadata = False if structured_like: inherit_format = string_as_bool( collection_elem.get( "inherit_format", None ) ) inherit_metadata = string_as_bool( collection_elem.get( "inherit_metadata", None ) ) default_format_source = collection_elem.get( "format_source", None ) default_metadata_source = collection_elem.get( "metadata_source", "" ) dataset_collectors = None if collection_elem.find( "discover_datasets" ) is not None: dataset_collectors = output_collect.dataset_collectors_from_elem( collection_elem ) structure = galaxy.tools.ToolOutputCollectionStructure( collection_type=collection_type, structured_like=structured_like, dataset_collectors=dataset_collectors, ) output_collection = galaxy.tools.ToolOutputCollection( name, structure, default_format=default_format, inherit_format=inherit_format, inherit_metadata=inherit_metadata, default_format_source=default_format_source, default_metadata_source=default_metadata_source, ) outputs[output_collection.name] = output_collection for data_elem in collection_elem.findall("data"): _parse( data_elem, default_format=default_format, default_format_source=default_format_source, default_metadata_source=default_metadata_source, ) for data_elem in collection_elem.findall("data"): output_name = data_elem.get("name") data = data_dict[output_name] assert data del data_dict[output_name] output_collection.outputs[output_name] = data output_collections[ name ] = output_collection for output_def in data_dict.values(): outputs[output_def.name] = output_def return outputs, output_collections
def __init__(self, **kwargs): self.config_dict = kwargs self.root = kwargs.get('root_dir', '.') # Resolve paths of other config files self.__parse_config_file_options(kwargs) # Collect the umask and primary gid from the environment self.umask = os.umask(077) # get the current umask os.umask(self.umask) # can't get w/o set, so set it back self.gid = os.getgid( ) # if running under newgrp(1) we'll need to fix the group of data created on the cluster self.version_major = VERSION_MAJOR self.version = VERSION # Database related configuration self.database = resolve_path( kwargs.get("database_file", "database/community.sqlite"), self.root) self.database_connection = kwargs.get("database_connection", False) self.database_engine_options = get_database_engine_options(kwargs) self.database_create_tables = string_as_bool( kwargs.get("database_create_tables", "True")) # Repository and Tool search API self.toolshed_search_on = string_as_bool( kwargs.get("toolshed_search_on", True)) self.whoosh_index_dir = kwargs.get("whoosh_index_dir", 'database/toolshed_whoosh_indexes') self.repo_name_boost = kwargs.get("repo_name_boost", 0.9) self.repo_description_boost = kwargs.get("repo_description_boost", 0.6) self.repo_long_description_boost = kwargs.get( "repo_long_description_boost", 0.5) self.repo_homepage_url_boost = kwargs.get("repo_homepage_url_boost", 0.3) self.repo_remote_repository_url_boost = kwargs.get( "repo_remote_repository_url_boost", 0.2) self.repo_owner_username_boost = kwargs.get( "repo_owner_username_boost", 0.3) self.tool_name_boost = kwargs.get("tool_name_boost", 1.2) self.tool_description_boost = kwargs.get("tool_description_boost", 0.6) self.tool_help_boost = kwargs.get("tool_help_boost", 0.4) self.tool_repo_owner_username = kwargs.get("tool_repo_owner_username", 0.3) # Analytics self.ga_code = kwargs.get("ga_code", None) self.session_duration = int(kwargs.get('session_duration', 0)) # Where dataset files are stored self.file_path = resolve_path( kwargs.get("file_path", "database/community_files"), self.root) self.new_file_path = resolve_path( kwargs.get("new_file_path", "database/tmp"), self.root) self.cookie_path = kwargs.get("cookie_path", "/") self.enable_quotas = string_as_bool(kwargs.get('enable_quotas', False)) self.id_secret = kwargs.get("id_secret", "USING THE DEFAULT IS NOT SECURE!") # Tool stuff self.tool_path = resolve_path(kwargs.get("tool_path", "tools"), self.root) self.tool_secret = kwargs.get("tool_secret", "") self.tool_data_path = resolve_path( kwargs.get("tool_data_path", "shed-tool-data"), os.getcwd()) self.tool_data_table_config_path = None self.integrated_tool_panel_config = resolve_path( kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.root) self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'builds.txt')), self.root) self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join(self.tool_data_path, 'shared', 'ucsc', 'chrom')), self.root) self.ftp_upload_dir = kwargs.get('ftp_upload_dir', None) # Install and test framework for testing tools contained in repositories. self.display_legacy_test_results = string_as_bool( kwargs.get('display_legacy_test_results', True)) self.num_tool_test_results_saved = kwargs.get( 'num_tool_test_results_saved', 5) self.update_integrated_tool_panel = False # Galaxy flavor Docker Image self.enable_galaxy_flavor_docker_image = string_as_bool( kwargs.get("enable_galaxy_flavor_docker_image", "False")) self.use_remote_user = string_as_bool( kwargs.get("use_remote_user", "False")) self.user_activation_on = kwargs.get('user_activation_on', None) self.activation_grace_period = kwargs.get('activation_grace_period', None) self.inactivity_box_content = kwargs.get('inactivity_box_content', None) self.registration_warning_message = kwargs.get( 'registration_warning_message', None) self.terms_url = kwargs.get('terms_url', None) self.blacklist_location = kwargs.get('blacklist_file', None) self.blacklist_content = None self.remote_user_maildomain = kwargs.get("remote_user_maildomain", None) self.remote_user_header = kwargs.get("remote_user_header", 'HTTP_REMOTE_USER') self.remote_user_logout_href = kwargs.get("remote_user_logout_href", None) self.remote_user_secret = kwargs.get("remote_user_secret", None) self.require_login = string_as_bool( kwargs.get("require_login", "False")) self.allow_user_creation = string_as_bool( kwargs.get("allow_user_creation", "True")) self.allow_user_deletion = string_as_bool( kwargs.get("allow_user_deletion", "False")) self.enable_openid = string_as_bool(kwargs.get('enable_openid', False)) self.template_path = resolve_path( kwargs.get("template_path", "templates"), self.root) self.template_cache = resolve_path( kwargs.get("template_cache_path", "database/compiled_templates/community"), self.root) self.admin_users = kwargs.get("admin_users", "") self.admin_users_list = [ u.strip() for u in self.admin_users.split(',') if u ] self.mailing_join_addr = kwargs.get('mailing_join_addr', "*****@*****.**") self.error_email_to = kwargs.get('error_email_to', None) self.smtp_server = kwargs.get('smtp_server', None) self.smtp_username = kwargs.get('smtp_username', None) self.smtp_password = kwargs.get('smtp_password', None) self.smtp_ssl = kwargs.get('smtp_ssl', None) self.start_job_runners = kwargs.get('start_job_runners', None) self.email_from = kwargs.get('email_from', None) self.nginx_upload_path = kwargs.get('nginx_upload_path', False) self.log_actions = string_as_bool(kwargs.get('log_actions', 'False')) self.brand = kwargs.get('brand', None) self.pretty_datetime_format = expand_pretty_datetime_format( kwargs.get('pretty_datetime_format', '$locale (UTC)')) # Configuration for the message box directly below the masthead. self.message_box_visible = kwargs.get('message_box_visible', False) self.message_box_content = kwargs.get('message_box_content', None) self.message_box_class = kwargs.get('message_box_class', 'info') self.support_url = kwargs.get( 'support_url', 'https://wiki.galaxyproject.org/Support') self.wiki_url = kwargs.get('wiki_url', 'https://wiki.galaxyproject.org/') self.blog_url = kwargs.get('blog_url', None) self.biostar_url = kwargs.get('biostar_url', None) self.screencasts_url = kwargs.get('screencasts_url', None) self.log_events = False self.cloud_controller_instance = False self.server_name = '' self.job_manager = '' self.default_job_handlers = [] self.default_cluster_job_runner = 'local:///' self.job_handlers = [] self.tool_handlers = [] self.tool_runners = [] # Error logging with sentry self.sentry_dsn = kwargs.get('sentry_dsn', None) # Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory. self.hgweb_config_dir = resolve_path( kwargs.get('hgweb_config_dir', ''), self.root) self.disable_push = string_as_bool(kwargs.get("disable_push", "True")) # Proxy features self.apache_xsendfile = kwargs.get('apache_xsendfile', False) self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False) self.drmaa_external_runjob_script = kwargs.get( 'drmaa_external_runjob_script', None) # Parse global_conf and save the parser global_conf = kwargs.get('global_conf', None) global_conf_parser = ConfigParser.ConfigParser() self.global_conf_parser = global_conf_parser if global_conf and "__file__" in global_conf: global_conf_parser.read(global_conf['__file__']) self.running_functional_tests = string_as_bool( kwargs.get('running_functional_tests', False)) self.citation_cache_type = kwargs.get("citation_cache_type", "file") self.citation_cache_data_dir = resolve_path( kwargs.get("citation_cache_data_dir", "database/tool_shed_citations/data"), self.root) self.citation_cache_lock_dir = resolve_path( kwargs.get("citation_cache_lock_dir", "database/tool_shed_citations/locks"), self.root)
def parse_config_xml(config_xml): try: a_xml = config_xml.findall('auth')[0] access_key = a_xml.get('access_key') secret_key = a_xml.get('secret_key') b_xml = config_xml.findall('bucket')[0] bucket_name = b_xml.get('name') use_rr = string_as_bool(b_xml.get('use_reduced_redundancy', "False")) max_chunk_size = int(b_xml.get('max_chunk_size', 250)) cn_xml = config_xml.findall('connection') if not cn_xml: cn_xml = {} else: cn_xml = cn_xml[0] host = cn_xml.get('host', None) port = int(cn_xml.get('port', 6000)) multipart = string_as_bool(cn_xml.get('multipart', 'True')) is_secure = string_as_bool(cn_xml.get('is_secure', 'True')) conn_path = cn_xml.get('conn_path', '/') c_xml = config_xml.findall('cache')[0] cache_size = float(c_xml.get('size', -1)) staging_path = c_xml.get('path', None) tag, attrs = 'extra_dir', ('type', 'path') extra_dirs = config_xml.findall(tag) if not extra_dirs: msg = f'No {tag} element in XML tree' log.error(msg) raise Exception(msg) extra_dirs = [{k: e.get(k) for k in attrs} for e in extra_dirs] return { 'auth': { 'access_key': access_key, 'secret_key': secret_key, }, 'bucket': { 'name': bucket_name, 'use_reduced_redundancy': use_rr, 'max_chunk_size': max_chunk_size, }, 'connection': { 'host': host, 'port': port, 'multipart': multipart, 'is_secure': is_secure, 'conn_path': conn_path, }, 'cache': { 'size': cache_size, 'path': staging_path, }, 'extra_dirs': extra_dirs, } except Exception: # Toss it back up after logging, we can't continue loading at this point. log.exception( "Malformed ObjectStore Configuration XML -- unable to continue") raise
def load( self, trans, **kwd ): """ load( self, trans, **kwd ): * POST /api/libraries/datasets Load dataset from the given source into the library. Source can be: user directory - root folder specified in galaxy.ini as "$user_library_import_dir" example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here} the folder with the user login has to be created beforehand (admin)import directory - root folder specified in galaxy ini as "$library_import_dir" example path: path/to/galaxy/$library_import_dir/{admin can browse everything here} (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini :param encoded_folder_id: the encoded id of the folder to import dataset(s) to :type encoded_folder_id: an encoded id string :param source: source the datasets should be loaded form :type source: str :param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy while linking is set to True all symlinks will be resolved _once_ :type link_data: bool :param preserve_dirs: flag whether to preserve the directory structure when importing dir if False only datasets will be imported :type preserve_dirs: bool :param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect) :type file_type: str :param dbkey: dbkey of the loaded genome, defaults to '?' (unknown) :type dbkey: str :returns: dict containing information about the created upload job :rtype: dictionary """ kwd[ 'space_to_tab' ] = 'False' kwd[ 'to_posix_lines' ] = 'True' kwd[ 'dbkey' ] = kwd.get( 'dbkey', '?' ) kwd[ 'file_type' ] = kwd.get( 'file_type', 'auto' ) kwd[' link_data_only' ] = 'link_to_files' if util.string_as_bool( kwd.get( 'link_data', False ) ) else 'copy_files' encoded_folder_id = kwd.get( 'encoded_folder_id', None ) if encoded_folder_id is not None: folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id ) else: raise exceptions.RequestParameterMissingException( 'The required atribute encoded_folder_id is missing.' ) path = kwd.get( 'path', None) if path is None: raise exceptions.RequestParameterMissingException( 'The required atribute path is missing.' ) folder = self.folder_manager.get( trans, folder_id ) source = kwd.get( 'source', None ) if source not in [ 'userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path' ]: raise exceptions.RequestParameterMissingException( 'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ') if source in [ 'importdir_file', 'importdir_folder' ]: if not trans.user_is_admin: raise exceptions.AdminRequiredException( 'Only admins can import from importdir.' ) if not trans.app.config.library_import_dir: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from importdir.' ) import_base_dir = trans.app.config.library_import_dir path = os.path.join( import_base_dir, path ) if source in [ 'userdir_file', 'userdir_folder' ]: user_login = trans.user.email user_base_dir = trans.app.config.user_library_import_dir if user_base_dir is None: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from user directories.' ) full_dir = os.path.join( user_base_dir, user_login ) if not path.lower().startswith( full_dir.lower() ): path = os.path.join( full_dir, path ) if not os.path.exists( path ): raise exceptions.RequestParameterInvalidException( 'Given path does not exist on the host.' ) if not self.folder_manager.can_add_item( trans, folder ): raise exceptions.InsufficientPermissionsException( 'You do not have proper permission to add items to the given folder.' ) if source == 'admin_path': if not trans.app.config.allow_library_path_paste: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from path.' ) if not trans.user_is_admin: raise exceptions.AdminRequiredException( 'Only admins can import from path.' ) # Set up the traditional tool state/params tool_id = 'upload1' tool = trans.app.toolbox.get_tool( tool_id ) state = tool.new_state( trans ) tool.update_state( trans, tool.inputs_by_page[ 0 ], state.inputs, kwd ) tool_params = state.inputs dataset_upload_inputs = [] for input in tool.inputs.itervalues(): if input.type == "upload_dataset": dataset_upload_inputs.append( input ) library_bunch = upload_common.handle_library_params( trans, {}, trans.security.encode_id( folder.id ) ) abspath_datasets = [] kwd[ 'filesystem_paths' ] = path if source in [ 'importdir_folder' ]: kwd[ 'filesystem_paths' ] = os.path.join( import_base_dir, path ) params = util.Params( kwd ) # user wants to import one file only if source in [ "userdir_file", "importdir_file" ]: file = os.path.abspath( path ) abspath_datasets.append( trans.webapp.controllers[ 'library_common' ].make_library_uploaded_dataset( trans, 'api', params, os.path.basename( file ), file, 'server_dir', library_bunch ) ) # user wants to import whole folder if source == "userdir_folder": uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets( trans, 'api', params, library_bunch, 200, '' ) uploaded_datasets = uploaded_datasets_bunch[ 0 ] if uploaded_datasets is None: raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' ) for ud in uploaded_datasets: ud.path = os.path.abspath( ud.path ) abspath_datasets.append( ud ) # user wants to import from path if source in [ "admin_path", "importdir_folder" ]: # validate the path is within root uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets( trans, 'api', params, library_bunch, 200, '' ) uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' ) for ud in uploaded_datasets: ud.path = os.path.abspath( ud.path ) abspath_datasets.append( ud ) json_file_path = upload_common.create_paramfile( trans, abspath_datasets ) data_list = [ ud.data for ud in abspath_datasets ] job, output = upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=folder ) # HACK: Prevent outputs_to_working_directory from overwriting inputs when "linking" job.add_parameter( 'link_data_only', dumps( kwd.get( 'link_data_only', 'copy_files' ) ) ) job.add_parameter( 'uuid', dumps( kwd.get( 'uuid', None ) ) ) trans.sa_session.add( job ) trans.sa_session.flush() job_dict = job.to_dict() job_dict[ 'id' ] = trans.security.encode_id( job_dict[ 'id' ] ) return job_dict
def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): """ Old display method, for transition - though still used by API and test framework. Datatypes should be very careful if overridding this method and this interface between datatypes and Galaxy will likely change. TOOD: Document alternatives to overridding this method (data providers?). """ # Relocate all composite datatype display to a common location. composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) composite_extensions.append( 'html') # for archiving composite datatypes # Prevent IE8 from sniffing content type since we're explicit about it. This prevents intentionally text/plain # content from being rendered in the browser trans.response.headers['X-Content-Type-Options'] = 'nosniff' if isinstance(data, string_types): return data if filename and filename != "index": # For files in extra_files_path file_path = trans.app.object_store.get_filename( data.dataset, extra_dir='dataset_%s_files' % data.dataset.id, alt_name=filename) if os.path.exists(file_path): if os.path.isdir(file_path): return trans.show_error_message( "Directory listing is not allowed." ) # TODO: Reconsider allowing listing of directories? mime = mimetypes.guess_type(file_path)[0] if not mime: try: mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split(file_path)[-1]) except: mime = "text/plain" self._clean_and_set_mime_type(trans, mime) return open(file_path) else: return paste.httpexceptions.HTTPNotFound( "Could not find '%s' on the extra files path %s." % (filename, file_path)) self._clean_and_set_mime_type(trans, data.get_mime()) trans.log_event("Display dataset id: %s" % str(data.id)) from galaxy import datatypes # DBTODO REMOVE THIS AT REFACTOR if to_ext or isinstance( data.datatype, datatypes.binary.Binary): # Saving the file, or binary file if data.extension in composite_extensions: return self._archive_composite_dataset(trans, data, **kwd) else: trans.response.headers['Content-Length'] = int( os.stat(data.file_name).st_size) if not to_ext: to_ext = data.extension valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] trans.response.set_content_type( "application/octet-stream" ) # force octet-stream so Safari doesn't append mime extensions to filename trans.response.headers[ "Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % ( data.hid, fname, to_ext) return open(data.file_name) if not os.path.exists(data.file_name): raise paste.httpexceptions.HTTPNotFound("File Not Found (%s)." % data.file_name) max_peek_size = 1000000 # 1 MB if isinstance(data.datatype, datatypes.text.Html): max_peek_size = 10000000 # 10 MB for html preview = util.string_as_bool(preview) if not preview or isinstance( data.datatype, datatypes.images.Image) or os.stat( data.file_name).st_size < max_peek_size: if trans.app.config.sanitize_all_html and trans.response.get_content_type( ) == "text/html": # Sanitize anytime we respond with plain text/html content. # Check to see if this dataset's parent job is whitelisted # We cannot currently trust imported datasets for rendering. if not data.creating_job.imported and data.creating_job.tool_id in trans.app.config.sanitize_whitelist: return open(data.file_name).read() # This is returning to the browser, it needs to be encoded. # TODO Ideally this happens a layer higher, but this is a bad # issue affecting many tools return sanitize_html(open( data.file_name).read()).encode('utf-8') return open(data.file_name) else: trans.response.set_content_type("text/html") return trans.stream_template_mako( "/dataset/large_file.mako", truncated_data=open(data.file_name).read(max_peek_size), data=data)
def __init__(self, **kwargs): self.config_dict = kwargs self.root = kwargs.get('root_dir', '.') # Collect the umask and primary gid from the environment self.umask = os.umask(077) # get the current umask os.umask(self.umask) # can't get w/o set, so set it back self.gid = os.getgid( ) # if running under newgrp(1) we'll need to fix the group of data created on the cluster # Database related configuration self.database = resolve_path( kwargs.get("database_file", "database/universe.d"), self.root) self.database_connection = kwargs.get("database_connection", False) self.database_engine_options = get_database_engine_options(kwargs) self.database_create_tables = string_as_bool( kwargs.get("database_create_tables", "True")) # Where dataset files are stored self.file_path = resolve_path( kwargs.get("file_path", "database/files"), self.root) self.new_file_path = resolve_path( kwargs.get("new_file_path", "database/tmp"), self.root) # dataset Track files self.track_store_path = kwargs.get("track_store_path", "${extra_files_path}/tracks") self.tool_path = resolve_path(kwargs.get("tool_path", "tools"), self.root) self.tool_data_path = resolve_path( kwargs.get("tool_data_path", "tool-data"), os.getcwd()) self.test_conf = resolve_path(kwargs.get("test_conf", ""), self.root) self.tool_config = resolve_path( kwargs.get('tool_config_file', 'tool_conf.xml'), self.root) self.tool_secret = kwargs.get("tool_secret", "") self.id_secret = kwargs.get("id_secret", "USING THE DEFAULT IS NOT SECURE!") self.set_metadata_externally = string_as_bool( kwargs.get("set_metadata_externally", "False")) self.use_remote_user = string_as_bool( kwargs.get("use_remote_user", "False")) self.remote_user_maildomain = kwargs.get("remote_user_maildomain", None) self.remote_user_logout_href = kwargs.get("remote_user_logout_href", None) self.require_login = string_as_bool( kwargs.get("require_login", "False")) self.allow_user_creation = string_as_bool( kwargs.get("allow_user_creation", "True")) self.allow_user_deletion = string_as_bool( kwargs.get("allow_user_deletion", "False")) self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get("new_user_dataset_access_role_default_private", "False")) self.template_path = resolve_path( kwargs.get("template_path", "templates"), self.root) self.template_cache = resolve_path( kwargs.get("template_cache_path", "database/compiled_templates"), self.root) self.local_job_queue_workers = int( kwargs.get("local_job_queue_workers", "5")) self.cluster_job_queue_workers = int( kwargs.get("cluster_job_queue_workers", "3")) self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO") self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5")) self.cluster_files_directory = os.path.abspath( kwargs.get("cluster_files_directory", "database/pbs")) self.job_working_directory = resolve_path( kwargs.get("job_working_directory", "database/job_working_directory"), self.root) self.outputs_to_working_directory = string_as_bool( kwargs.get('outputs_to_working_directory', False)) self.output_size_limit = int(kwargs.get('output_size_limit', 0)) self.job_walltime = kwargs.get('job_walltime', None) self.admin_users = kwargs.get("admin_users", "") self.sendmail_path = kwargs.get('sendmail_path', "/usr/sbin/sendmail") self.mailing_join_addr = kwargs.get('mailing_join_addr', "*****@*****.**") self.error_email_to = kwargs.get('error_email_to', None) self.smtp_server = kwargs.get('smtp_server', None) self.start_job_runners = kwargs.get('start_job_runners', None) self.default_cluster_job_runner = kwargs.get( 'default_cluster_job_runner', 'local:///') self.pbs_application_server = kwargs.get('pbs_application_server', "") self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "") self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "") self.pbs_stage_path = kwargs.get('pbs_stage_path', "") self.use_heartbeat = string_as_bool( kwargs.get('use_heartbeat', 'False')) self.use_memdump = string_as_bool(kwargs.get('use_memdump', 'False')) self.log_memory_usage = string_as_bool( kwargs.get('log_memory_usage', 'False')) self.log_actions = string_as_bool(kwargs.get('log_actions', 'False')) self.log_events = string_as_bool(kwargs.get('log_events', 'False')) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla").lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "main,test,tair").lower().split(",") self.genetrack_display_sites = kwargs.get( 'genetrack_display_sites', "main,test").lower().split(",") self.brand = kwargs.get('brand', None) self.wiki_url = kwargs.get('wiki_url', 'http://g2.trac.bx.psu.edu/') self.bugs_email = kwargs.get('bugs_email', None) self.blog_url = kwargs.get('blog_url', None) self.screencasts_url = kwargs.get('screencasts_url', None) self.library_import_dir = kwargs.get('library_import_dir', None) if self.library_import_dir is not None and not os.path.exists( self.library_import_dir): raise ConfigurationError( "library_import_dir specified in config (%s) does not exist" % self.library_import_dir) self.user_library_import_dir = kwargs.get('user_library_import_dir', None) if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir): raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir) self.allow_library_path_paste = kwargs.get('allow_library_path_paste', False) # Configuration options for taking advantage of nginx features self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False) self.nginx_upload_store = kwargs.get('nginx_upload_store', False) self.nginx_upload_path = kwargs.get('nginx_upload_path', False) if self.nginx_upload_store: self.nginx_upload_store = os.path.abspath(self.nginx_upload_store) # Parse global_conf and save the parser global_conf = kwargs.get('global_conf', None) global_conf_parser = ConfigParser.ConfigParser() self.global_conf_parser = global_conf_parser if global_conf and "__file__" in global_conf: global_conf_parser.read(global_conf['__file__']) #Store per-tool runner config try: self.tool_runners = global_conf_parser.items("galaxy:tool_runners") except ConfigParser.NoSectionError: self.tool_runners = [] self.datatypes_config = kwargs.get('datatypes_config_file', 'datatypes_conf.xml') # Cloud configuration options self.cloud_controller_instance = string_as_bool( kwargs.get('cloud_controller_instance', 'False')) if self.cloud_controller_instance == True: self.enable_cloud_execution = string_as_bool( kwargs.get('enable_cloud_execution', 'True')) else: self.enable_cloud_execution = string_as_bool( kwargs.get('enable_cloud_execution', 'False'))
def _create(self, trans, payload, **kwd): # HACK: for now, if action is rerun, rerun tool. action = payload.get('action', None) if action == 'rerun': return self._rerun_tool(trans, payload, **kwd) # -- Execute tool. -- # Get tool. tool_version = payload.get('tool_version', None) tool = trans.app.toolbox.get_tool(payload['tool_id'], tool_version) if 'tool_id' in payload else None if not tool or not tool.allow_user_access(trans.user): raise exceptions.MessageException('Tool not found or not accessible.') if trans.app.config.user_activation_on: if not trans.user: log.warning("Anonymous user attempts to execute tool, but account activation is turned on.") elif not trans.user.active: log.warning("User \"%s\" attempts to execute tool, but account activation is turned on and user account is not active." % trans.user.email) # Set running history from payload parameters. # History not set correctly as part of this API call for # dataset upload. history_id = payload.get('history_id', None) if history_id: decoded_id = self.decode_id(history_id) target_history = self.history_manager.get_owned(decoded_id, trans.user, current_history=trans.history) else: target_history = None # Set up inputs. inputs = payload.get('inputs', {}) # Find files coming in as multipart file data and add to inputs. for k, v in payload.items(): if k.startswith('files_') or k.startswith('__files_'): inputs[k] = v # for inputs that are coming from the Library, copy them into the history input_patch = {} for k, v in inputs.items(): if isinstance(v, dict) and v.get('src', '') == 'ldda' and 'id' in v: ldda = trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(self.decode_id(v['id'])) if trans.user_is_admin or trans.app.security_agent.can_access_dataset(trans.get_current_user_roles(), ldda.dataset): input_patch[k] = ldda.to_history_dataset_association(target_history, add_to_history=True) for k, v in input_patch.items(): inputs[k] = v # TODO: encode data ids and decode ids. # TODO: handle dbkeys params = util.Params(inputs, sanitize=False) incoming = params.__dict__ # use_cached_job can be passed in via the top-level payload or among the tool inputs. # I think it should be a top-level parameter, but because the selector is implemented # as a regular tool parameter we accept both. use_cached_job = payload.get('use_cached_job', False) or util.string_as_bool(inputs.get('use_cached_job', 'false')) vars = tool.handle_input(trans, incoming, history=target_history, use_cached_job=use_cached_job) # TODO: check for errors and ensure that output dataset(s) are available. output_datasets = vars.get('out_data', []) rval = {'outputs': [], 'output_collections': [], 'jobs': [], 'implicit_collections': []} job_errors = vars.get('job_errors', []) if job_errors: # If we are here - some jobs were successfully executed but some failed. rval['errors'] = job_errors outputs = rval['outputs'] # TODO:?? poss. only return ids? for output_name, output in output_datasets: output_dict = output.to_dict() # add the output name back into the output data structure # so it's possible to figure out which newly created elements # correspond with which tool file outputs output_dict['output_name'] = output_name outputs.append(trans.security.encode_dict_ids(output_dict, skip_startswith="metadata_")) for job in vars.get('jobs', []): rval['jobs'].append(self.encode_all_ids(trans, job.to_dict(view='collection'), recursive=True)) for output_name, collection_instance in vars.get('output_collections', []): history = target_history or trans.history output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history) output_dict['output_name'] = output_name rval['output_collections'].append(output_dict) for output_name, collection_instance in vars.get('implicit_collections', {}).items(): history = target_history or trans.history output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history) output_dict['output_name'] = output_name rval['implicit_collections'].append(output_dict) return rval
def index(self, trans, deleted='False', f_email=None, f_name=None, f_any=None, **kwd): """ GET /api/users GET /api/users/deleted Displays a collection (list) of users. :param deleted: (optional) If true, show deleted users :type deleted: bool :param f_email: (optional) An email address to filter on. (Non-admin users can only use this if ``expose_user_email`` is ``True`` in galaxy.ini) :type f_email: str :param f_name: (optional) A username to filter on. (Non-admin users can only use this if ``expose_user_name`` is ``True`` in galaxy.ini) :type f_name: str :param f_any: (optional) Filter on username OR email. (Non-admin users can use this, the email filter and username filter will only be active if their corresponding ``expose_user_*`` is ``True`` in galaxy.ini) :type f_any: str """ rval = [] query = trans.sa_session.query(trans.app.model.User) deleted = util.string_as_bool(deleted) if f_email and (trans.user_is_admin or trans.app.config.expose_user_email): query = query.filter( trans.app.model.User.email.like("%%%s%%" % f_email)) if f_name and (trans.user_is_admin or trans.app.config.expose_user_name): query = query.filter( trans.app.model.User.username.like("%%%s%%" % f_name)) if f_any: if trans.user_is_admin: query = query.filter( or_(trans.app.model.User.email.like("%%%s%%" % f_any), trans.app.model.User.username.like("%%%s%%" % f_any))) else: if trans.app.config.expose_user_email and trans.app.config.expose_user_name: query = query.filter( or_( trans.app.model.User.email.like("%%%s%%" % f_any), trans.app.model.User.username.like("%%%s%%" % f_any))) elif trans.app.config.expose_user_email: query = query.filter( trans.app.model.User.email.like("%%%s%%" % f_any)) elif trans.app.config.expose_user_name: query = query.filter( trans.app.model.User.username.like("%%%s%%" % f_any)) if deleted: # only admins can see deleted users if not trans.user_is_admin: return [] query = query.filter( trans.app.model.User.table.c.deleted == true()) else: # special case: user can see only their own user # special case2: if the galaxy admin has specified that other user email/names are # exposed, we don't want special case #1 if not trans.user_is_admin and not trans.app.config.expose_user_name and not trans.app.config.expose_user_email: item = trans.user.to_dict( value_mapper={'id': trans.security.encode_id}) return [item] query = query.filter( trans.app.model.User.table.c.deleted == false()) for user in query: item = user.to_dict(value_mapper={'id': trans.security.encode_id}) # If NOT configured to expose_email, do not expose email UNLESS the user is self, or # the user is an admin if user is not trans.user and not trans.user_is_admin: expose_keys = ["id"] if trans.app.config.expose_user_name: expose_keys.append("username") if trans.app.config.expose_user_email: expose_keys.append("email") new_item = {} for key, value in item.items(): if key in expose_keys: new_item[key] = value item = new_item # TODO: move into api_values rval.append(item) return rval
def __init__( self, elem, display_application, build_sites ): filename = None data_table = None if elem.get( 'site_type', None ) is not None: filename = build_sites.get( elem.get( 'site_type' ) ) else: filename = elem.get( 'from_file', None ) if filename is None: data_table_name = elem.get( 'from_data_table', None ) if data_table_name: data_table = display_application.app.tool_data_tables.get( data_table_name, None ) assert data_table is not None, 'Unable to find data table named "%s".' % data_table_name assert filename is not None or data_table is not None,'Filename or data Table is required for dynamic_links.' skip_startswith = elem.get( 'skip_startswith', None ) separator = elem.get( 'separator', '\t' ) id_col = elem.get( 'id', None ) try: id_col = int( id_col ) except: if data_table: if id_col is None: id_col = data_table.columns.get( 'id', None ) if id_col is None: id_col = data_table.columns.get( 'value', None ) try: id_col = int( id_col ) except: # id is set to a string or None, use column by that name if available id_col = data_table.columns.get( id_col, None ) id_col = int( id_col ) name_col = elem.get( 'name', None ) try: name_col = int( name_col ) except: if data_table: if name_col is None: name_col = data_table.columns.get( 'name', None ) else: name_col = data_table.columns.get( name_col, None ) else: name_col = None if name_col is None: name_col = id_col max_col = max( id_col, name_col ) dynamic_params = {} if data_table is not None: max_col = max( [ max_col ] + data_table.columns.values() ) for key, value in data_table.columns.items(): dynamic_params[key] = { 'column': value, 'split': False, 'separator': ',' } for dynamic_param in elem.findall( 'dynamic_param' ): name = dynamic_param.get( 'name' ) value = int( dynamic_param.get( 'value' ) ) split = string_as_bool( dynamic_param.get( 'split', False ) ) param_separator = dynamic_param.get( 'separator', ',' ) max_col = max( max_col, value ) dynamic_params[name] = { 'column': value, 'split': split, 'separator': param_separator } if filename: data_iter = open( filename ) elif data_table: version, data_iter = data_table.get_version_fields() display_application.add_data_table_watch( data_table.name, version ) links = [] for line in data_iter: if isinstance( line, basestring ): if not skip_startswith or not line.startswith( skip_startswith ): line = line.rstrip( '\n\r' ) if not line: continue fields = line.split( separator ) else: continue else: fields = line if len( fields ) > max_col: new_elem = deepcopy( elem ) new_elem.set( 'id', fields[id_col] ) new_elem.set( 'name', fields[name_col] ) dynamic_values = {} for key, attributes in dynamic_params.iteritems(): value = fields[ attributes[ 'column' ] ] if attributes['split']: value = value.split( attributes['separator'] ) dynamic_values[key] = value #now populate links.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) ) else: log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) ) self.links = links
def get_bool(self, key, default): return string_as_bool(self.get(key, default))
def load(self, trans, payload=None, **kwd): """ Load dataset(s) from the given source into the library. * POST /api/libraries/datasets :param payload: dictionary structure containing: :param encoded_folder_id: the encoded id of the folder to import dataset(s) to :type encoded_folder_id: an encoded id string :param source: source the datasets should be loaded from Source can be: user directory - root folder specified in galaxy.ini as "$user_library_import_dir" example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here} the folder with the user login has to be created beforehand (admin)import directory - root folder specified in galaxy ini as "$library_import_dir" example path: path/to/galaxy/$library_import_dir/{admin can browse everything here} (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini :type source: str :param link_data: flag whether to link the dataset to data or copy it to Galaxy, defaults to copy while linking is set to True all symlinks will be resolved _once_ :type link_data: bool :param preserve_dirs: flag whether to preserve the directory structure when importing dir if False only datasets will be imported :type preserve_dirs: bool :param file_type: file type of the loaded datasets, defaults to 'auto' (autodetect) :type file_type: str :param dbkey: dbkey of the loaded genome, defaults to '?' (unknown) :type dbkey: str :param tag_using_filenames: flag whether to generate dataset tags from filenames :type tag_using_filenames: bool :type dictionary :returns: dict containing information about the created upload job :rtype: dictionary :raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException InsufficientPermissionsException, ObjectNotFound """ if payload: kwd.update(payload) kwd['space_to_tab'] = False kwd['to_posix_lines'] = True kwd['dbkey'] = kwd.get('dbkey', '?') kwd['file_type'] = kwd.get('file_type', 'auto') kwd['link_data_only'] = 'link_to_files' if util.string_as_bool( kwd.get('link_data', False)) else 'copy_files' kwd['tag_using_filenames'] = util.string_as_bool( kwd.get('tag_using_filenames', None)) encoded_folder_id = kwd.get('encoded_folder_id', None) if encoded_folder_id is not None: folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id) else: raise exceptions.RequestParameterMissingException( 'The required attribute encoded_folder_id is missing.') path = kwd.get('path', None) if path is None: raise exceptions.RequestParameterMissingException( 'The required attribute path is missing.') folder = self.folder_manager.get(trans, folder_id) source = kwd.get('source', None) if source not in [ 'userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path' ]: raise exceptions.RequestParameterMissingException( 'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ' ) elif source in ['importdir_file', 'importdir_folder']: if not trans.user_is_admin(): raise exceptions.AdminRequiredException( 'Only admins can import from importdir.') if not trans.app.config.library_import_dir: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from importdir.' ) import_base_dir = trans.app.config.library_import_dir if not safe_relpath(path): # admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed. # the reasoning here is that galaxy admins may not have direct filesystem access or can only access # library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the # import directory. if they have filesystem access, all bets are off. raise exceptions.RequestParameterInvalidException( 'The given path is invalid.') path = os.path.join(import_base_dir, path) elif source in ['userdir_file', 'userdir_folder']: unsafe = None username = trans.user.username if trans.app.config.user_library_import_check_permissions else None user_login = trans.user.email user_base_dir = trans.app.config.user_library_import_dir if user_base_dir is None: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from user directories.' ) full_dir = os.path.join(user_base_dir, user_login) if not safe_contains(full_dir, path, whitelist=trans.app.config. user_library_import_symlink_whitelist): # the path is a symlink outside the user dir path = os.path.join(full_dir, path) log.error( 'User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', path, os.path.realpath(path)) raise exceptions.RequestParameterInvalidException( 'The given path is invalid.') if trans.app.config.user_library_import_check_permissions and not full_path_permission_for_user( full_dir, path, username): log.error( 'User attempted to import a path that resolves to a path outside of their import dir: ' '%s -> %s and cannot be read by them.', path, os.path.realpath(path)) raise exceptions.RequestParameterInvalidException( 'The given path is invalid.') path = os.path.join(full_dir, path) for unsafe in unsafe_walk( path, whitelist=[full_dir] + trans.app.config.user_library_import_symlink_whitelist, username=username): # the path is a dir and contains files that symlink outside the user dir error = 'User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', \ path, os.path.realpath(path) if trans.app.config.user_library_import_check_permissions: error += ' or is not readable for them.' log.error(error) if unsafe: raise exceptions.RequestParameterInvalidException( 'The given path is invalid.') if not os.path.exists(path): raise exceptions.RequestParameterInvalidException( 'Given path does not exist on the host.') if not self.folder_manager.can_add_item(trans, folder): raise exceptions.InsufficientPermissionsException( 'You do not have proper permission to add items to the given folder.' ) elif source == 'admin_path': if not trans.app.config.allow_library_path_paste: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from path.' ) if not trans.user_is_admin(): raise exceptions.AdminRequiredException( 'Only admins can import from path.') # Set up the traditional tool state/params tool_id = 'upload1' tool = trans.app.toolbox.get_tool(tool_id) state = tool.new_state(trans) populate_state(trans, tool.inputs, kwd, state.inputs) tool_params = state.inputs dataset_upload_inputs = [] for input in tool.inputs.values(): if input.type == "upload_dataset": dataset_upload_inputs.append(input) library_bunch = upload_common.handle_library_params( trans, {}, trans.security.encode_id(folder.id)) abspath_datasets = [] kwd['filesystem_paths'] = path if source in ['importdir_folder']: kwd['filesystem_paths'] = os.path.join(import_base_dir, path) # user wants to import one file only elif source in ["userdir_file", "importdir_file"]: file = os.path.abspath(path) abspath_datasets.append( self._make_library_uploaded_dataset(trans, kwd, os.path.basename(file), file, 'server_dir', library_bunch)) # user wants to import whole folder elif source == "userdir_folder": uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets( trans, kwd, library_bunch, 200, '') uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.') for ud in uploaded_datasets: ud.path = os.path.abspath(ud.path) abspath_datasets.append(ud) # user wants to import from path if source in ["admin_path", "importdir_folder"]: # validate the path is within root uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets( trans, kwd, library_bunch, 200, '') uploaded_datasets = uploaded_datasets_bunch[0] if uploaded_datasets is None: raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.') for ud in uploaded_datasets: ud.path = os.path.abspath(ud.path) abspath_datasets.append(ud) json_file_path = upload_common.create_paramfile( trans, abspath_datasets) data_list = [ud.data for ud in abspath_datasets] job_params = {} job_params['link_data_only'] = dumps( kwd.get('link_data_only', 'copy_files')) job_params['uuid'] = dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=folder, job_params=job_params) trans.sa_session.add(job) trans.sa_session.flush() job_dict = job.to_dict() job_dict['id'] = trans.security.encode_id(job_dict['id']) return job_dict
def is_shed_tool_conf(self): has_tool_path = self.parse_tool_path() is not None is_shed_conf = string_as_bool(self.root.get("is_shed_conf", "True")) return has_tool_path and is_shed_conf
def parse_monitor(self): return string_as_bool(self.root.get('monitor', DEFAULT_MONITOR))
def create(self, trans, library_id, payload, **kwd): """ create( self, trans, library_id, payload, **kwd ) * POST /api/libraries/{library_id}/contents: create a new library file or folder To copy an HDA into a library send ``create_type`` of 'file' and the HDA's encoded id in ``from_hda_id`` (and optionally ``ldda_message``). To copy an HDCA into a library send ``create_type`` of 'file' and the HDCA's encoded id in ``from_hdca_id`` (and optionally ``ldda_message``). :type library_id: str :param library_id: the encoded id of the library where to create the new item :type payload: dict :param payload: dictionary structure containing: * folder_id: the encoded id of the parent folder of the new item * create_type: the type of item to create ('file', 'folder' or 'collection') * from_hda_id: (optional, only if create_type is 'file') the encoded id of an accessible HDA to copy into the library * ldda_message: (optional) the new message attribute of the LDDA created * extended_metadata: (optional) sub-dictionary containing any extended metadata to associate with the item * upload_option: (optional) one of 'upload_file' (default), 'upload_directory' or 'upload_paths' * server_dir: (optional, only if upload_option is 'upload_directory') relative path of the subdirectory of Galaxy ``library_import_dir`` (if admin) or ``user_library_import_dir`` (if non-admin) to upload. All and only the files (i.e. no subdirectories) contained in the specified directory will be uploaded. * filesystem_paths: (optional, only if upload_option is 'upload_paths' and the user is an admin) file paths on the Galaxy server to upload to the library, one file per line * link_data_only: (optional, only when upload_option is 'upload_directory' or 'upload_paths') either 'copy_files' (default) or 'link_to_files'. Setting to 'link_to_files' symlinks instead of copying the files * name: (optional, only if create_type is 'folder') name of the folder to create * description: (optional, only if create_type is 'folder') description of the folder to create * tag_using_filename: (optional) create tags on datasets using the file's original name :returns: a dictionary describing the new item unless ``from_hdca_id`` is supplied, in that case a list of such dictionaries is returned. :rtype: object """ if 'create_type' not in payload: trans.response.status = 400 return "Missing required 'create_type' parameter." else: create_type = payload.pop('create_type') if create_type not in ('file', 'folder', 'collection'): trans.response.status = 400 return "Invalid value for 'create_type' parameter ( %s ) specified." % create_type if 'folder_id' not in payload: trans.response.status = 400 return "Missing required 'folder_id' parameter." else: folder_id = payload.pop('folder_id') class_name, folder_id = self._decode_library_content_id(folder_id) try: # security is checked in the downstream controller parent = self.get_library_folder(trans, folder_id, check_ownership=False, check_accessible=False) except Exception as e: return str(e) # The rest of the security happens in the library_common controller. real_folder_id = trans.security.encode_id(parent.id) payload['tag_using_filenames'] = util.string_as_bool(payload.get('tag_using_filenames', None)) # are we copying an HDA to the library folder? # we'll need the id and any message to attach, then branch to that private function from_hda_id, from_hdca_id, ldda_message = (payload.pop('from_hda_id', None), payload.pop('from_hdca_id', None), payload.pop('ldda_message', '')) if create_type == 'file': if from_hda_id: return self._copy_hda_to_library_folder(trans, self.hda_manager, self.decode_id(from_hda_id), real_folder_id, ldda_message) if from_hdca_id: return self._copy_hdca_to_library_folder(trans, self.hda_manager, self.decode_id(from_hdca_id), real_folder_id, ldda_message) # check for extended metadata, store it and pop it out of the param # otherwise sanitize_param will have a fit ex_meta_payload = payload.pop('extended_metadata', None) # Now create the desired content object, either file or folder. if create_type == 'file': status, output = self._upload_library_dataset(trans, library_id, real_folder_id, **payload) elif create_type == 'folder': status, output = self._create_folder(trans, real_folder_id, library_id, **payload) elif create_type == 'collection': # Not delegating to library_common, so need to check access to parent # folder here. self.check_user_can_add_to_library_item(trans, parent, check_accessible=True) create_params = api_payload_to_create_params(payload) create_params['parent'] = parent service = trans.app.dataset_collections_service dataset_collection_instance = service.create(**create_params) return [dictify_dataset_collection_instance(dataset_collection_instance, security=trans.security, parent=parent)] if status != 200: trans.response.status = status return output else: rval = [] for v in output.values(): if ex_meta_payload is not None: # If there is extended metadata, store it, attach it to the dataset, and index it ex_meta = ExtendedMetadata(ex_meta_payload) trans.sa_session.add(ex_meta) v.extended_metadata = ex_meta trans.sa_session.add(v) trans.sa_session.flush() for path, value in self._scan_json_block(ex_meta_payload): meta_i = ExtendedMetadataIndex(ex_meta, path, value) trans.sa_session.add(meta_i) trans.sa_session.flush() if type(v) == trans.app.model.LibraryDatasetDatasetAssociation: v = v.library_dataset encoded_id = trans.security.encode_id(v.id) if create_type == 'folder': encoded_id = 'F' + encoded_id rval.append(dict(id=encoded_id, name=v.name, url=url_for('library_content', library_id=library_id, id=encoded_id))) return rval
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): super().__init__(app_info=app_info, **kwds) self.namespace = namespace self.hash_func = hash_func self.auto_install = string_as_bool(auto_install)
def delete(self, trans, library_id, id, **kwd): """ delete( self, trans, library_id, id, **kwd ) * DELETE /api/libraries/{library_id}/contents/{id} delete the LibraryDataset with the given ``id`` :type id: str :param id: the encoded id of the library dataset to delete :type kwd: dict :param kwd: (optional) dictionary structure containing: * payload: a dictionary itself containing: * purge: if True, purge the LD :rtype: dict :returns: an error object if an error occurred or a dictionary containing: * id: the encoded id of the library dataset, * deleted: if the library dataset was marked as deleted, * purged: if the library dataset was purged """ # a request body is optional here purge = False if kwd.get('payload', None): purge = util.string_as_bool(kwd['payload'].get('purge', False)) rval = {'id': id} try: ld = self.get_library_dataset(trans, id, check_ownership=False, check_accessible=True) user_is_admin = trans.user_is_admin can_modify = trans.app.security_agent.can_modify_library_item(trans.user.all_roles(), ld) log.debug('is_admin: %s, can_modify: %s', user_is_admin, can_modify) if not (user_is_admin or can_modify): trans.response.status = 403 rval.update({'error': 'Unauthorized to delete or purge this library dataset'}) return rval ld.deleted = True if purge: ld.purged = True trans.sa_session.add(ld) trans.sa_session.flush() # TODO: had to change this up a bit from Dataset.user_can_purge dataset = ld.library_dataset_dataset_association.dataset no_history_assoc = len(dataset.history_associations) == len(dataset.purged_history_associations) no_library_assoc = dataset.library_associations == [ld.library_dataset_dataset_association] can_purge_dataset = not dataset.purged and no_history_assoc and no_library_assoc if can_purge_dataset: try: ld.library_dataset_dataset_association.dataset.full_delete() trans.sa_session.add(ld.dataset) except Exception: pass # flush now to preserve deleted state in case of later interruption trans.sa_session.flush() rval['purged'] = True trans.sa_session.flush() rval['deleted'] = True except exceptions.httpexceptions.HTTPInternalServerError: log.exception('Library_contents API, delete: uncaught HTTPInternalServerError: %s, %s', id, str(kwd)) raise except exceptions.httpexceptions.HTTPException: raise except Exception as exc: log.exception('library_contents API, delete: uncaught exception: %s, %s', id, str(kwd)) trans.response.status = 500 rval.update({'error': str(exc)}) return rval
def __init__(self, elem, parent): ExternalServiceParameter.__init__(self, elem, parent) self.strip = string_as_bool(elem.get('strip', 'False')) self.text = elem.text