def __delete_dataset( self, trans, history_id, id, purge, **kwd ):
        # get purge from the query or from the request body payload (a request body is optional here)
        purge = util.string_as_bool( purge )
        if kwd.get( 'payload', None ):
            # payload takes priority
            purge = util.string_as_bool( kwd['payload'].get( 'purge', purge ) )

        hda = self.mgrs.hdas.get( trans, self._decode_id( trans, id ),
            check_ownership=True, check_accessible=True )
        self.mgrs.hdas.err_if_uploading( trans, hda )

        rval = { 'id' : id }
        hda.deleted = True
        if purge:
            if not trans.app.config.allow_user_dataset_purge:
                raise exceptions.ConfigDoesNotAllowException( 'This instance does not allow user dataset purging' )
            hda.purged = True
            trans.sa_session.add( hda )
            trans.sa_session.flush()
            if hda.dataset.user_can_purge:
                try:
                    hda.dataset.full_delete()
                    trans.sa_session.add( hda.dataset )
                except:
                    pass
                # flush now to preserve deleted state in case of later interruption
                trans.sa_session.flush()
            rval[ 'purged' ] = True

        self.stop_hda_creating_job( hda )
        trans.sa_session.flush()

        rval[ 'deleted' ] = True
        return rval
Esempio n. 2
0
 def __init__(self, **kwargs):
     self.app = kwargs['app']
     self.verbose = string_as_bool(kwargs.get('verbose', False))
     self.user_submission = string_as_bool(kwargs.get('user_submission', False))
     self.report_directory = kwargs.get("directory", tempfile.gettempdir())
     if not os.path.exists(self.report_directory):
         os.makedirs(self.report_directory)
Esempio n. 3
0
 def _parse_output(
     self,
     data_elem,
     tool,
     default_format="data",
     default_format_source=None,
     default_metadata_source="",
 ):
     output = ToolOutput( data_elem.get("name") )
     output_format = data_elem.get("format", default_format)
     auto_format = string_as_bool( data_elem.get( "auto_format", "false" ) )
     if auto_format and output_format != "data":
         raise ValueError("Setting format and auto_format is not supported at this time.")
     elif auto_format:
         output_format = "_sniff_"
     output.format = output_format
     output.change_format = data_elem.findall("change_format")
     output.format_source = data_elem.get("format_source", default_format_source)
     output.metadata_source = data_elem.get("metadata_source", default_metadata_source)
     output.parent = data_elem.get("parent", None)
     output.label = xml_text( data_elem, "label" )
     output.count = int( data_elem.get("count", 1) )
     output.filters = data_elem.findall( 'filter' )
     output.tool = tool
     output.from_work_dir = data_elem.get("from_work_dir", None)
     output.hidden = string_as_bool( data_elem.get("hidden", "") )
     output.actions = ToolOutputActionGroup( output, data_elem.find( 'actions' ) )
     output.dataset_collector_descriptions = dataset_collector_descriptions_from_elem( data_elem, legacy=self.legacy_defaults )
     return output
Esempio n. 4
0
    def __api_import_new_workflow(self, trans, payload, **kwd):
        data = payload["workflow"]

        publish = util.string_as_bool(payload.get("publish", False))
        # If 'publish' set, default to importable.
        importable = util.string_as_bool(payload.get("importable", publish))

        if publish and not importable:
            raise exceptions.RequestParameterInvalidException("Published workflow must be importable.")

        from_dict_kwds = dict(source="API", publish=publish)
        workflow, missing_tool_tups = self._workflow_from_dict(trans, data, **from_dict_kwds)

        if importable:
            self._make_item_accessible(trans.sa_session, workflow)
            trans.sa_session.flush()

        # galaxy workflow newly created id
        workflow_id = workflow.id
        # api encoded, id
        encoded_id = trans.security.encode_id(workflow_id)

        # return list
        rval = []

        item = workflow.to_dict(value_mapper={"id": trans.security.encode_id})
        item["url"] = url_for("workflow", id=encoded_id)

        rval.append(item)

        return item
Esempio n. 5
0
 def from_elem( cls, xml_root ):
     provider_elem = xml_root
     provider_id = provider_elem.get( 'id', None )
     provider_name = provider_elem.get( 'name', provider_id )
     op_endpoint_url = provider_elem.find( 'op_endpoint_url' )
     if op_endpoint_url is not None:
         op_endpoint_url = op_endpoint_url.text
     never_associate_with_user = string_as_bool( provider_elem.get( 'never_associate_with_user', 'False' ) )
     assert (provider_id and provider_name and op_endpoint_url), Exception( "OpenID Provider improperly configured" )
     assert provider_id not in RESERVED_PROVIDER_IDS, Exception( 'Specified OpenID Provider uses a reserved id: %s' % ( provider_id ) )
     sreg_required = []
     sreg_optional = []
     use_for = {}
     store_user_preference = {}
     use_default_sreg = True
     for elem in provider_elem.findall( 'sreg' ):
         use_default_sreg = False
         for field_elem in elem.findall( 'field' ):
             sreg_name = field_elem.get( 'name' )
             assert sreg_name, Exception( 'A name is required for a sreg element' )
             if string_as_bool( field_elem.get( 'required' ) ):
                 sreg_required.append( sreg_name )
             else:
                 sreg_optional.append( sreg_name )
             for use_elem in field_elem.findall( 'use_for' ):
                 use_for[ use_elem.get( 'name' ) ] = sreg_name
             for store_user_preference_elem in field_elem.findall( 'store_user_preference' ):
                 store_user_preference[ store_user_preference_elem.get( 'name' ) ] = sreg_name
     if use_default_sreg:
         sreg_required = None
         sreg_optional = None
         use_for = None
     return cls( provider_id, provider_name, op_endpoint_url, sreg_required=sreg_required, sreg_optional=sreg_optional, use_for=use_for, store_user_preference=store_user_preference, never_associate_with_user=never_associate_with_user )
Esempio n. 6
0
 def _get_attribute_as_bool( self, attribute, default, elem=None ):
     if elem is None:
         elem = self.root
     return string_as_bool( elem.get( attribute, default ) )
Esempio n. 7
0
    def _create(self, trans: GalaxyWebTransaction, payload, **kwd):
        action = payload.get('action')
        if action == 'rerun':
            raise Exception("'rerun' action has been deprecated")

        # Get tool.
        tool_version = payload.get('tool_version')
        tool_id = payload.get('tool_id')
        tool_uuid = payload.get('tool_uuid')
        get_kwds = dict(
            tool_id=tool_id,
            tool_uuid=tool_uuid,
            tool_version=tool_version,
        )
        if tool_id is None and tool_uuid is None:
            raise exceptions.RequestParameterMissingException("Must specify either a tool_id or a tool_uuid.")

        tool = trans.app.toolbox.get_tool(**get_kwds)
        if not tool:
            log.debug(f"Not found tool with kwds [{get_kwds}]")
            raise exceptions.ToolMissingException('Tool not found.')
        if not tool.allow_user_access(trans.user):
            raise exceptions.ItemAccessibilityException('Tool not accessible.')
        if trans.app.config.user_activation_on:
            if not trans.user:
                log.warning("Anonymous user attempts to execute tool, but account activation is turned on.")
            elif not trans.user.active:
                log.warning(f"User \"{trans.user.email}\" attempts to execute tool, but account activation is turned on and user account is not active.")

        # Set running history from payload parameters.
        # History not set correctly as part of this API call for
        # dataset upload.
        history_id = payload.get('history_id')
        if history_id:
            decoded_id = self.decode_id(history_id)
            target_history = self.history_manager.get_owned(decoded_id, trans.user, current_history=trans.history)
        else:
            target_history = None

        # Set up inputs.
        inputs = payload.get('inputs', {})
        if not isinstance(inputs, dict):
            raise exceptions.RequestParameterInvalidException(f"inputs invalid {inputs}")

        # Find files coming in as multipart file data and add to inputs.
        for k, v in payload.items():
            if k.startswith('files_') or k.startswith('__files_'):
                inputs[k] = v

        # for inputs that are coming from the Library, copy them into the history
        self._patch_library_inputs(trans, inputs, target_history)

        # TODO: encode data ids and decode ids.
        # TODO: handle dbkeys
        params = util.Params(inputs, sanitize=False)
        incoming = params.__dict__

        # use_cached_job can be passed in via the top-level payload or among the tool inputs.
        # I think it should be a top-level parameter, but because the selector is implemented
        # as a regular tool parameter we accept both.
        use_cached_job = payload.get('use_cached_job', False) or util.string_as_bool(inputs.get('use_cached_job', 'false'))

        input_format = str(payload.get('input_format', 'legacy'))

        vars = tool.handle_input(trans, incoming, history=target_history, use_cached_job=use_cached_job, input_format=input_format)

        # TODO: check for errors and ensure that output dataset(s) are available.
        output_datasets = vars.get('out_data', [])
        rval: Dict[str, Any] = {'outputs': [], 'output_collections': [], 'jobs': [], 'implicit_collections': []}
        rval['produces_entry_points'] = tool.produces_entry_points
        job_errors = vars.get('job_errors', [])
        if job_errors:
            # If we are here - some jobs were successfully executed but some failed.
            rval['errors'] = job_errors

        outputs = rval['outputs']
        # TODO:?? poss. only return ids?
        for output_name, output in output_datasets:
            output_dict = output.to_dict()
            # add the output name back into the output data structure
            # so it's possible to figure out which newly created elements
            # correspond with which tool file outputs
            output_dict['output_name'] = output_name
            outputs.append(trans.security.encode_dict_ids(output_dict, skip_startswith="metadata_"))

        new_pja_flush = False
        for job in vars.get('jobs', []):
            rval['jobs'].append(self.encode_all_ids(trans, job.to_dict(view='collection'), recursive=True))
            if inputs.get('send_email_notification', False):
                # Unless an anonymous user is invoking this via the API it
                # should never be an option, but check and enforce that here
                if trans.user is None:
                    raise exceptions.ToolExecutionError("Anonymously run jobs cannot send an email notification.")
                else:
                    job_email_action = PostJobAction('EmailAction')
                    job.add_post_job_action(job_email_action)
                    new_pja_flush = True

        if new_pja_flush:
            trans.sa_session.flush()

        for output_name, collection_instance in vars.get('output_collections', []):
            history = target_history or trans.history
            output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history)
            output_dict['output_name'] = output_name
            rval['output_collections'].append(output_dict)

        for output_name, collection_instance in vars.get('implicit_collections', {}).items():
            history = target_history or trans.history
            output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history)
            output_dict['output_name'] = output_name
            rval['implicit_collections'].append(output_dict)

        return rval
Esempio n. 8
0
 def parse_hidden(self):
     hidden = xml_text(self.root, "hidden")
     if hidden:
         hidden = string_as_bool(hidden)
     return hidden
Esempio n. 9
0
from galaxy.tools.util import maf_utilities
from galaxy.util import string_as_bool

assert sys.version_info[:2] >= ( 2, 4 )

def __main__():    
    try:
        maf_reader = maf.Reader( open( sys.argv[1] ) )
    except Exception, e:
        maf_utilities.tool_fail( "Error opening MAF: %s" % e )
    try:
        out = maf.Writer( open( sys.argv[2], "w") )
    except Exception, e:
        maf_utilities.tool_fail( "Error opening file for output: %s" % e )
    try:
        collapse_columns = string_as_bool( sys.argv[3] )
    except Exception, e:
        maf_utilities.tool_fail( "Error determining collapse columns value: %s" % e )
    
    start_count = 0
    end_count = 0
    for start_count, start_block in enumerate( maf_reader ):
        for block in maf_utilities.iter_blocks_split_by_species( start_block ):
            if collapse_columns:
                block.remove_all_gap_columns()
            out.write( block )
            end_count += 1
    out.close()
    
    if end_count:
        print "%i alignment blocks created from %i original blocks." % ( end_count, start_count + 1 )
Esempio n. 10
0
    def create(self, trans, cntrller='user', redirect_url='', refresh_frames=None, **kwd):
        refresh_frames = refresh_frames or []
        params = util.Params(kwd)
        # If the honeypot field is not empty we are dealing with a bot.
        honeypot_field = params.get('bear_field', '')
        if honeypot_field != '':
            return trans.show_error_message("You've been flagged as a possible bot. If you are not, please try registering again and fill the form out carefully. <a target=\"_top\" href=\"%s\">Go to the home page</a>.") % url_for('/')

        message = util.restore_text(params.get('message', ''))
        status = params.get('status', 'done')
        use_panels = util.string_as_bool(kwd.get('use_panels', True))
        email = util.restore_text(params.get('email', ''))
        # Do not sanitize passwords, so take from kwd
        # instead of params ( which were sanitized )
        password = kwd.get('password', '')
        confirm = kwd.get('confirm', '')
        username = util.restore_text(params.get('username', ''))
        subscribe = params.get('subscribe', '')
        subscribe_checked = CheckboxField.is_checked(subscribe)
        referer = trans.request.referer or ''
        redirect = kwd.get('redirect', referer).strip()
        is_admin = trans.user_is_admin
        success = False
        show_user_prepopulate_form = False
        if not trans.app.config.allow_user_creation and not trans.user_is_admin:
            message = 'User registration is disabled.  Please contact your local Galaxy administrator for an account.'
            if trans.app.config.error_email_to is not None:
                message += f' Contact: {trans.app.config.error_email_to}'
            status = 'error'
        else:
            # check user is allowed to register
            message, status = trans.app.auth_manager.check_registration_allowed(email, username, password)
            if not message:
                # Create the user, save all the user info and login to Galaxy
                if params.get('create_user_button', False):
                    # Check email and password validity
                    message = self.__validate(trans, email, password, confirm, username)
                    if not message:
                        # All the values are valid
                        message, status, user, success = self.__register(trans, subscribe_checked=subscribe_checked, **kwd)
                        if success and not is_admin:
                            # The handle_user_login() method has a call to the history_set_default_permissions() method
                            # (needed when logging in with a history), user needs to have default permissions set before logging in
                            trans.handle_user_login(user)
                            trans.log_event("User created a new account")
                            trans.log_event("User logged in")
                    else:
                        status = 'error'
        registration_warning_message = trans.app.config.registration_warning_message
        if success:
            if is_admin:
                redirect_url = web.url_for('/admin/users?status=success&message=Created new user account.')
            else:
                redirect_url = web.url_for('/')
        return trans.fill_template('/webapps/tool_shed/user/register.mako',
                                   cntrller=cntrller,
                                   email=email,
                                   username=username,
                                   subscribe_checked=subscribe_checked,
                                   show_user_prepopulate_form=show_user_prepopulate_form,
                                   use_panels=use_panels,
                                   redirect=redirect,
                                   redirect_url=redirect_url,
                                   refresh_frames=refresh_frames,
                                   registration_warning_message=registration_warning_message,
                                   message=message,
                                   status=status)
Esempio n. 11
0
 def get_bool(self, key, default):
     if key in self.config_dict:
         return string_as_bool(self.config_dict[key])
     else:
         return default
Esempio n. 12
0
 def __init__(self, spec):
     MetadataParameter.__init__(self, spec)
     self.values = self.spec.get("values")
     self.multiple = string_as_bool(self.spec.get("multiple"))
Esempio n. 13
0
    def __api_import_new_workflow(self, trans, payload, **kwd):
        data = payload['workflow']

        import_tools = util.string_as_bool(payload.get("import_tools", False))
        if import_tools and not trans.user_is_admin():
            raise exceptions.AdminRequiredException()

        publish = util.string_as_bool(payload.get("publish", False))
        # If 'publish' set, default to importable.
        importable = util.string_as_bool(payload.get("importable", publish))
        # Galaxy will try to upgrade tool versions that don't match exactly during import,
        # this prevents that.
        exact_tools = util.string_as_bool(payload.get("exact_tools", False))

        if publish and not importable:
            raise exceptions.RequestParameterInvalidException(
                "Published workflow must be importable.")

        from_dict_kwds = dict(
            source="API",
            publish=publish,
            exact_tools=exact_tools,
        )
        workflow, missing_tool_tups = self._workflow_from_dict(
            trans, data, **from_dict_kwds)

        if importable:
            self._make_item_accessible(trans.sa_session, workflow)
            trans.sa_session.flush()

        # galaxy workflow newly created id
        workflow_id = workflow.id
        # api encoded, id
        encoded_id = trans.security.encode_id(workflow_id)

        # return list
        rval = []

        item = workflow.to_dict(value_mapper={'id': trans.security.encode_id})
        item['url'] = url_for('workflow', id=encoded_id)
        item['owner'] = workflow.user.username
        item['number_of_steps'] = len(workflow.latest_workflow.steps)
        rval.append(item)

        #
        if import_tools:
            tools = {}
            for key in data['steps']:
                item = data['steps'][key]
                if item is not None:
                    if 'tool_shed_repository' in item:
                        tool_shed_repository = item['tool_shed_repository']
                        if 'owner' in tool_shed_repository and 'changeset_revision' in tool_shed_repository and 'name' in tool_shed_repository and 'tool_shed' in tool_shed_repository:
                            toolstr = tool_shed_repository['owner'] \
                                + tool_shed_repository['changeset_revision'] \
                                + tool_shed_repository['name'] \
                                + tool_shed_repository['tool_shed']
                            tools[toolstr] = tool_shed_repository
            irm = InstallRepositoryManager(self.app)
            for k in tools:
                item = tools[k]
                tool_shed_url = 'https://' + item['tool_shed'] + '/'
                name = item['name']
                owner = item['owner']
                changeset_revision = item['changeset_revision']
                irm.install(tool_shed_url, name, owner, changeset_revision,
                            payload)
        return item
Esempio n. 14
0
    def get_workflows_list(self, trans, kwd):
        """
        Displays a collection of workflows.

        :param  show_published:      if True, show also published workflows
        :type   show_published:      boolean
        :param  missing_tools:       if True, include a list of missing tools per workflow
        :type   missing_tools:       boolean
        """
        show_published = util.string_as_bool(kwd.get('show_published',
                                                     'False'))
        missing_tools = util.string_as_bool(kwd.get('missing_tools', 'False'))
        rval = []
        filter1 = (trans.app.model.StoredWorkflow.user == trans.user)
        user = trans.get_user()
        if show_published:
            filter1 = or_(filter1,
                          (trans.app.model.StoredWorkflow.published == true()))
        for wf in trans.sa_session.query(
                trans.app.model.StoredWorkflow).options(
                    eagerload("latest_workflow").undefer(
                        "step_count").lazyload("steps")).filter(
                            filter1,
                            trans.app.model.StoredWorkflow.table.c.deleted ==
                            false()).order_by(
                                desc(trans.app.model.StoredWorkflow.table.c.
                                     update_time)).all():

            item = wf.to_dict(value_mapper={'id': trans.security.encode_id})
            encoded_id = trans.security.encode_id(wf.id)
            item['url'] = url_for('workflow', id=encoded_id)
            item['owner'] = wf.user.username
            item['number_of_steps'] = wf.latest_workflow.step_count
            item['show_in_tool_panel'] = False
            for x in user.stored_workflow_menu_entries:
                if x.stored_workflow_id == wf.id:
                    item['show_in_tool_panel'] = True
                    break
            rval.append(item)
        for wf_sa in trans.sa_session.query(
                trans.app.model.StoredWorkflowUserShareAssociation
        ).options(
                eagerload("stored_workflow").joinedload(
                    "latest_workflow").undefer("step_count").lazyload("steps")
        ).filter_by(user=trans.user).filter(
                trans.app.model.StoredWorkflow.deleted == false()).order_by(
                    desc(trans.app.model.StoredWorkflow.update_time)).all():
            item = wf_sa.stored_workflow.to_dict(
                value_mapper={'id': trans.security.encode_id})
            encoded_id = trans.security.encode_id(wf_sa.stored_workflow.id)
            item['url'] = url_for('workflow', id=encoded_id)
            item['slug'] = wf_sa.stored_workflow.slug
            item['owner'] = wf_sa.stored_workflow.user.username
            item[
                'number_of_steps'] = wf_sa.stored_workflow.latest_workflow.step_count
            item['show_in_tool_panel'] = False
            for x in user.stored_workflow_menu_entries:
                if x.stored_workflow_id == wf_sa.id:
                    item['show_in_tool_panel'] = True
                    break
            rval.append(item)
        if missing_tools:
            workflows_missing_tools = []
            workflows = []
            workflows_by_toolshed = dict()
            for key, value in enumerate(rval):
                tool_ids = []
                workflow_details = self.workflow_contents_manager.workflow_to_dict(
                    trans,
                    self.__get_stored_workflow(trans, value['id']),
                    style='instance')
                if 'steps' in workflow_details:
                    for step in workflow_details['steps']:
                        tool_id = workflow_details['steps'][step].get(
                            'tool_id')
                        if tool_id and tool_id not in tool_ids and self.app.toolbox.is_missing_shed_tool(
                                tool_id):
                            tool_ids.append(tool_id)
                if len(tool_ids) > 0:
                    value['missing_tools'] = tool_ids
                    workflows_missing_tools.append(value)
            for workflow in workflows_missing_tools:
                for tool_id in workflow['missing_tools']:
                    toolshed, _, owner, name, tool, version = tool_id.split(
                        '/')
                    shed_url = self.__get_full_shed_url(toolshed)
                    repo_identifier = '/'.join([toolshed, owner, name])
                    if repo_identifier not in workflows_by_toolshed:
                        workflows_by_toolshed[repo_identifier] = dict(
                            shed=shed_url.rstrip('/'),
                            repository=name,
                            owner=owner,
                            tools=[tool_id],
                            workflows=[workflow['name']])
                    else:
                        if tool_id not in workflows_by_toolshed[
                                repo_identifier]['tools']:
                            workflows_by_toolshed[repo_identifier][
                                'tools'].append(tool_id)
                        if workflow['name'] not in workflows_by_toolshed[
                                repo_identifier]['workflows']:
                            workflows_by_toolshed[repo_identifier][
                                'workflows'].append(workflow['name'])
            for repo_tag in workflows_by_toolshed:
                workflows.append(workflows_by_toolshed[repo_tag])
            return workflows
        return rval
    def install(self, trans, **kwd):
        """
        POST /api/tool_shed_repositories/install
        Initiate the installation of a repository.

        :param install_resolver_dependencies: True to install resolvable dependencies.
        :param install_tool_dependencies: True to install tool dependencies.
        :param install_repository_dependencies: True to install repository dependencies.
        :param tool_panel_section_id: The unique identifier for an existing tool panel section
        :param new_tool_panel_section_label: Create a new tool panel section with this label
        :param shed_tool_conf: The shed tool config file to use for this installation
        :param tool_shed_url: The URL for the toolshed whence this repository is being installed
        :param changeset: The changeset to update to after cloning the repository
        """
        irm = InstallRepositoryManager(self.app)
        tool_shed_url = kwd.get('tool_shed_url', None)
        repositories = json.loads(kwd.get('repositories', '[]'))
        repo_info_dict = self.__get_repo_info_dict(trans, repositories,
                                                   tool_shed_url)
        includes_tools = False
        includes_tools_for_display_in_tool_panel = False
        has_repository_dependencies = False
        includes_tool_dependencies = False
        install_resolver_dependencies = util.asbool(
            kwd.get('install_resolver_dependencies', False))
        for encoded_repo_info_dict in repo_info_dict.get(
                'repo_info_dicts', []):
            decoded_repo_info_dict = encoding_util.tool_shed_decode(
                encoded_repo_info_dict)
            if not includes_tools:
                includes_tools = util.string_as_bool(
                    decoded_repo_info_dict.get('includes_tools', False))
            if not includes_tools_for_display_in_tool_panel:
                includes_tools_for_display_in_tool_panel = \
                    util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) )
            if not has_repository_dependencies:
                has_repository_dependencies = util.string_as_bool(
                    repo_info_dict.get('has_repository_dependencies', False))
            if not includes_tool_dependencies:
                includes_tool_dependencies = util.string_as_bool(
                    repo_info_dict.get('includes_tool_dependencies', False))
        encoded_repo_info_dicts = util.listify(
            repo_info_dict.get('repo_info_dicts', []))
        repo_info_dicts = [
            encoding_util.tool_shed_decode(encoded_repo_info_dict)
            for encoded_repo_info_dict in encoded_repo_info_dicts
        ]
        tool_panel_section_id = kwd.get('tool_panel_section_id', None)
        new_tool_panel_section_label = kwd.get('new_tool_panel_section', None)
        tool_panel_section_mapping = json.loads(
            kwd.get('tool_panel_section', '{}'))
        install_tool_dependencies = util.asbool(
            kwd.get('install_tool_dependencies', False))
        install_repository_dependencies = util.asbool(
            kwd.get('install_repository_dependencies', False))
        shed_tool_conf = kwd.get('shed_tool_conf', None)
        tool_path = suc.get_tool_path_by_shed_tool_conf_filename(
            self.app, shed_tool_conf)
        installation_dict = dict(
            install_repository_dependencies=install_repository_dependencies,
            new_tool_panel_section_label=new_tool_panel_section_label,
            no_changes_checked=False,
            repo_info_dicts=repo_info_dicts,
            tool_panel_section_id=tool_panel_section_id,
            tool_path=tool_path,
            tool_shed_url=tool_shed_url)
        new_repositories, tool_panel_keys, repo_info_dicts, filtered_repos = irm.handle_tool_shed_repositories(
            installation_dict)
        if new_repositories:
            installation_dict = dict(
                created_or_updated_tool_shed_repositories=new_repositories,
                filtered_repo_info_dicts=filtered_repos,
                has_repository_dependencies=has_repository_dependencies,
                includes_tool_dependencies=includes_tool_dependencies,
                includes_tools=includes_tools,
                includes_tools_for_display_in_tool_panel=
                includes_tools_for_display_in_tool_panel,
                install_repository_dependencies=install_repository_dependencies,
                install_tool_dependencies=install_tool_dependencies,
                message='',
                new_tool_panel_section_label=new_tool_panel_section_label,
                tool_panel_section_mapping=tool_panel_section_mapping,
                install_resolver_dependencies=install_resolver_dependencies,
                shed_tool_conf=shed_tool_conf,
                status='ok',
                tool_panel_section_id=tool_panel_section_id,
                tool_panel_section_keys=tool_panel_keys,
                tool_path=tool_path,
                tool_shed_url=tool_shed_url)
            encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = \
                irm.initiate_repository_installation( installation_dict )
            return json.dumps(
                dict(
                    operation='install',
                    api=True,
                    install_resolver_dependencies=install_resolver_dependencies,
                    install_tool_dependencies=install_tool_dependencies,
                    encoded_kwd=encoded_kwd,
                    reinstalling=False,
                    tool_shed_repository_ids=json.dumps(
                        [repo[0] for repo in repositories]),
                    repositories=[
                        trans.security.encode_id(repo.id)
                        for repo in new_repositories
                    ]))
Esempio n. 16
0
    def display_data(self, trans, data, preview=False, filename=None, to_ext=None, **kwd):
        """
        Displays data in central pane if preview is `True`, else handles download.

        Datatypes should be very careful if overridding this method and this interface
        between datatypes and Galaxy will likely change.

        TOOD: Document alternatives to overridding this method (data
        providers?).
        """
        headers = kwd.get("headers", {})
        # Relocate all composite datatype display to a common location.
        composite_extensions = trans.app.datatypes_registry.get_composite_extensions()
        composite_extensions.append('html')  # for archiving composite datatypes
        # Prevent IE8 from sniffing content type since we're explicit about it.  This prevents intentionally text/plain
        # content from being rendered in the browser
        headers['X-Content-Type-Options'] = 'nosniff'
        if isinstance(data, str):
            return smart_str(data), headers
        if filename and filename != "index":
            # For files in extra_files_path
            extra_dir = data.dataset.extra_files_path_name
            file_path = trans.app.object_store.get_filename(data.dataset, extra_dir=extra_dir, alt_name=filename)
            if os.path.exists(file_path):
                if os.path.isdir(file_path):
                    with tempfile.NamedTemporaryFile(mode='w', delete=False, dir=trans.app.config.new_file_path, prefix='gx_html_autocreate_') as tmp_fh:
                        tmp_file_name = tmp_fh.name
                        dir_items = sorted(os.listdir(file_path))
                        base_path, item_name = os.path.split(file_path)
                        tmp_fh.write('<html><head><h3>Directory %s contents: %d items</h3></head>\n' % (escape(item_name), len(dir_items)))
                        tmp_fh.write('<body><p/><table cellpadding="2">\n')
                        for index, fname in enumerate(dir_items):
                            if index % 2 == 0:
                                bgcolor = '#D8D8D8'
                            else:
                                bgcolor = '#FFFFFF'
                            # Can't have an href link here because there is no route
                            # defined for files contained within multiple subdirectory
                            # levels of the primary dataset.  Something like this is
                            # close, but not quite correct:
                            # href = url_for(controller='dataset', action='display',
                            # dataset_id=trans.security.encode_id(data.dataset.id),
                            # preview=preview, filename=fname, to_ext=to_ext)
                            tmp_fh.write(f'<tr bgcolor="{bgcolor}"><td>{escape(fname)}</td></tr>\n')
                        tmp_fh.write('</table></body></html>\n')
                    return self._yield_user_file_content(trans, data, tmp_file_name, headers), headers
                mime = mimetypes.guess_type(file_path)[0]
                if not mime:
                    try:
                        mime = trans.app.datatypes_registry.get_mimetype_by_extension(".".split(file_path)[-1])
                    except Exception:
                        mime = "text/plain"
                self._clean_and_set_mime_type(trans, mime, headers)
                return self._yield_user_file_content(trans, data, file_path, headers), headers
            else:
                raise ObjectNotFound(f"Could not find '{filename}' on the extra files path {file_path}.")
        self._clean_and_set_mime_type(trans, data.get_mime(), headers)

        trans.log_event(f"Display dataset id: {str(data.id)}")
        from galaxy.datatypes import (  # DBTODO REMOVE THIS AT REFACTOR
            binary,
            images,
            text,
        )

        if to_ext or isinstance(
            data.datatype, binary.Binary
        ):  # Saving the file, or binary file
            if data.extension in composite_extensions:
                return self._archive_composite_dataset(trans, data, headers, do_action=kwd.get('do_action', 'zip'))
            else:
                headers['Content-Length'] = str(os.stat(data.file_name).st_size)
                filename = self._download_filename(data, to_ext, hdca=kwd.get("hdca"), element_identifier=kwd.get("element_identifier"), filename_pattern=kwd.get("filename_pattern"))
                headers['content-type'] = "application/octet-stream"  # force octet-stream so Safari doesn't append mime extensions to filename
                headers["Content-Disposition"] = f'attachment; filename="{filename}"'
                return open(data.file_name, 'rb'), headers
        if not os.path.exists(data.file_name):
            raise ObjectNotFound(f"File Not Found ({data.file_name}).")
        max_peek_size = DEFAULT_MAX_PEEK_SIZE  # 1 MB
        if isinstance(data.datatype, text.Html):
            max_peek_size = 10000000  # 10 MB for html
        preview = util.string_as_bool(preview)
        if (
            not preview
            or isinstance(data.datatype, images.Image)
            or os.stat(data.file_name).st_size < max_peek_size
        ):
            return self._yield_user_file_content(trans, data, data.file_name, headers), headers
        else:
            headers["content-type"] = "text/html"
            return trans.fill_template_mako("/dataset/large_file.mako",
                                            truncated_data=open(data.file_name, 'rb').read(max_peek_size),
                                            data=data), headers
Esempio n. 17
0
def __parse_test_attributes(output_elem,
                            attrib,
                            parse_elements=False,
                            parse_discovered_datasets=False,
                            profile=None):
    assert_list = __parse_assert_list(output_elem)

    # Allow either file or value to specify a target file to compare result with
    # file was traditionally used by outputs and value by extra files.
    file = attrib.pop('file', attrib.pop('value', None))

    # File no longer required if an list of assertions was present.
    attributes = {}
    # Method of comparison
    attributes['compare'] = attrib.pop('compare', 'diff').lower()
    # Number of lines to allow to vary in logs (for dates, etc)
    attributes['lines_diff'] = int(attrib.pop('lines_diff', '0'))
    # Allow a file size to vary if sim_size compare
    attributes['delta'] = int(attrib.pop('delta', DEFAULT_DELTA))
    attributes['delta_frac'] = float(
        attrib['delta_frac']) if 'delta_frac' in attrib else DEFAULT_DELTA_FRAC
    attributes['sort'] = string_as_bool(attrib.pop('sort', False))
    attributes['decompress'] = string_as_bool(attrib.pop('decompress', False))
    extra_files = []
    if 'ftype' in attrib:
        attributes['ftype'] = attrib['ftype']
    for extra in output_elem.findall('extra_files'):
        extra_files.append(__parse_extra_files_elem(extra))
    metadata = {}
    for metadata_elem in output_elem.findall('metadata'):
        metadata[metadata_elem.get('name')] = metadata_elem.get('value')
    md5sum = attrib.get("md5", None)
    checksum = attrib.get("checksum", None)
    element_tests = {}
    if parse_elements:
        element_tests = __parse_element_tests(output_elem, profile=profile)

    primary_datasets = {}
    if parse_discovered_datasets:
        for primary_elem in (output_elem.findall("discovered_dataset") or []):
            primary_attrib = dict(primary_elem.attrib)
            designation = primary_attrib.pop('designation', None)
            if designation is None:
                raise Exception(
                    "Test primary dataset does not have a 'designation'")
            primary_datasets[designation] = __parse_test_attributes(
                primary_elem, primary_attrib)

    has_checksum = md5sum or checksum
    has_nested_tests = extra_files or element_tests or primary_datasets
    if not (assert_list or file or metadata or has_checksum
            or has_nested_tests):
        raise Exception(
            "Test output defines nothing to check (e.g. must have a 'file' check against, assertions to check, metadata or checksum tests, etc...)"
        )
    attributes['assert_list'] = assert_list
    attributes['extra_files'] = extra_files
    attributes['metadata'] = metadata
    attributes['md5'] = md5sum
    attributes['checksum'] = checksum
    attributes['elements'] = element_tests
    attributes['primary_datasets'] = primary_datasets
    return file, attributes
Esempio n. 18
0
 def _get_option_value(self, key, default):
     root = self.root
     for option_elem in root.findall("options"):
         if key in option_elem.attrib:
             return string_as_bool(option_elem.get(key))
     return default
Esempio n. 19
0
    def parse_outputs(self, tool):
        out_elem = self.root.find("outputs")
        outputs = odict()
        output_collections = odict()
        if out_elem is None:
            return outputs, output_collections

        data_dict = odict()

        def _parse(data_elem, **kwds):
            output_def = self._parse_output(data_elem, tool, **kwds)
            data_dict[output_def.name] = output_def
            return output_def

        map(_parse, out_elem.findall("data"))

        for collection_elem in out_elem.findall("collection"):
            name = collection_elem.get( "name" )
            default_format = collection_elem.get( "format", "data" )
            collection_type = collection_elem.get( "type", None )
            structured_like = collection_elem.get( "structured_like", None )
            inherit_format = False
            inherit_metadata = False
            if structured_like:
                inherit_format = string_as_bool( collection_elem.get( "inherit_format", None ) )
                inherit_metadata = string_as_bool( collection_elem.get( "inherit_metadata", None ) )
            default_format_source = collection_elem.get( "format_source", None )
            default_metadata_source = collection_elem.get( "metadata_source", "" )

            dataset_collectors = None
            if collection_elem.find( "discover_datasets" ) is not None:
                dataset_collectors = output_collect.dataset_collectors_from_elem( collection_elem )
            structure = galaxy.tools.ToolOutputCollectionStructure(
                collection_type=collection_type,
                structured_like=structured_like,
                dataset_collectors=dataset_collectors,
            )
            output_collection = galaxy.tools.ToolOutputCollection(
                name,
                structure,
                default_format=default_format,
                inherit_format=inherit_format,
                inherit_metadata=inherit_metadata,
                default_format_source=default_format_source,
                default_metadata_source=default_metadata_source,
            )
            outputs[output_collection.name] = output_collection

            for data_elem in collection_elem.findall("data"):
                _parse(
                    data_elem,
                    default_format=default_format,
                    default_format_source=default_format_source,
                    default_metadata_source=default_metadata_source,
                )

            for data_elem in collection_elem.findall("data"):
                output_name = data_elem.get("name")
                data = data_dict[output_name]
                assert data
                del data_dict[output_name]
                output_collection.outputs[output_name] = data
            output_collections[ name ] = output_collection

        for output_def in data_dict.values():
            outputs[output_def.name] = output_def
        return outputs, output_collections
Esempio n. 20
0
    def __init__(self, **kwargs):
        self.config_dict = kwargs
        self.root = kwargs.get('root_dir', '.')

        # Resolve paths of other config files
        self.__parse_config_file_options(kwargs)

        # Collect the umask and primary gid from the environment
        self.umask = os.umask(077)  # get the current umask
        os.umask(self.umask)  # can't get w/o set, so set it back
        self.gid = os.getgid(
        )  # if running under newgrp(1) we'll need to fix the group of data created on the cluster
        self.version_major = VERSION_MAJOR
        self.version = VERSION
        # Database related configuration
        self.database = resolve_path(
            kwargs.get("database_file", "database/community.sqlite"),
            self.root)
        self.database_connection = kwargs.get("database_connection", False)
        self.database_engine_options = get_database_engine_options(kwargs)
        self.database_create_tables = string_as_bool(
            kwargs.get("database_create_tables", "True"))
        # Repository and Tool search API
        self.toolshed_search_on = string_as_bool(
            kwargs.get("toolshed_search_on", True))
        self.whoosh_index_dir = kwargs.get("whoosh_index_dir",
                                           'database/toolshed_whoosh_indexes')
        self.repo_name_boost = kwargs.get("repo_name_boost", 0.9)
        self.repo_description_boost = kwargs.get("repo_description_boost", 0.6)
        self.repo_long_description_boost = kwargs.get(
            "repo_long_description_boost", 0.5)
        self.repo_homepage_url_boost = kwargs.get("repo_homepage_url_boost",
                                                  0.3)
        self.repo_remote_repository_url_boost = kwargs.get(
            "repo_remote_repository_url_boost", 0.2)
        self.repo_owner_username_boost = kwargs.get(
            "repo_owner_username_boost", 0.3)
        self.tool_name_boost = kwargs.get("tool_name_boost", 1.2)
        self.tool_description_boost = kwargs.get("tool_description_boost", 0.6)
        self.tool_help_boost = kwargs.get("tool_help_boost", 0.4)
        self.tool_repo_owner_username = kwargs.get("tool_repo_owner_username",
                                                   0.3)
        # Analytics
        self.ga_code = kwargs.get("ga_code", None)
        self.session_duration = int(kwargs.get('session_duration', 0))
        # Where dataset files are stored
        self.file_path = resolve_path(
            kwargs.get("file_path", "database/community_files"), self.root)
        self.new_file_path = resolve_path(
            kwargs.get("new_file_path", "database/tmp"), self.root)
        self.cookie_path = kwargs.get("cookie_path", "/")
        self.enable_quotas = string_as_bool(kwargs.get('enable_quotas', False))
        self.id_secret = kwargs.get("id_secret",
                                    "USING THE DEFAULT IS NOT SECURE!")
        # Tool stuff
        self.tool_path = resolve_path(kwargs.get("tool_path", "tools"),
                                      self.root)
        self.tool_secret = kwargs.get("tool_secret", "")
        self.tool_data_path = resolve_path(
            kwargs.get("tool_data_path", "shed-tool-data"), os.getcwd())
        self.tool_data_table_config_path = None
        self.integrated_tool_panel_config = resolve_path(
            kwargs.get('integrated_tool_panel_config',
                       'integrated_tool_panel.xml'), self.root)
        self.builds_file_path = resolve_path(
            kwargs.get(
                "builds_file_path",
                os.path.join(self.tool_data_path, 'shared', 'ucsc',
                             'builds.txt')), self.root)
        self.len_file_path = resolve_path(
            kwargs.get(
                "len_file_path",
                os.path.join(self.tool_data_path, 'shared', 'ucsc', 'chrom')),
            self.root)
        self.ftp_upload_dir = kwargs.get('ftp_upload_dir', None)
        # Install and test framework for testing tools contained in repositories.
        self.display_legacy_test_results = string_as_bool(
            kwargs.get('display_legacy_test_results', True))
        self.num_tool_test_results_saved = kwargs.get(
            'num_tool_test_results_saved', 5)
        self.update_integrated_tool_panel = False
        # Galaxy flavor Docker Image
        self.enable_galaxy_flavor_docker_image = string_as_bool(
            kwargs.get("enable_galaxy_flavor_docker_image", "False"))
        self.use_remote_user = string_as_bool(
            kwargs.get("use_remote_user", "False"))
        self.user_activation_on = kwargs.get('user_activation_on', None)
        self.activation_grace_period = kwargs.get('activation_grace_period',
                                                  None)
        self.inactivity_box_content = kwargs.get('inactivity_box_content',
                                                 None)
        self.registration_warning_message = kwargs.get(
            'registration_warning_message', None)
        self.terms_url = kwargs.get('terms_url', None)
        self.blacklist_location = kwargs.get('blacklist_file', None)
        self.blacklist_content = None
        self.remote_user_maildomain = kwargs.get("remote_user_maildomain",
                                                 None)
        self.remote_user_header = kwargs.get("remote_user_header",
                                             'HTTP_REMOTE_USER')
        self.remote_user_logout_href = kwargs.get("remote_user_logout_href",
                                                  None)
        self.remote_user_secret = kwargs.get("remote_user_secret", None)
        self.require_login = string_as_bool(
            kwargs.get("require_login", "False"))
        self.allow_user_creation = string_as_bool(
            kwargs.get("allow_user_creation", "True"))
        self.allow_user_deletion = string_as_bool(
            kwargs.get("allow_user_deletion", "False"))
        self.enable_openid = string_as_bool(kwargs.get('enable_openid', False))
        self.template_path = resolve_path(
            kwargs.get("template_path", "templates"), self.root)
        self.template_cache = resolve_path(
            kwargs.get("template_cache_path",
                       "database/compiled_templates/community"), self.root)
        self.admin_users = kwargs.get("admin_users", "")
        self.admin_users_list = [
            u.strip() for u in self.admin_users.split(',') if u
        ]
        self.mailing_join_addr = kwargs.get('mailing_join_addr',
                                            "*****@*****.**")
        self.error_email_to = kwargs.get('error_email_to', None)
        self.smtp_server = kwargs.get('smtp_server', None)
        self.smtp_username = kwargs.get('smtp_username', None)
        self.smtp_password = kwargs.get('smtp_password', None)
        self.smtp_ssl = kwargs.get('smtp_ssl', None)
        self.start_job_runners = kwargs.get('start_job_runners', None)
        self.email_from = kwargs.get('email_from', None)
        self.nginx_upload_path = kwargs.get('nginx_upload_path', False)
        self.log_actions = string_as_bool(kwargs.get('log_actions', 'False'))
        self.brand = kwargs.get('brand', None)
        self.pretty_datetime_format = expand_pretty_datetime_format(
            kwargs.get('pretty_datetime_format', '$locale (UTC)'))
        # Configuration for the message box directly below the masthead.
        self.message_box_visible = kwargs.get('message_box_visible', False)
        self.message_box_content = kwargs.get('message_box_content', None)
        self.message_box_class = kwargs.get('message_box_class', 'info')
        self.support_url = kwargs.get(
            'support_url', 'https://wiki.galaxyproject.org/Support')
        self.wiki_url = kwargs.get('wiki_url',
                                   'https://wiki.galaxyproject.org/')
        self.blog_url = kwargs.get('blog_url', None)
        self.biostar_url = kwargs.get('biostar_url', None)
        self.screencasts_url = kwargs.get('screencasts_url', None)
        self.log_events = False
        self.cloud_controller_instance = False
        self.server_name = ''
        self.job_manager = ''
        self.default_job_handlers = []
        self.default_cluster_job_runner = 'local:///'
        self.job_handlers = []
        self.tool_handlers = []
        self.tool_runners = []
        # Error logging with sentry
        self.sentry_dsn = kwargs.get('sentry_dsn', None)
        # Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory.
        self.hgweb_config_dir = resolve_path(
            kwargs.get('hgweb_config_dir', ''), self.root)
        self.disable_push = string_as_bool(kwargs.get("disable_push", "True"))
        # Proxy features
        self.apache_xsendfile = kwargs.get('apache_xsendfile', False)
        self.nginx_x_accel_redirect_base = kwargs.get(
            'nginx_x_accel_redirect_base', False)
        self.drmaa_external_runjob_script = kwargs.get(
            'drmaa_external_runjob_script', None)
        # Parse global_conf and save the parser
        global_conf = kwargs.get('global_conf', None)
        global_conf_parser = ConfigParser.ConfigParser()
        self.global_conf_parser = global_conf_parser
        if global_conf and "__file__" in global_conf:
            global_conf_parser.read(global_conf['__file__'])
        self.running_functional_tests = string_as_bool(
            kwargs.get('running_functional_tests', False))
        self.citation_cache_type = kwargs.get("citation_cache_type", "file")
        self.citation_cache_data_dir = resolve_path(
            kwargs.get("citation_cache_data_dir",
                       "database/tool_shed_citations/data"), self.root)
        self.citation_cache_lock_dir = resolve_path(
            kwargs.get("citation_cache_lock_dir",
                       "database/tool_shed_citations/locks"), self.root)
Esempio n. 21
0
def parse_config_xml(config_xml):
    try:
        a_xml = config_xml.findall('auth')[0]
        access_key = a_xml.get('access_key')
        secret_key = a_xml.get('secret_key')

        b_xml = config_xml.findall('bucket')[0]
        bucket_name = b_xml.get('name')
        use_rr = string_as_bool(b_xml.get('use_reduced_redundancy', "False"))
        max_chunk_size = int(b_xml.get('max_chunk_size', 250))

        cn_xml = config_xml.findall('connection')
        if not cn_xml:
            cn_xml = {}
        else:
            cn_xml = cn_xml[0]

        host = cn_xml.get('host', None)
        port = int(cn_xml.get('port', 6000))
        multipart = string_as_bool(cn_xml.get('multipart', 'True'))
        is_secure = string_as_bool(cn_xml.get('is_secure', 'True'))
        conn_path = cn_xml.get('conn_path', '/')

        c_xml = config_xml.findall('cache')[0]
        cache_size = float(c_xml.get('size', -1))

        staging_path = c_xml.get('path', None)

        tag, attrs = 'extra_dir', ('type', 'path')
        extra_dirs = config_xml.findall(tag)
        if not extra_dirs:
            msg = f'No {tag} element in XML tree'
            log.error(msg)
            raise Exception(msg)
        extra_dirs = [{k: e.get(k) for k in attrs} for e in extra_dirs]

        return {
            'auth': {
                'access_key': access_key,
                'secret_key': secret_key,
            },
            'bucket': {
                'name': bucket_name,
                'use_reduced_redundancy': use_rr,
                'max_chunk_size': max_chunk_size,
            },
            'connection': {
                'host': host,
                'port': port,
                'multipart': multipart,
                'is_secure': is_secure,
                'conn_path': conn_path,
            },
            'cache': {
                'size': cache_size,
                'path': staging_path,
            },
            'extra_dirs': extra_dirs,
        }
    except Exception:
        # Toss it back up after logging, we can't continue loading at this point.
        log.exception(
            "Malformed ObjectStore Configuration XML -- unable to continue")
        raise
Esempio n. 22
0
    def load( self, trans, **kwd ):
        """
        load( self, trans, **kwd ):
        * POST /api/libraries/datasets
        Load dataset from the given source into the library.
        Source can be:
            user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
                example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
                the folder with the user login has to be created beforehand
            (admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
                example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
            (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini

        :param  encoded_folder_id:      the encoded id of the folder to import dataset(s) to
        :type   encoded_folder_id:      an encoded id string
        :param  source:                 source the datasets should be loaded form
        :type   source:                 str
        :param  link_data:              flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
                                        while linking is set to True all symlinks will be resolved _once_
        :type   link_data:              bool
        :param  preserve_dirs:          flag whether to preserve the directory structure when importing dir
                                        if False only datasets will be imported
        :type   preserve_dirs:          bool
        :param  file_type:              file type of the loaded datasets, defaults to 'auto' (autodetect)
        :type   file_type:              str
        :param  dbkey:                  dbkey of the loaded genome, defaults to '?' (unknown)
        :type   dbkey:                  str

        :returns:   dict containing information about the created upload job
        :rtype:     dictionary
        """

        kwd[ 'space_to_tab' ] = 'False'
        kwd[ 'to_posix_lines' ] = 'True'
        kwd[ 'dbkey' ] = kwd.get( 'dbkey', '?' )
        kwd[ 'file_type' ] = kwd.get( 'file_type', 'auto' )
        kwd[' link_data_only' ] = 'link_to_files' if util.string_as_bool( kwd.get( 'link_data', False ) ) else 'copy_files'
        encoded_folder_id = kwd.get( 'encoded_folder_id', None )
        if encoded_folder_id is not None:
            folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id )
        else:
            raise exceptions.RequestParameterMissingException( 'The required atribute encoded_folder_id is missing.' )
        path = kwd.get( 'path', None)
        if path is None:
            raise exceptions.RequestParameterMissingException( 'The required atribute path is missing.' )
        folder = self.folder_manager.get( trans, folder_id )

        source = kwd.get( 'source', None )
        if source not in [ 'userdir_file', 'userdir_folder', 'importdir_file', 'importdir_folder', 'admin_path' ]:
            raise exceptions.RequestParameterMissingException( 'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". ')
        if source in [ 'importdir_file', 'importdir_folder' ]:
            if not trans.user_is_admin:
                raise exceptions.AdminRequiredException( 'Only admins can import from importdir.' )
            if not trans.app.config.library_import_dir:
                raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from importdir.' )
            import_base_dir = trans.app.config.library_import_dir
            path = os.path.join( import_base_dir, path )
        if source in [ 'userdir_file', 'userdir_folder' ]:
            user_login = trans.user.email
            user_base_dir = trans.app.config.user_library_import_dir
            if user_base_dir is None:
                raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from user directories.' )
            full_dir = os.path.join( user_base_dir, user_login )
            if not path.lower().startswith( full_dir.lower() ):
                path = os.path.join( full_dir, path )
            if not os.path.exists( path ):
                raise exceptions.RequestParameterInvalidException( 'Given path does not exist on the host.' )
            if not self.folder_manager.can_add_item( trans, folder ):
                raise exceptions.InsufficientPermissionsException( 'You do not have proper permission to add items to the given folder.' )
        if source == 'admin_path':
            if not trans.app.config.allow_library_path_paste:
                raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow admins to import into library from path.' )
            if not trans.user_is_admin:
                raise exceptions.AdminRequiredException( 'Only admins can import from path.' )

        # Set up the traditional tool state/params
        tool_id = 'upload1'
        tool = trans.app.toolbox.get_tool( tool_id )
        state = tool.new_state( trans )
        tool.update_state( trans, tool.inputs_by_page[ 0 ], state.inputs, kwd )
        tool_params = state.inputs
        dataset_upload_inputs = []
        for input in tool.inputs.itervalues():
            if input.type == "upload_dataset":
                dataset_upload_inputs.append( input )
        library_bunch = upload_common.handle_library_params( trans, {}, trans.security.encode_id( folder.id ) )
        abspath_datasets = []
        kwd[ 'filesystem_paths' ] = path
        if source in [ 'importdir_folder' ]:
            kwd[ 'filesystem_paths' ] = os.path.join( import_base_dir, path )
        params = util.Params( kwd )
        # user wants to import one file only
        if source in [ "userdir_file", "importdir_file" ]:
            file = os.path.abspath( path )
            abspath_datasets.append( trans.webapp.controllers[ 'library_common' ].make_library_uploaded_dataset(
                trans, 'api', params, os.path.basename( file ), file, 'server_dir', library_bunch ) )
        # user wants to import whole folder
        if source == "userdir_folder":
            uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
                trans, 'api', params, library_bunch, 200, '' )
            uploaded_datasets = uploaded_datasets_bunch[ 0 ]
            if uploaded_datasets is None:
                raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
            for ud in uploaded_datasets:
                ud.path = os.path.abspath( ud.path )
                abspath_datasets.append( ud )
        #  user wants to import from path
        if source in [ "admin_path", "importdir_folder" ]:
            # validate the path is within root
            uploaded_datasets_bunch = trans.webapp.controllers[ 'library_common' ].get_path_paste_uploaded_datasets(
                trans, 'api', params, library_bunch, 200, '' )
            uploaded_datasets = uploaded_datasets_bunch[0]
            if uploaded_datasets is None:
                raise exceptions.ObjectNotFound( 'Given folder does not contain any datasets.' )
            for ud in uploaded_datasets:
                ud.path = os.path.abspath( ud.path )
                abspath_datasets.append( ud )
        json_file_path = upload_common.create_paramfile( trans, abspath_datasets )
        data_list = [ ud.data for ud in abspath_datasets ]
        job, output = upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=folder )
        # HACK: Prevent outputs_to_working_directory from overwriting inputs when "linking"
        job.add_parameter( 'link_data_only', dumps( kwd.get( 'link_data_only', 'copy_files' ) ) )
        job.add_parameter( 'uuid', dumps( kwd.get( 'uuid', None ) ) )
        trans.sa_session.add( job )
        trans.sa_session.flush()
        job_dict = job.to_dict()
        job_dict[ 'id' ] = trans.security.encode_id( job_dict[ 'id' ] )
        return job_dict
Esempio n. 23
0
    def display_data(self,
                     trans,
                     data,
                     preview=False,
                     filename=None,
                     to_ext=None,
                     size=None,
                     offset=None,
                     **kwd):
        """ Old display method, for transition - though still used by API and
        test framework. Datatypes should be very careful if overridding this
        method and this interface between datatypes and Galaxy will likely
        change.

        TOOD: Document alternatives to overridding this method (data
        providers?).
        """
        # Relocate all composite datatype display to a common location.
        composite_extensions = trans.app.datatypes_registry.get_composite_extensions(
        )
        composite_extensions.append(
            'html')  # for archiving composite datatypes
        # Prevent IE8 from sniffing content type since we're explicit about it.  This prevents intentionally text/plain
        # content from being rendered in the browser
        trans.response.headers['X-Content-Type-Options'] = 'nosniff'
        if isinstance(data, string_types):
            return data
        if filename and filename != "index":
            # For files in extra_files_path
            file_path = trans.app.object_store.get_filename(
                data.dataset,
                extra_dir='dataset_%s_files' % data.dataset.id,
                alt_name=filename)
            if os.path.exists(file_path):
                if os.path.isdir(file_path):
                    return trans.show_error_message(
                        "Directory listing is not allowed."
                    )  # TODO: Reconsider allowing listing of directories?
                mime = mimetypes.guess_type(file_path)[0]
                if not mime:
                    try:
                        mime = trans.app.datatypes_registry.get_mimetype_by_extension(
                            ".".split(file_path)[-1])
                    except:
                        mime = "text/plain"
                self._clean_and_set_mime_type(trans, mime)
                return open(file_path)
            else:
                return paste.httpexceptions.HTTPNotFound(
                    "Could not find '%s' on the extra files path %s." %
                    (filename, file_path))
        self._clean_and_set_mime_type(trans, data.get_mime())

        trans.log_event("Display dataset id: %s" % str(data.id))
        from galaxy import datatypes  # DBTODO REMOVE THIS AT REFACTOR
        if to_ext or isinstance(
                data.datatype,
                datatypes.binary.Binary):  # Saving the file, or binary file
            if data.extension in composite_extensions:
                return self._archive_composite_dataset(trans, data, **kwd)
            else:
                trans.response.headers['Content-Length'] = int(
                    os.stat(data.file_name).st_size)
                if not to_ext:
                    to_ext = data.extension
                valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
                fname = ''.join(c in valid_chars and c or '_'
                                for c in data.name)[0:150]
                trans.response.set_content_type(
                    "application/octet-stream"
                )  # force octet-stream so Safari doesn't append mime extensions to filename
                trans.response.headers[
                    "Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (
                        data.hid, fname, to_ext)
                return open(data.file_name)
        if not os.path.exists(data.file_name):
            raise paste.httpexceptions.HTTPNotFound("File Not Found (%s)." %
                                                    data.file_name)
        max_peek_size = 1000000  # 1 MB
        if isinstance(data.datatype, datatypes.text.Html):
            max_peek_size = 10000000  # 10 MB for html
        preview = util.string_as_bool(preview)
        if not preview or isinstance(
                data.datatype, datatypes.images.Image) or os.stat(
                    data.file_name).st_size < max_peek_size:
            if trans.app.config.sanitize_all_html and trans.response.get_content_type(
            ) == "text/html":
                # Sanitize anytime we respond with plain text/html content.
                # Check to see if this dataset's parent job is whitelisted
                # We cannot currently trust imported datasets for rendering.
                if not data.creating_job.imported and data.creating_job.tool_id in trans.app.config.sanitize_whitelist:
                    return open(data.file_name).read()
                # This is returning to the browser, it needs to be encoded.
                # TODO Ideally this happens a layer higher, but this is a bad
                # issue affecting many tools
                return sanitize_html(open(
                    data.file_name).read()).encode('utf-8')
            return open(data.file_name)
        else:
            trans.response.set_content_type("text/html")
            return trans.stream_template_mako(
                "/dataset/large_file.mako",
                truncated_data=open(data.file_name).read(max_peek_size),
                data=data)
Esempio n. 24
0
 def __init__(self, **kwargs):
     self.config_dict = kwargs
     self.root = kwargs.get('root_dir', '.')
     # Collect the umask and primary gid from the environment
     self.umask = os.umask(077)  # get the current umask
     os.umask(self.umask)  # can't get w/o set, so set it back
     self.gid = os.getgid(
     )  # if running under newgrp(1) we'll need to fix the group of data created on the cluster
     # Database related configuration
     self.database = resolve_path(
         kwargs.get("database_file", "database/universe.d"), self.root)
     self.database_connection = kwargs.get("database_connection", False)
     self.database_engine_options = get_database_engine_options(kwargs)
     self.database_create_tables = string_as_bool(
         kwargs.get("database_create_tables", "True"))
     # Where dataset files are stored
     self.file_path = resolve_path(
         kwargs.get("file_path", "database/files"), self.root)
     self.new_file_path = resolve_path(
         kwargs.get("new_file_path", "database/tmp"), self.root)
     # dataset Track files
     self.track_store_path = kwargs.get("track_store_path",
                                        "${extra_files_path}/tracks")
     self.tool_path = resolve_path(kwargs.get("tool_path", "tools"),
                                   self.root)
     self.tool_data_path = resolve_path(
         kwargs.get("tool_data_path", "tool-data"), os.getcwd())
     self.test_conf = resolve_path(kwargs.get("test_conf", ""), self.root)
     self.tool_config = resolve_path(
         kwargs.get('tool_config_file', 'tool_conf.xml'), self.root)
     self.tool_secret = kwargs.get("tool_secret", "")
     self.id_secret = kwargs.get("id_secret",
                                 "USING THE DEFAULT IS NOT SECURE!")
     self.set_metadata_externally = string_as_bool(
         kwargs.get("set_metadata_externally", "False"))
     self.use_remote_user = string_as_bool(
         kwargs.get("use_remote_user", "False"))
     self.remote_user_maildomain = kwargs.get("remote_user_maildomain",
                                              None)
     self.remote_user_logout_href = kwargs.get("remote_user_logout_href",
                                               None)
     self.require_login = string_as_bool(
         kwargs.get("require_login", "False"))
     self.allow_user_creation = string_as_bool(
         kwargs.get("allow_user_creation", "True"))
     self.allow_user_deletion = string_as_bool(
         kwargs.get("allow_user_deletion", "False"))
     self.new_user_dataset_access_role_default_private = string_as_bool(
         kwargs.get("new_user_dataset_access_role_default_private",
                    "False"))
     self.template_path = resolve_path(
         kwargs.get("template_path", "templates"), self.root)
     self.template_cache = resolve_path(
         kwargs.get("template_cache_path", "database/compiled_templates"),
         self.root)
     self.local_job_queue_workers = int(
         kwargs.get("local_job_queue_workers", "5"))
     self.cluster_job_queue_workers = int(
         kwargs.get("cluster_job_queue_workers", "3"))
     self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO")
     self.job_queue_cleanup_interval = int(
         kwargs.get("job_queue_cleanup_interval", "5"))
     self.cluster_files_directory = os.path.abspath(
         kwargs.get("cluster_files_directory", "database/pbs"))
     self.job_working_directory = resolve_path(
         kwargs.get("job_working_directory",
                    "database/job_working_directory"), self.root)
     self.outputs_to_working_directory = string_as_bool(
         kwargs.get('outputs_to_working_directory', False))
     self.output_size_limit = int(kwargs.get('output_size_limit', 0))
     self.job_walltime = kwargs.get('job_walltime', None)
     self.admin_users = kwargs.get("admin_users", "")
     self.sendmail_path = kwargs.get('sendmail_path', "/usr/sbin/sendmail")
     self.mailing_join_addr = kwargs.get('mailing_join_addr',
                                         "*****@*****.**")
     self.error_email_to = kwargs.get('error_email_to', None)
     self.smtp_server = kwargs.get('smtp_server', None)
     self.start_job_runners = kwargs.get('start_job_runners', None)
     self.default_cluster_job_runner = kwargs.get(
         'default_cluster_job_runner', 'local:///')
     self.pbs_application_server = kwargs.get('pbs_application_server', "")
     self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "")
     self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "")
     self.pbs_stage_path = kwargs.get('pbs_stage_path', "")
     self.use_heartbeat = string_as_bool(
         kwargs.get('use_heartbeat', 'False'))
     self.use_memdump = string_as_bool(kwargs.get('use_memdump', 'False'))
     self.log_memory_usage = string_as_bool(
         kwargs.get('log_memory_usage', 'False'))
     self.log_actions = string_as_bool(kwargs.get('log_actions', 'False'))
     self.log_events = string_as_bool(kwargs.get('log_events', 'False'))
     self.ucsc_display_sites = kwargs.get(
         'ucsc_display_sites', "main,test,archaea,ucla").lower().split(",")
     self.gbrowse_display_sites = kwargs.get(
         'gbrowse_display_sites', "main,test,tair").lower().split(",")
     self.genetrack_display_sites = kwargs.get(
         'genetrack_display_sites', "main,test").lower().split(",")
     self.brand = kwargs.get('brand', None)
     self.wiki_url = kwargs.get('wiki_url', 'http://g2.trac.bx.psu.edu/')
     self.bugs_email = kwargs.get('bugs_email', None)
     self.blog_url = kwargs.get('blog_url', None)
     self.screencasts_url = kwargs.get('screencasts_url', None)
     self.library_import_dir = kwargs.get('library_import_dir', None)
     if self.library_import_dir is not None and not os.path.exists(
             self.library_import_dir):
         raise ConfigurationError(
             "library_import_dir specified in config (%s) does not exist" %
             self.library_import_dir)
     self.user_library_import_dir = kwargs.get('user_library_import_dir',
                                               None)
     if self.user_library_import_dir is not None and not os.path.exists(
             self.user_library_import_dir):
         raise ConfigurationError(
             "user_library_import_dir specified in config (%s) does not exist"
             % self.user_library_import_dir)
     self.allow_library_path_paste = kwargs.get('allow_library_path_paste',
                                                False)
     # Configuration options for taking advantage of nginx features
     self.nginx_x_accel_redirect_base = kwargs.get(
         'nginx_x_accel_redirect_base', False)
     self.nginx_upload_store = kwargs.get('nginx_upload_store', False)
     self.nginx_upload_path = kwargs.get('nginx_upload_path', False)
     if self.nginx_upload_store:
         self.nginx_upload_store = os.path.abspath(self.nginx_upload_store)
     # Parse global_conf and save the parser
     global_conf = kwargs.get('global_conf', None)
     global_conf_parser = ConfigParser.ConfigParser()
     self.global_conf_parser = global_conf_parser
     if global_conf and "__file__" in global_conf:
         global_conf_parser.read(global_conf['__file__'])
     #Store per-tool runner config
     try:
         self.tool_runners = global_conf_parser.items("galaxy:tool_runners")
     except ConfigParser.NoSectionError:
         self.tool_runners = []
     self.datatypes_config = kwargs.get('datatypes_config_file',
                                        'datatypes_conf.xml')
     # Cloud configuration options
     self.cloud_controller_instance = string_as_bool(
         kwargs.get('cloud_controller_instance', 'False'))
     if self.cloud_controller_instance == True:
         self.enable_cloud_execution = string_as_bool(
             kwargs.get('enable_cloud_execution', 'True'))
     else:
         self.enable_cloud_execution = string_as_bool(
             kwargs.get('enable_cloud_execution', 'False'))
Esempio n. 25
0
    def _create(self, trans, payload, **kwd):
        # HACK: for now, if action is rerun, rerun tool.
        action = payload.get('action', None)
        if action == 'rerun':
            return self._rerun_tool(trans, payload, **kwd)

        # -- Execute tool. --

        # Get tool.
        tool_version = payload.get('tool_version', None)
        tool = trans.app.toolbox.get_tool(payload['tool_id'], tool_version) if 'tool_id' in payload else None
        if not tool or not tool.allow_user_access(trans.user):
            raise exceptions.MessageException('Tool not found or not accessible.')
        if trans.app.config.user_activation_on:
            if not trans.user:
                log.warning("Anonymous user attempts to execute tool, but account activation is turned on.")
            elif not trans.user.active:
                log.warning("User \"%s\" attempts to execute tool, but account activation is turned on and user account is not active." % trans.user.email)

        # Set running history from payload parameters.
        # History not set correctly as part of this API call for
        # dataset upload.
        history_id = payload.get('history_id', None)
        if history_id:
            decoded_id = self.decode_id(history_id)
            target_history = self.history_manager.get_owned(decoded_id, trans.user, current_history=trans.history)
        else:
            target_history = None

        # Set up inputs.
        inputs = payload.get('inputs', {})
        # Find files coming in as multipart file data and add to inputs.
        for k, v in payload.items():
            if k.startswith('files_') or k.startswith('__files_'):
                inputs[k] = v

        # for inputs that are coming from the Library, copy them into the history
        input_patch = {}
        for k, v in inputs.items():
            if isinstance(v, dict) and v.get('src', '') == 'ldda' and 'id' in v:
                ldda = trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(self.decode_id(v['id']))
                if trans.user_is_admin or trans.app.security_agent.can_access_dataset(trans.get_current_user_roles(), ldda.dataset):
                    input_patch[k] = ldda.to_history_dataset_association(target_history, add_to_history=True)

        for k, v in input_patch.items():
            inputs[k] = v

        # TODO: encode data ids and decode ids.
        # TODO: handle dbkeys
        params = util.Params(inputs, sanitize=False)
        incoming = params.__dict__

        # use_cached_job can be passed in via the top-level payload or among the tool inputs.
        # I think it should be a top-level parameter, but because the selector is implemented
        # as a regular tool parameter we accept both.
        use_cached_job = payload.get('use_cached_job', False) or util.string_as_bool(inputs.get('use_cached_job', 'false'))
        vars = tool.handle_input(trans, incoming, history=target_history, use_cached_job=use_cached_job)

        # TODO: check for errors and ensure that output dataset(s) are available.
        output_datasets = vars.get('out_data', [])
        rval = {'outputs': [], 'output_collections': [], 'jobs': [], 'implicit_collections': []}

        job_errors = vars.get('job_errors', [])
        if job_errors:
            # If we are here - some jobs were successfully executed but some failed.
            rval['errors'] = job_errors

        outputs = rval['outputs']
        # TODO:?? poss. only return ids?
        for output_name, output in output_datasets:
            output_dict = output.to_dict()
            # add the output name back into the output data structure
            # so it's possible to figure out which newly created elements
            # correspond with which tool file outputs
            output_dict['output_name'] = output_name
            outputs.append(trans.security.encode_dict_ids(output_dict, skip_startswith="metadata_"))

        for job in vars.get('jobs', []):
            rval['jobs'].append(self.encode_all_ids(trans, job.to_dict(view='collection'), recursive=True))

        for output_name, collection_instance in vars.get('output_collections', []):
            history = target_history or trans.history
            output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history)
            output_dict['output_name'] = output_name
            rval['output_collections'].append(output_dict)

        for output_name, collection_instance in vars.get('implicit_collections', {}).items():
            history = target_history or trans.history
            output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history)
            output_dict['output_name'] = output_name
            rval['implicit_collections'].append(output_dict)

        return rval
Esempio n. 26
0
    def index(self,
              trans,
              deleted='False',
              f_email=None,
              f_name=None,
              f_any=None,
              **kwd):
        """
        GET /api/users
        GET /api/users/deleted
        Displays a collection (list) of users.

        :param deleted: (optional) If true, show deleted users
        :type  deleted: bool

        :param f_email: (optional) An email address to filter on. (Non-admin
                        users can only use this if ``expose_user_email`` is ``True`` in
                        galaxy.ini)
        :type  f_email: str

        :param f_name: (optional) A username to filter on. (Non-admin users
                       can only use this if ``expose_user_name`` is ``True`` in
                       galaxy.ini)
        :type  f_name: str

        :param f_any: (optional) Filter on username OR email. (Non-admin users
                       can use this, the email filter and username filter will
                       only be active if their corresponding ``expose_user_*`` is
                       ``True`` in galaxy.ini)
        :type  f_any: str
        """
        rval = []
        query = trans.sa_session.query(trans.app.model.User)
        deleted = util.string_as_bool(deleted)

        if f_email and (trans.user_is_admin
                        or trans.app.config.expose_user_email):
            query = query.filter(
                trans.app.model.User.email.like("%%%s%%" % f_email))

        if f_name and (trans.user_is_admin
                       or trans.app.config.expose_user_name):
            query = query.filter(
                trans.app.model.User.username.like("%%%s%%" % f_name))

        if f_any:
            if trans.user_is_admin:
                query = query.filter(
                    or_(trans.app.model.User.email.like("%%%s%%" % f_any),
                        trans.app.model.User.username.like("%%%s%%" % f_any)))
            else:
                if trans.app.config.expose_user_email and trans.app.config.expose_user_name:
                    query = query.filter(
                        or_(
                            trans.app.model.User.email.like("%%%s%%" % f_any),
                            trans.app.model.User.username.like("%%%s%%" %
                                                               f_any)))
                elif trans.app.config.expose_user_email:
                    query = query.filter(
                        trans.app.model.User.email.like("%%%s%%" % f_any))
                elif trans.app.config.expose_user_name:
                    query = query.filter(
                        trans.app.model.User.username.like("%%%s%%" % f_any))

        if deleted:
            # only admins can see deleted users
            if not trans.user_is_admin:
                return []
            query = query.filter(
                trans.app.model.User.table.c.deleted == true())
        else:
            # special case: user can see only their own user
            # special case2: if the galaxy admin has specified that other user email/names are
            #   exposed, we don't want special case #1
            if not trans.user_is_admin and not trans.app.config.expose_user_name and not trans.app.config.expose_user_email:
                item = trans.user.to_dict(
                    value_mapper={'id': trans.security.encode_id})
                return [item]
            query = query.filter(
                trans.app.model.User.table.c.deleted == false())
        for user in query:
            item = user.to_dict(value_mapper={'id': trans.security.encode_id})
            # If NOT configured to expose_email, do not expose email UNLESS the user is self, or
            # the user is an admin
            if user is not trans.user and not trans.user_is_admin:
                expose_keys = ["id"]
                if trans.app.config.expose_user_name:
                    expose_keys.append("username")
                if trans.app.config.expose_user_email:
                    expose_keys.append("email")
                new_item = {}
                for key, value in item.items():
                    if key in expose_keys:
                        new_item[key] = value
                item = new_item

            # TODO: move into api_values
            rval.append(item)
        return rval
Esempio n. 27
0
 def __init__( self, elem, display_application, build_sites ):
     filename = None
     data_table = None
     if elem.get( 'site_type', None ) is not None:
         filename = build_sites.get( elem.get( 'site_type' ) )
     else:
         filename = elem.get( 'from_file', None )
     if filename is None:
         data_table_name = elem.get( 'from_data_table', None )
         if data_table_name:
             data_table = display_application.app.tool_data_tables.get( data_table_name, None )
             assert data_table is not None, 'Unable to find data table named "%s".' % data_table_name
     
     assert filename is not None or data_table is not None,'Filename or data Table is required for dynamic_links.'
     skip_startswith = elem.get( 'skip_startswith', None )
     separator = elem.get( 'separator', '\t' )
     id_col = elem.get( 'id', None )
     try:
         id_col = int( id_col )
     except:
         if data_table:
             if id_col is None:
                 id_col = data_table.columns.get( 'id', None )
             if id_col is None:
                 id_col = data_table.columns.get( 'value', None )
             try:
                 id_col = int( id_col )
             except:
                 # id is set to a string or None, use column by that name if available
                 id_col = data_table.columns.get( id_col, None )
                 id_col = int( id_col )
     name_col = elem.get( 'name', None )
     try:
         name_col = int( name_col )
     except:
         if data_table:
             if name_col is None:
                 name_col = data_table.columns.get( 'name', None )
             else:
                 name_col = data_table.columns.get( name_col, None )
         else:
             name_col = None
     if name_col is None:
         name_col = id_col
     max_col = max( id_col, name_col )
     dynamic_params = {}
     if data_table is not None:
         max_col = max(  [ max_col ] + data_table.columns.values() )
         for key, value in data_table.columns.items():
             dynamic_params[key] = { 'column': value, 'split': False, 'separator': ',' }
     for dynamic_param in elem.findall( 'dynamic_param' ):
         name = dynamic_param.get( 'name' )
         value = int( dynamic_param.get( 'value' ) )
         split = string_as_bool( dynamic_param.get( 'split', False ) )
         param_separator =  dynamic_param.get( 'separator', ',' )
         max_col = max( max_col, value )
         dynamic_params[name] = { 'column': value, 'split': split, 'separator': param_separator }
     if filename:
         data_iter = open( filename )
     elif data_table:
         version, data_iter = data_table.get_version_fields()
         display_application.add_data_table_watch( data_table.name, version )
     links = []
     for line in data_iter:
         if isinstance( line, basestring ):
             if not skip_startswith or not line.startswith( skip_startswith ):
                 line = line.rstrip( '\n\r' )
                 if not line:
                     continue
                 fields = line.split( separator )
             else:
                 continue
         else:
             fields = line
         if len( fields ) > max_col:
             new_elem = deepcopy( elem )
             new_elem.set( 'id', fields[id_col] )
             new_elem.set( 'name', fields[name_col] )
             dynamic_values = {}
             for key, attributes in dynamic_params.iteritems():
                 value = fields[ attributes[ 'column' ] ]
                 if attributes['split']:
                     value = value.split( attributes['separator'] )
                 dynamic_values[key] = value
             #now populate
             links.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) )
         else:
             log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) )
     self.links = links
Esempio n. 28
0
 def get_bool(self, key, default):
     return string_as_bool(self.get(key, default))
Esempio n. 29
0
    def load(self, trans, payload=None, **kwd):
        """
        Load dataset(s) from the given source into the library.

        * POST /api/libraries/datasets

        :param   payload: dictionary structure containing:
            :param  encoded_folder_id:      the encoded id of the folder to import dataset(s) to
            :type   encoded_folder_id:      an encoded id string
            :param  source:                 source the datasets should be loaded from
                    Source can be:
                        user directory - root folder specified in galaxy.ini as "$user_library_import_dir"
                            example path: path/to/galaxy/$user_library_import_dir/[email protected]/{user can browse everything here}
                            the folder with the user login has to be created beforehand
                        (admin)import directory - root folder specified in galaxy ini as "$library_import_dir"
                            example path: path/to/galaxy/$library_import_dir/{admin can browse everything here}
                        (admin)any absolute or relative path - option allowed with "allow_library_path_paste" in galaxy.ini
            :type   source:                 str
            :param  link_data:              flag whether to link the dataset to data or copy it to Galaxy, defaults to copy
                                            while linking is set to True all symlinks will be resolved _once_
            :type   link_data:              bool
            :param  preserve_dirs:          flag whether to preserve the directory structure when importing dir
                                            if False only datasets will be imported
            :type   preserve_dirs:          bool
            :param  file_type:              file type of the loaded datasets, defaults to 'auto' (autodetect)
            :type   file_type:              str
            :param  dbkey:                  dbkey of the loaded genome, defaults to '?' (unknown)
            :type   dbkey:                  str
            :param  tag_using_filenames:    flag whether to generate dataset tags from filenames
            :type   tag_using_filenames:    bool
        :type   dictionary

        :returns:   dict containing information about the created upload job
        :rtype:     dictionary

        :raises: RequestParameterMissingException, AdminRequiredException, ConfigDoesNotAllowException, RequestParameterInvalidException
                    InsufficientPermissionsException, ObjectNotFound
        """
        if payload:
            kwd.update(payload)
        kwd['space_to_tab'] = False
        kwd['to_posix_lines'] = True
        kwd['dbkey'] = kwd.get('dbkey', '?')
        kwd['file_type'] = kwd.get('file_type', 'auto')
        kwd['link_data_only'] = 'link_to_files' if util.string_as_bool(
            kwd.get('link_data', False)) else 'copy_files'
        kwd['tag_using_filenames'] = util.string_as_bool(
            kwd.get('tag_using_filenames', None))
        encoded_folder_id = kwd.get('encoded_folder_id', None)
        if encoded_folder_id is not None:
            folder_id = self.folder_manager.cut_and_decode(
                trans, encoded_folder_id)
        else:
            raise exceptions.RequestParameterMissingException(
                'The required attribute encoded_folder_id is missing.')
        path = kwd.get('path', None)
        if path is None:
            raise exceptions.RequestParameterMissingException(
                'The required attribute path is missing.')
        folder = self.folder_manager.get(trans, folder_id)

        source = kwd.get('source', None)
        if source not in [
                'userdir_file', 'userdir_folder', 'importdir_file',
                'importdir_folder', 'admin_path'
        ]:
            raise exceptions.RequestParameterMissingException(
                'You have to specify "source" parameter. Possible values are "userdir_file", "userdir_folder", "admin_path", "importdir_file" and "importdir_folder". '
            )
        elif source in ['importdir_file', 'importdir_folder']:
            if not trans.user_is_admin():
                raise exceptions.AdminRequiredException(
                    'Only admins can import from importdir.')
            if not trans.app.config.library_import_dir:
                raise exceptions.ConfigDoesNotAllowException(
                    'The configuration of this Galaxy instance does not allow admins to import into library from importdir.'
                )
            import_base_dir = trans.app.config.library_import_dir
            if not safe_relpath(path):
                # admins shouldn't be able to explicitly specify a path outside server_dir, but symlinks are allowed.
                # the reasoning here is that galaxy admins may not have direct filesystem access or can only access
                # library_import_dir via FTP (which cannot create symlinks), and may rely on sysadmins to set up the
                # import directory. if they have filesystem access, all bets are off.
                raise exceptions.RequestParameterInvalidException(
                    'The given path is invalid.')
            path = os.path.join(import_base_dir, path)
        elif source in ['userdir_file', 'userdir_folder']:
            unsafe = None
            username = trans.user.username if trans.app.config.user_library_import_check_permissions else None
            user_login = trans.user.email
            user_base_dir = trans.app.config.user_library_import_dir
            if user_base_dir is None:
                raise exceptions.ConfigDoesNotAllowException(
                    'The configuration of this Galaxy instance does not allow upload from user directories.'
                )
            full_dir = os.path.join(user_base_dir, user_login)

            if not safe_contains(full_dir,
                                 path,
                                 whitelist=trans.app.config.
                                 user_library_import_symlink_whitelist):
                # the path is a symlink outside the user dir
                path = os.path.join(full_dir, path)
                log.error(
                    'User attempted to import a path that resolves to a path outside of their import dir: %s -> %s',
                    path, os.path.realpath(path))
                raise exceptions.RequestParameterInvalidException(
                    'The given path is invalid.')
            if trans.app.config.user_library_import_check_permissions and not full_path_permission_for_user(
                    full_dir, path, username):
                log.error(
                    'User attempted to import a path that resolves to a path outside of their import dir: '
                    '%s -> %s and cannot be read by them.', path,
                    os.path.realpath(path))
                raise exceptions.RequestParameterInvalidException(
                    'The given path is invalid.')
            path = os.path.join(full_dir, path)
            for unsafe in unsafe_walk(
                    path,
                    whitelist=[full_dir] +
                    trans.app.config.user_library_import_symlink_whitelist,
                    username=username):
                # the path is a dir and contains files that symlink outside the user dir
                error = 'User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', \
                        path, os.path.realpath(path)
                if trans.app.config.user_library_import_check_permissions:
                    error += ' or is not readable for them.'
                log.error(error)
            if unsafe:
                raise exceptions.RequestParameterInvalidException(
                    'The given path is invalid.')
            if not os.path.exists(path):
                raise exceptions.RequestParameterInvalidException(
                    'Given path does not exist on the host.')
            if not self.folder_manager.can_add_item(trans, folder):
                raise exceptions.InsufficientPermissionsException(
                    'You do not have proper permission to add items to the given folder.'
                )
        elif source == 'admin_path':
            if not trans.app.config.allow_library_path_paste:
                raise exceptions.ConfigDoesNotAllowException(
                    'The configuration of this Galaxy instance does not allow admins to import into library from path.'
                )
            if not trans.user_is_admin():
                raise exceptions.AdminRequiredException(
                    'Only admins can import from path.')

        # Set up the traditional tool state/params
        tool_id = 'upload1'
        tool = trans.app.toolbox.get_tool(tool_id)
        state = tool.new_state(trans)
        populate_state(trans, tool.inputs, kwd, state.inputs)
        tool_params = state.inputs
        dataset_upload_inputs = []
        for input in tool.inputs.values():
            if input.type == "upload_dataset":
                dataset_upload_inputs.append(input)
        library_bunch = upload_common.handle_library_params(
            trans, {}, trans.security.encode_id(folder.id))
        abspath_datasets = []
        kwd['filesystem_paths'] = path
        if source in ['importdir_folder']:
            kwd['filesystem_paths'] = os.path.join(import_base_dir, path)
        # user wants to import one file only
        elif source in ["userdir_file", "importdir_file"]:
            file = os.path.abspath(path)
            abspath_datasets.append(
                self._make_library_uploaded_dataset(trans, kwd,
                                                    os.path.basename(file),
                                                    file, 'server_dir',
                                                    library_bunch))
        # user wants to import whole folder
        elif source == "userdir_folder":
            uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets(
                trans, kwd, library_bunch, 200, '')
            uploaded_datasets = uploaded_datasets_bunch[0]
            if uploaded_datasets is None:
                raise exceptions.ObjectNotFound(
                    'Given folder does not contain any datasets.')
            for ud in uploaded_datasets:
                ud.path = os.path.abspath(ud.path)
                abspath_datasets.append(ud)
        #  user wants to import from path
        if source in ["admin_path", "importdir_folder"]:
            # validate the path is within root
            uploaded_datasets_bunch = self._get_path_paste_uploaded_datasets(
                trans, kwd, library_bunch, 200, '')
            uploaded_datasets = uploaded_datasets_bunch[0]
            if uploaded_datasets is None:
                raise exceptions.ObjectNotFound(
                    'Given folder does not contain any datasets.')
            for ud in uploaded_datasets:
                ud.path = os.path.abspath(ud.path)
                abspath_datasets.append(ud)
        json_file_path = upload_common.create_paramfile(
            trans, abspath_datasets)
        data_list = [ud.data for ud in abspath_datasets]
        job_params = {}
        job_params['link_data_only'] = dumps(
            kwd.get('link_data_only', 'copy_files'))
        job_params['uuid'] = dumps(kwd.get('uuid', None))
        job, output = upload_common.create_job(trans,
                                               tool_params,
                                               tool,
                                               json_file_path,
                                               data_list,
                                               folder=folder,
                                               job_params=job_params)
        trans.sa_session.add(job)
        trans.sa_session.flush()
        job_dict = job.to_dict()
        job_dict['id'] = trans.security.encode_id(job_dict['id'])
        return job_dict
Esempio n. 30
0
 def is_shed_tool_conf(self):
     has_tool_path = self.parse_tool_path() is not None
     is_shed_conf = string_as_bool(self.root.get("is_shed_conf", "True"))
     return has_tool_path and is_shed_conf
Esempio n. 31
0
 def parse_monitor(self):
     return string_as_bool(self.root.get('monitor', DEFAULT_MONITOR))
Esempio n. 32
0
    def create(self, trans, library_id, payload, **kwd):
        """
        create( self, trans, library_id, payload, **kwd )
        * POST /api/libraries/{library_id}/contents:
            create a new library file or folder

        To copy an HDA into a library send ``create_type`` of 'file' and
        the HDA's encoded id in ``from_hda_id`` (and optionally ``ldda_message``).

        To copy an HDCA into a library send ``create_type`` of 'file' and
        the HDCA's encoded id in ``from_hdca_id`` (and optionally ``ldda_message``).

        :type   library_id: str
        :param  library_id: the encoded id of the library where to create the new item
        :type   payload:    dict
        :param  payload:    dictionary structure containing:

            * folder_id:    the encoded id of the parent folder of the new item
            * create_type:  the type of item to create ('file', 'folder' or 'collection')
            * from_hda_id:  (optional, only if create_type is 'file') the
                encoded id of an accessible HDA to copy into the library
            * ldda_message: (optional) the new message attribute of the LDDA created
            * extended_metadata: (optional) sub-dictionary containing any extended
                metadata to associate with the item
            * upload_option: (optional) one of 'upload_file' (default), 'upload_directory' or 'upload_paths'
            * server_dir: (optional, only if upload_option is
                'upload_directory') relative path of the subdirectory of Galaxy
                ``library_import_dir`` (if admin) or ``user_library_import_dir``
                (if non-admin) to upload. All and only the files (i.e.
                no subdirectories) contained in the specified directory will be
                uploaded.
            * filesystem_paths: (optional, only if upload_option is
                'upload_paths' and the user is an admin) file paths on the
                Galaxy server to upload to the library, one file per line
            * link_data_only: (optional, only when upload_option is
                'upload_directory' or 'upload_paths') either 'copy_files'
                (default) or 'link_to_files'. Setting to 'link_to_files'
                symlinks instead of copying the files
            * name: (optional, only if create_type is 'folder') name of the
                folder to create
            * description: (optional, only if create_type is 'folder')
                description of the folder to create
            * tag_using_filename: (optional)
                create tags on datasets using the file's original name

        :returns:   a dictionary describing the new item unless ``from_hdca_id`` is supplied,
                    in that case a list of such dictionaries is returned.
        :rtype:     object
        """
        if 'create_type' not in payload:
            trans.response.status = 400
            return "Missing required 'create_type' parameter."
        else:
            create_type = payload.pop('create_type')
        if create_type not in ('file', 'folder', 'collection'):
            trans.response.status = 400
            return "Invalid value for 'create_type' parameter ( %s ) specified." % create_type

        if 'folder_id' not in payload:
            trans.response.status = 400
            return "Missing required 'folder_id' parameter."
        else:
            folder_id = payload.pop('folder_id')
            class_name, folder_id = self._decode_library_content_id(folder_id)
        try:
            # security is checked in the downstream controller
            parent = self.get_library_folder(trans, folder_id, check_ownership=False, check_accessible=False)
        except Exception as e:
            return str(e)
        # The rest of the security happens in the library_common controller.
        real_folder_id = trans.security.encode_id(parent.id)

        payload['tag_using_filenames'] = util.string_as_bool(payload.get('tag_using_filenames', None))

        # are we copying an HDA to the library folder?
        #   we'll need the id and any message to attach, then branch to that private function
        from_hda_id, from_hdca_id, ldda_message = (payload.pop('from_hda_id', None), payload.pop('from_hdca_id', None), payload.pop('ldda_message', ''))
        if create_type == 'file':
            if from_hda_id:
                return self._copy_hda_to_library_folder(trans, self.hda_manager, self.decode_id(from_hda_id), real_folder_id, ldda_message)
            if from_hdca_id:
                return self._copy_hdca_to_library_folder(trans, self.hda_manager, self.decode_id(from_hdca_id), real_folder_id, ldda_message)

        # check for extended metadata, store it and pop it out of the param
        # otherwise sanitize_param will have a fit
        ex_meta_payload = payload.pop('extended_metadata', None)

        # Now create the desired content object, either file or folder.
        if create_type == 'file':
            status, output = self._upload_library_dataset(trans, library_id, real_folder_id, **payload)
        elif create_type == 'folder':
            status, output = self._create_folder(trans, real_folder_id, library_id, **payload)
        elif create_type == 'collection':
            # Not delegating to library_common, so need to check access to parent
            # folder here.
            self.check_user_can_add_to_library_item(trans, parent, check_accessible=True)
            create_params = api_payload_to_create_params(payload)
            create_params['parent'] = parent
            service = trans.app.dataset_collections_service
            dataset_collection_instance = service.create(**create_params)
            return [dictify_dataset_collection_instance(dataset_collection_instance, security=trans.security, parent=parent)]
        if status != 200:
            trans.response.status = status
            return output
        else:
            rval = []
            for v in output.values():
                if ex_meta_payload is not None:
                    # If there is extended metadata, store it, attach it to the dataset, and index it
                    ex_meta = ExtendedMetadata(ex_meta_payload)
                    trans.sa_session.add(ex_meta)
                    v.extended_metadata = ex_meta
                    trans.sa_session.add(v)
                    trans.sa_session.flush()
                    for path, value in self._scan_json_block(ex_meta_payload):
                        meta_i = ExtendedMetadataIndex(ex_meta, path, value)
                        trans.sa_session.add(meta_i)
                    trans.sa_session.flush()
                if type(v) == trans.app.model.LibraryDatasetDatasetAssociation:
                    v = v.library_dataset
                encoded_id = trans.security.encode_id(v.id)
                if create_type == 'folder':
                    encoded_id = 'F' + encoded_id
                rval.append(dict(id=encoded_id,
                                 name=v.name,
                                 url=url_for('library_content', library_id=library_id, id=encoded_id)))
            return rval
Esempio n. 33
0
 def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
     super().__init__(app_info=app_info, **kwds)
     self.namespace = namespace
     self.hash_func = hash_func
     self.auto_install = string_as_bool(auto_install)
Esempio n. 34
0
    def delete(self, trans, library_id, id, **kwd):
        """
        delete( self, trans, library_id, id, **kwd )
        * DELETE /api/libraries/{library_id}/contents/{id}
            delete the LibraryDataset with the given ``id``

        :type   id:     str
        :param  id:     the encoded id of the library dataset to delete
        :type   kwd:    dict
        :param  kwd:    (optional) dictionary structure containing:

            * payload:     a dictionary itself containing:
                * purge:   if True, purge the LD

        :rtype:     dict
        :returns:   an error object if an error occurred or a dictionary containing:
            * id:         the encoded id of the library dataset,
            * deleted:    if the library dataset was marked as deleted,
            * purged:     if the library dataset was purged
        """
        # a request body is optional here
        purge = False
        if kwd.get('payload', None):
            purge = util.string_as_bool(kwd['payload'].get('purge', False))

        rval = {'id': id}
        try:
            ld = self.get_library_dataset(trans, id, check_ownership=False, check_accessible=True)
            user_is_admin = trans.user_is_admin
            can_modify = trans.app.security_agent.can_modify_library_item(trans.user.all_roles(), ld)
            log.debug('is_admin: %s, can_modify: %s', user_is_admin, can_modify)
            if not (user_is_admin or can_modify):
                trans.response.status = 403
                rval.update({'error': 'Unauthorized to delete or purge this library dataset'})
                return rval

            ld.deleted = True
            if purge:
                ld.purged = True
                trans.sa_session.add(ld)
                trans.sa_session.flush()

                # TODO: had to change this up a bit from Dataset.user_can_purge
                dataset = ld.library_dataset_dataset_association.dataset
                no_history_assoc = len(dataset.history_associations) == len(dataset.purged_history_associations)
                no_library_assoc = dataset.library_associations == [ld.library_dataset_dataset_association]
                can_purge_dataset = not dataset.purged and no_history_assoc and no_library_assoc

                if can_purge_dataset:
                    try:
                        ld.library_dataset_dataset_association.dataset.full_delete()
                        trans.sa_session.add(ld.dataset)
                    except Exception:
                        pass
                    # flush now to preserve deleted state in case of later interruption
                    trans.sa_session.flush()
                rval['purged'] = True
            trans.sa_session.flush()
            rval['deleted'] = True

        except exceptions.httpexceptions.HTTPInternalServerError:
            log.exception('Library_contents API, delete: uncaught HTTPInternalServerError: %s, %s',
                          id, str(kwd))
            raise
        except exceptions.httpexceptions.HTTPException:
            raise
        except Exception as exc:
            log.exception('library_contents API, delete: uncaught exception: %s, %s',
                          id, str(kwd))
            trans.response.status = 500
            rval.update({'error': str(exc)})
        return rval
Esempio n. 35
0
 def __init__(self, elem, parent):
     ExternalServiceParameter.__init__(self, elem, parent)
     self.strip = string_as_bool(elem.get('strip', 'False'))
     self.text = elem.text