def tool_shed_is_this_tool_shed(toolshed_base_url): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url) cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url( str(url_for('/', qualified=True))) return cleaned_toolshed_base_url == cleaned_tool_shed
def password_manager_for_url(self, url): """ If the tool shed is using external auth, the client to the tool shed must authenticate to that as well. This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the url passed in. Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at some stage... """ url_sans_protocol = remove_protocol_from_tool_shed_url(url) for shed_name, shed_url in self.tool_sheds.items(): shed_url_sans_protocol = remove_protocol_from_tool_shed_url(shed_url) if url_sans_protocol.startswith(shed_url_sans_protocol): return self.tool_sheds_auth[shed_name] log.debug("Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str(url)) return None
def get_or_create_tool_shed_repository(self, tool_shed, name, owner, changeset_revision): """ Return a tool shed repository database record defined by the combination of tool shed, repository name, repository owner and changeset_revision or installed_changeset_revision. A new tool shed repository record will be created if one is not located. """ install_model = self.app.install_model # We store the port in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed) # This method is used only in Galaxy, not the tool shed. repository = repository_util.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision) if not repository: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed) repository_clone_url = os.path.join(tool_shed_url, 'repos', owner, name) ctx_rev = get_ctx_rev(self.app, tool_shed_url, name, owner, changeset_revision) repository = repository_util.create_or_update_tool_shed_repository( app=self.app, name=name, description=None, installed_changeset_revision=changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, status=install_model.ToolShedRepository.installation_status. NEW, metadata_dict={}, current_changeset_revision=None, owner=owner, dist_to_shed=False) return repository
def get_installed_repository(app, tool_shed=None, name=None, owner=None, changeset_revision=None, installed_changeset_revision=None, repository_id=None, refresh=False, from_cache=False): """ Return a tool shed repository database record defined by the combination of a toolshed, repository name, repository owner and either current or originally installed changeset_revision. """ # We store the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed) if from_cache and hasattr(app, 'tool_shed_repository_cache'): if refresh: app.tool_shed_repository_cache.rebuild() return app.tool_shed_repository_cache.get_installed_repository(tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision, changeset_revision=changeset_revision, repository_id=repository_id) query = app.install_model.context.query(app.install_model.ToolShedRepository) if repository_id: clause_list = [app.install_model.ToolShedRepository.table.c.id == repository_id] else: clause_list = [app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, app.install_model.ToolShedRepository.table.c.name == name, app.install_model.ToolShedRepository.table.c.owner == owner] if changeset_revision is not None: clause_list.append(app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision) if installed_changeset_revision is not None: clause_list.append(app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision) return query.filter(and_(*clause_list)).first()
def get_repository_for_dependency_relationship(app, tool_shed, name, owner, changeset_revision): """ Return an installed tool_shed_repository database record that is defined by either the current changeset revision or the installed_changeset_revision. """ # This method is used only in Galaxy, not the Tool Shed. We store the port (if one exists) in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed) if tool_shed is None or name is None or owner is None or changeset_revision is None: message = "Unable to retrieve the repository record from the database because one or more of the following " message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \ (str(tool_shed), str(name), str(owner), str(changeset_revision)) raise Exception(message) app.tool_shed_repository_cache.rebuild() repository = get_installed_repository( app=app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=changeset_revision) if not repository: repository = get_installed_repository( app=app, tool_shed=tool_shed, name=name, owner=owner, changeset_revision=changeset_revision) if not repository: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed) repository_clone_url = os.path.join(tool_shed_url, 'repos', owner, name) repo_info_tuple = (None, repository_clone_url, changeset_revision, None, owner, None, None) repository, pcr = repository_was_previously_installed( app, tool_shed_url, name, repo_info_tuple) if not repository: # The received changeset_revision is no longer installable, so get the next changeset_revision # in the repository's changelog in the tool shed that is associated with repository_metadata. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed) params = dict(name=name, owner=owner, changeset_revision=changeset_revision) pathspec = ['repository', 'next_installable_changeset_revision'] text = util.url_get( tool_shed_url, auth=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) if text: repository = get_installed_repository(app=app, tool_shed=tool_shed, name=name, owner=owner, changeset_revision=text) return repository
def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy when a repository and its dependencies are being installed. Inspect the received repository_dependencies and determine if the encoded id of each required repository is in the received tsr_ids. If so, then determine whether that required repository should be imported / installed prior to its dependent repository. Return a list of encoded repository ids, each of which is contained in the received list of tsr_ids, and whose associated repositories must be imported / installed prior to the dependent repository associated with the received repository_dependencies. """ prior_tsr_ids = [] if repository_dependencies: for key, rd_tups in repository_dependencies.items(): if key in ['description', 'root_key']: continue for rd_tup in rd_tups: tool_shed, \ name, \ owner, \ changeset_revision, \ prior_installation_required, \ only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple(rd_tup) # If only_if_compiling_contained_td is False, then the repository dependency # is not required to be installed prior to the dependent repository even if # prior_installation_required is True. This is because the only meaningful # content of the repository dependency is its contained tool dependency, which # is required in order to compile the dependent repository's tool dependency. # In the scenario where the repository dependency is not installed prior to the # dependent repository's tool dependency compilation process, the tool dependency # compilation framework will install the repository dependency prior to compilation # of the dependent repository's tool dependency. if not util.asbool(only_if_compiling_contained_td): if util.asbool(prior_installation_required): if is_tool_shed_client(app): # We store the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed) repository = get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision) else: repository = get_repository_by_name_and_owner( app, name, owner) if repository: encoded_repository_id = app.security.encode_id( repository.id) if encoded_repository_id in tsr_ids: prior_tsr_ids.append(encoded_repository_id) return prior_tsr_ids
def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td): """ Assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. """ # The tool_shed portion of the key must be the value that is stored in the tool_shed_repository.tool_shed column # of the Galaxy database for an installed repository. This value does not include the protocol, but does include # the port if there is one. tool_shed = remove_protocol_from_tool_shed_url(toolshed_base_url) return '%s%s%s%s%s%s%s%s%s%s%s' % ( tool_shed, STRSEP, str(repository_name), STRSEP, str(repository_owner), STRSEP, str(changeset_revision), STRSEP, str(prior_installation_required), STRSEP, str(only_if_compiling_contained_td))
def handle_complex_repository_dependency_for_package( self, elem, package_name, package_version, tool_shed_repository, from_tool_migration_manager=False): """ Inspect the repository defined by a complex repository dependency definition and take certain steps to enable installation of the received package name and version to proceed. The received elem is the <repository> tag set which defines the complex repository dependency. The received tool_shed_repository is the installed tool shed repository for which the tool dependency defined by the received package_name and package_version is being installed. """ handled_tool_dependencies = [] tool_shed_url = elem.attrib['toolshed'] required_repository_name = elem.attrib['name'] required_repository_owner = elem.attrib['owner'] default_required_repository_changeset_revision = elem.attrib[ 'changeset_revision'] required_repository = get_repository_for_dependency_relationship( self.app, tool_shed_url, required_repository_name, required_repository_owner, default_required_repository_changeset_revision) tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) tmp_filename = None if required_repository: required_repository_changeset_revision = required_repository.installed_changeset_revision # Define the installation directory for the required tool dependency package in the required repository. required_repository_package_install_dir = \ tool_dependency_util.get_tool_dependency_install_dir(app=self.app, repository_name=required_repository_name, repository_owner=required_repository_owner, repository_changeset_revision=required_repository_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version) # Define this dependent repository's tool dependency installation directory that will contain # the env.sh file with a path to the required repository's installed tool dependency package. dependent_install_dir = \ tool_dependency_util.get_tool_dependency_install_dir(app=self.app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version) if os.path.exists(dependent_install_dir): # The install manager handles tool migration stages and the sync_database_with_file_system() # method handles two scenarios: (1) where a Galaxy file system environment related to installed # Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy # database tables associated with these installed items, and (2) the Tool Shed's install and test # framework which installs repositories in 2 stages, those of type tool_dependency_definition # followed by those containing valid tools and tool functional test components. Neither of these # scenarios apply when the install manager is running. if from_tool_migration_manager: can_install_tool_dependency = True else: # Notice that we'll throw away the following tool_dependency if it can be installed. tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system( self.app, tool_shed_repository, package_name, package_version, dependent_install_dir, tool_dependency_type='package') if not can_install_tool_dependency: log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), " "so appending it to the list of handled tool dependencies.", str(tool_dependency.name), str(tool_dependency.version)) handled_tool_dependencies.append(tool_dependency) else: can_install_tool_dependency = True if can_install_tool_dependency: # Set this dependent repository's tool dependency env.sh file with a path to the required repository's # installed tool dependency package. We can get everything we need from the discovered installed # required_repository. if required_repository.is_deactivated_or_installed: if not os.path.exists( required_repository_package_install_dir): log.error( f'Missing required tool dependency directory {str(required_repository_package_install_dir)}' ) repo_files_dir = required_repository.repo_files_directory( self.app) if not repo_files_dir: message = "Unable to locate the repository directory for revision %s of installed repository %s owned by %s." % \ (str(required_repository.changeset_revision), str(required_repository.name), str(required_repository.owner)) raise Exception(message) tool_dependencies_config = get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml') if tool_dependencies_config: config_to_use = tool_dependencies_config else: message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \ (str(required_repository.changeset_revision), str(required_repository.name), str(required_repository.owner)) raise Exception(message) else: # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. text = get_updated_changeset_revisions_from_tool_shed( app=self.app, tool_shed_url=tool_shed, name=required_repository_name, owner=required_repository_owner, changeset_revision= required_repository_changeset_revision) if text: updated_changeset_revisions = listify(text) # The list of changeset revisions is in reverse order, so the newest will be first. required_repository_changeset_revision = updated_changeset_revisions[ 0] # Make a call to the tool shed to get the required repository's tool_dependencies.xml file. tmp_filename = self.create_temporary_tool_dependencies_config( tool_shed, required_repository_name, required_repository_owner, required_repository_changeset_revision) config_to_use = tmp_filename handled_tool_dependencies = \ self.create_tool_dependency_with_initialized_env_sh_file(dependent_install_dir=dependent_install_dir, tool_shed_repository=tool_shed_repository, required_repository=required_repository, package_name=package_name, package_version=package_version, tool_dependencies_config=config_to_use) self.remove_file(tmp_filename) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ (str(required_repository_name), str(required_repository_owner), str(default_required_repository_changeset_revision)) raise Exception(message) return handled_tool_dependencies
def get_env_shell_file_paths(self, elem): # Currently only the following tag set is supported. # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> # <package name="numpy" version="1.7.1" /> # </repository> env_shell_file_paths = [] toolshed = elem.get('toolshed', None) repository_name = elem.get('name', None) repository_owner = elem.get('owner', None) changeset_revision = elem.get('changeset_revision', None) if toolshed and repository_name and repository_owner and changeset_revision: # The protocol is not stored, but the port is if it exists. toolshed = remove_protocol_from_tool_shed_url(toolshed) repository = get_repository_for_dependency_relationship(self.app, toolshed, repository_name, repository_owner, changeset_revision) if repository: for sub_elem in elem: tool_dependency_type = sub_elem.tag tool_dependency_name = sub_elem.get('name') tool_dependency_version = sub_elem.get('version') if tool_dependency_type and tool_dependency_name and tool_dependency_version: # Get the tool_dependency so we can get its installation directory. tool_dependency = None for tool_dependency in repository.tool_dependencies: if tool_dependency.type == tool_dependency_type and \ tool_dependency.name == tool_dependency_name and \ tool_dependency.version == tool_dependency_version: break if tool_dependency: installation_directory = tool_dependency.installation_directory(self.app) env_shell_file_path = self.get_env_shell_file_path(installation_directory) if env_shell_file_path: env_shell_file_paths.append(env_shell_file_path) else: error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ (str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name)) log.debug(error_message) continue else: error_message = "Skipping tool dependency definition because unable to locate tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ (str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version), str(repository.name)) log.debug(error_message) continue else: error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ (str(tool_dependency_type), str(tool_dependency_name), str(tool_dependency_version)) log.debug(error_message) continue else: error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ (str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision)) log.debug(error_message) else: error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ (str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision)) log.debug(error_message) return env_shell_file_paths