def get_or_create_tool_shed_repository( self, tool_shed, name, owner, changeset_revision ): """ Return a tool shed repository database record defined by the combination of tool shed, repository name, repository owner and changeset_revision or installed_changeset_revision. A new tool shed repository record will be created if one is not located. """ install_model = self.app.install_model # We store the port in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) # This method is used only in Galaxy, not the tool shed. repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision ) if not repository: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision ) repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=None, installed_changeset_revision=changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=install_model.ToolShedRepository.installation_status.NEW, current_changeset_revision=None, owner=owner, dist_to_shed=False ) return repository
def create_or_update_tool_shed_repository_record(self, name, owner, changeset_revision, description=None): # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> relative_clone_dir = os.path.join(self.tool_shed, 'repos', owner, name, changeset_revision) clone_dir = os.path.join(self.tool_path, relative_clone_dir) if not self.__iscloned(clone_dir): repository_clone_url = os.path.join(self.tool_shed_url, 'repos', owner, name) ctx_rev = suc.get_ctx_rev(self.app, self.tool_shed_url, name, owner, changeset_revision) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=self.app.install_model.ToolShedRepository. installation_status.NEW, current_changeset_revision=None, owner=self.repository_owner, dist_to_shed=True) return tool_shed_repository return None
def create_or_update_tool_shed_repository_record( self, name, owner, changeset_revision, description=None ): # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> relative_clone_dir = os.path.join( self.tool_shed, 'repos', owner, name, changeset_revision ) clone_dir = os.path.join( self.tool_path, relative_clone_dir ) if not self.__iscloned( clone_dir ): repository_clone_url = os.path.join( self.tool_shed_url, 'repos', owner, name ) ctx_rev = suc.get_ctx_rev( self.app, self.tool_shed_url, name, owner, changeset_revision ) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=self.app.install_model.ToolShedRepository.installation_status.NEW, current_changeset_revision=None, owner=self.repository_owner, dist_to_shed=True ) return tool_shed_repository return None
def create_repository_dependency_objects( self, tool_path, tool_shed_url, repo_info_dicts, install_repository_dependencies=False, no_changes_checked=False, tool_panel_section_id=None, new_tool_panel_section_label=None ): """ Discover all repository dependencies and make sure all tool_shed_repository and associated repository_dependency records exist as well as the dependency relationships between installed repositories. This method is called when uninstalled repositories are being reinstalled. If the user elected to install repository dependencies, all items in the all_repo_info_dicts list will be processed. However, if repository dependencies are not to be installed, only those items contained in the received repo_info_dicts list will be processed. """ install_model = self.app.install_model log.debug( "Creating repository dependency objects..." ) # The following list will be maintained within this method to contain all created # or updated tool shed repositories, including repository dependencies that may not # be installed. all_created_or_updated_tool_shed_repositories = [] # There will be a one-to-one mapping between items in 3 lists: # created_or_updated_tool_shed_repositories, tool_panel_section_keys # and filtered_repo_info_dicts. The 3 lists will filter out repository # dependencies that are not to be installed. created_or_updated_tool_shed_repositories = [] tool_panel_section_keys = [] # Repositories will be filtered (e.g., if already installed, if elected # to not be installed, etc), so filter the associated repo_info_dicts accordingly. filtered_repo_info_dicts = [] # Discover all repository dependencies and retrieve information for installing # them. Even if the user elected to not install repository dependencies we have # to make sure all repository dependency objects exist so that the appropriate # repository dependency relationships can be built. all_required_repo_info_dict = self.get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ) all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] ) if not all_repo_info_dicts: # No repository dependencies were discovered so process the received repositories. all_repo_info_dicts = [ rid for rid in repo_info_dicts ] for repo_info_dict in all_repo_info_dicts: # If the user elected to install repository dependencies, all items in the # all_repo_info_dicts list will be processed. However, if repository dependencies # are not to be installed, only those items contained in the received repo_info_dicts # list will be processed but the all_repo_info_dicts list will be used to create all # defined repository dependency relationships. if self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ) or install_repository_dependencies: for name, repo_info_tuple in repo_info_dict.items(): can_update_db_record = False description, \ repository_clone_url, \ changeset_revision, \ ctx_rev, \ repository_owner, \ repository_dependencies, \ tool_dependencies = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) # See if the repository has an existing record in the database. repository_db_record, installed_changeset_revision = \ suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) if repository_db_record: if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.INSTALLED, install_model.ToolShedRepository.installation_status.CLONING, install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES, install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: debug_msg = "Skipping installation of revision %s of repository '%s' because it was installed " % \ ( str( changeset_revision ), str( repository_db_record.name ) ) debug_msg += "with the (possibly updated) revision %s and its current installation status is '%s'." % \ ( str( installed_changeset_revision ), str( repository_db_record.status ) ) log.debug( debug_msg ) can_update_db_record = False else: if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.ERROR, install_model.ToolShedRepository.installation_status.NEW, install_model.ToolShedRepository.installation_status.UNINSTALLED ]: # The current tool shed repository is not currently installed, so we can update its # record in the database. name = repository_db_record.name installed_changeset_revision = repository_db_record.installed_changeset_revision metadata_dict = repository_db_record.metadata dist_to_shed = repository_db_record.dist_to_shed can_update_db_record = True elif repository_db_record.status in [ install_model.ToolShedRepository.installation_status.DEACTIVATED ]: # The current tool shed repository is deactivated, so updating its database record # is not necessary - just activate it. log.debug( "Reactivating deactivated tool_shed_repository '%s'." % str( repository_db_record.name ) ) self.app.installed_repository_manager.activate_repository( repository_db_record ) # No additional updates to the database record are necessary. can_update_db_record = False elif repository_db_record.status not in [ install_model.ToolShedRepository.installation_status.NEW ]: # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find # the previously installed and uninstalled repository instead of creating a new record. changeset_revision = repository_db_record.installed_changeset_revision self.reset_previously_installed_repository( repository_db_record ) can_update_db_record = True else: # No record exists in the database for the repository currently being processed. installed_changeset_revision = changeset_revision metadata_dict = {} dist_to_shed = False can_update_db_record = True if can_update_db_record: # The database record for the tool shed repository currently being processed can be updated. # Get the repository metadata to see where it was previously located in the tool panel. tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository_db_record and repository_db_record.metadata: tool_section, tool_panel_section_key = \ tpm.handle_tool_panel_selection( toolbox=self.app.toolbox, metadata=repository_db_record.metadata, no_changes_checked=no_changes_checked, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) else: # We're installing a new tool shed repository that does not yet have a database record. tool_panel_section_key, tool_section = \ tpm.handle_tool_panel_section( self.app.toolbox, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) tool_shed_repository = \ suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=installed_changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=install_model.ToolShedRepository.installation_status.NEW, current_changeset_revision=changeset_revision, owner=repository_owner, dist_to_shed=False ) if tool_shed_repository not in all_created_or_updated_tool_shed_repositories: all_created_or_updated_tool_shed_repositories.append( tool_shed_repository ) # Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if # it is supposed to be installed. if install_repository_dependencies or self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ): if tool_shed_repository not in created_or_updated_tool_shed_repositories: # Keep the one-to-one mapping between items in 3 lists. created_or_updated_tool_shed_repositories.append( tool_shed_repository ) tool_panel_section_keys.append( tool_panel_section_key ) filtered_repo_info_dicts.append( repo_info_dict ) # Build repository dependency relationships even if the user chose to not install repository dependencies. self.build_repository_dependency_relationships( all_repo_info_dicts, all_created_or_updated_tool_shed_repositories ) return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts
def create_repository_dependency_objects( self, tool_path, tool_shed_url, repo_info_dicts, install_repository_dependencies=False, no_changes_checked=False, tool_panel_section_id=None, new_tool_panel_section_label=None ): """ Discover all repository dependencies and make sure all tool_shed_repository and associated repository_dependency records exist as well as the dependency relationships between installed repositories. This method is called when uninstalled repositories are being reinstalled. If the user elected to install repository dependencies, all items in the all_repo_info_dicts list will be processed. However, if repository dependencies are not to be installed, only those items contained in the received repo_info_dicts list will be processed. """ install_model = self.app.install_model log.debug( "Creating repository dependency objects..." ) # The following list will be maintained within this method to contain all created # or updated tool shed repositories, including repository dependencies that may not # be installed. all_created_or_updated_tool_shed_repositories = [] # There will be a one-to-one mapping between items in 3 lists: # created_or_updated_tool_shed_repositories, tool_panel_section_keys # and filtered_repo_info_dicts. The 3 lists will filter out repository # dependencies that are not to be installed. created_or_updated_tool_shed_repositories = [] tool_panel_section_keys = [] # Repositories will be filtered (e.g., if already installed, if elected # to not be installed, etc), so filter the associated repo_info_dicts accordingly. filtered_repo_info_dicts = [] # Discover all repository dependencies and retrieve information for installing # them. Even if the user elected to not install repository dependencies we have # to make sure all repository dependency objects exist so that the appropriate # repository dependency relationships can be built. all_required_repo_info_dict = self.get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ) all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] ) if not all_repo_info_dicts: # No repository dependencies were discovered so process the received repositories. all_repo_info_dicts = [ rid for rid in repo_info_dicts ] for repo_info_dict in all_repo_info_dicts: # If the user elected to install repository dependencies, all items in the # all_repo_info_dicts list will be processed. However, if repository dependencies # are not to be installed, only those items contained in the received repo_info_dicts # list will be processed but the all_repo_info_dicts list will be used to create all # defined repository dependency relationships. if self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ) or install_repository_dependencies: for name, repo_info_tuple in repo_info_dict.items(): can_update_db_record = False description, \ repository_clone_url, \ changeset_revision, \ ctx_rev, \ repository_owner, \ repository_dependencies, \ tool_dependencies = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) # See if the repository has an existing record in the database. repository_db_record, installed_changeset_revision = \ suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) if repository_db_record: if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.INSTALLED, install_model.ToolShedRepository.installation_status.CLONING, install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES, install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: debug_msg = "Skipping installation of revision %s of repository '%s' because it was installed " % \ ( str( changeset_revision ), str( repository_db_record.name ) ) debug_msg += "with the (possibly updated) revision %s and its current installation status is '%s'." % \ ( str( installed_changeset_revision ), str( repository_db_record.status ) ) log.debug( debug_msg ) can_update_db_record = False else: if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.ERROR, install_model.ToolShedRepository.installation_status.NEW, install_model.ToolShedRepository.installation_status.UNINSTALLED ]: # The current tool shed repository is not currently installed, so we can update its # record in the database. name = repository_db_record.name installed_changeset_revision = repository_db_record.installed_changeset_revision can_update_db_record = True elif repository_db_record.status in [ install_model.ToolShedRepository.installation_status.DEACTIVATED ]: # The current tool shed repository is deactivated, so updating its database record # is not necessary - just activate it. log.debug( "Reactivating deactivated tool_shed_repository '%s'." % str( repository_db_record.name ) ) self.app.installed_repository_manager.activate_repository( repository_db_record ) # No additional updates to the database record are necessary. can_update_db_record = False elif repository_db_record.status not in [ install_model.ToolShedRepository.installation_status.NEW ]: # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find # the previously installed and uninstalled repository instead of creating a new record. changeset_revision = repository_db_record.installed_changeset_revision self.reset_previously_installed_repository( repository_db_record ) can_update_db_record = True else: # No record exists in the database for the repository currently being processed. installed_changeset_revision = changeset_revision can_update_db_record = True if can_update_db_record: # The database record for the tool shed repository currently being processed can be updated. # Get the repository metadata to see where it was previously located in the tool panel. tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository_db_record and repository_db_record.metadata: _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( toolbox=self.app.toolbox, metadata=repository_db_record.metadata, no_changes_checked=no_changes_checked, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) else: # We're installing a new tool shed repository that does not yet have a database record. tool_panel_section_key, _ = \ tpm.handle_tool_panel_section( self.app.toolbox, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) tool_shed_repository = \ suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=installed_changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=install_model.ToolShedRepository.installation_status.NEW, current_changeset_revision=changeset_revision, owner=repository_owner, dist_to_shed=False ) if tool_shed_repository not in all_created_or_updated_tool_shed_repositories: all_created_or_updated_tool_shed_repositories.append( tool_shed_repository ) # Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if # it is supposed to be installed. if install_repository_dependencies or self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ): if tool_shed_repository not in created_or_updated_tool_shed_repositories: # Keep the one-to-one mapping between items in 3 lists. created_or_updated_tool_shed_repositories.append( tool_shed_repository ) tool_panel_section_keys.append( tool_panel_section_key ) filtered_repo_info_dicts.append( repo_info_dict ) # Build repository dependency relationships even if the user chose to not install repository dependencies. self.build_repository_dependency_relationships( all_repo_info_dicts, all_created_or_updated_tool_shed_repositories ) return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts
def install_repository( self, repository_elem, install_dependencies ): # Install a single repository, loading contained tools into the tool panel. name = repository_elem.get( 'name' ) description = repository_elem.get( 'description' ) installed_changeset_revision = repository_elem.get( 'changeset_revision' ) # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> relative_clone_dir = os.path.join( self.tool_shed, 'repos', self.repository_owner, name, installed_changeset_revision ) clone_dir = os.path.join( self.tool_path, relative_clone_dir ) if self.__isinstalled( clone_dir ): print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir else: tool_shed_url = suc.get_url_from_tool_shed( self.app, self.tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) relative_install_dir = os.path.join( relative_clone_dir, name ) install_dir = os.path.join( clone_dir, name ) ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, self.repository_owner, installed_changeset_revision ) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=installed_changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=self.app.model.ToolShedRepository.installation_status.NEW, current_changeset_revision=None, owner=self.repository_owner, dist_to_shed=True ) suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING ) cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: self.handle_repository_contents( tool_shed_repository=tool_shed_repository, repository_clone_url=repository_clone_url, relative_install_dir=relative_install_dir, repository_elem=repository_elem, install_dependencies=install_dependencies ) self.app.sa_session.refresh( tool_shed_repository ) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision ) text = common_util.tool_shed_get( self.app, tool_shed_url, url ) if text: tool_version_dicts = from_json_string( text ) tool_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) else: # Set the tool versions since they seem to be missing for this repository in the tool shed. # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. for tool_dict in metadata_dict[ 'tools' ]: flush_needed = False tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] tool_version = tool_dict[ 'version' ] tool_version_using_old_id = tool_util.get_tool_version( self.app, old_tool_id ) tool_version_using_guid = tool_util.get_tool_version( self.app, tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, tool_shed_repository=tool_shed_repository ) self.app.sa_session.add( tool_version_using_old_id ) self.app.sa_session.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, tool_shed_repository=tool_shed_repository ) self.app.sa_session.add( tool_version_using_guid ) self.app.sa_session.flush() # Associate the two versions as parent / child. tool_version_association = tool_util.get_tool_version_association( self.app, tool_version_using_old_id, tool_version_using_guid ) if not tool_version_association: tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, parent_id=tool_version_using_old_id.id ) self.app.sa_session.add( tool_version_association ) self.app.sa_session.flush() suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED )