def install_repository( self, repository_elem, install_dependencies ): # Install a single repository, loading contained tools into the tool panel. name = repository_elem.get( 'name' ) description = repository_elem.get( 'description' ) installed_changeset_revision = repository_elem.get( 'changeset_revision' ) # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> relative_clone_dir = os.path.join( self.tool_shed, 'repos', self.repository_owner, name, installed_changeset_revision ) clone_dir = os.path.join( self.tool_path, relative_clone_dir ) if self.__isinstalled( clone_dir ): print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir else: tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) relative_install_dir = os.path.join( relative_clone_dir, name ) install_dir = os.path.join( clone_dir, name ) ctx_rev = suc.get_ctx_rev( tool_shed_url, name, self.repository_owner, installed_changeset_revision ) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=installed_changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=self.app.model.ToolShedRepository.installation_status.NEW, current_changeset_revision=None, owner=self.repository_owner, dist_to_shed=True ) shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING ) cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: self.handle_repository_contents( tool_shed_repository=tool_shed_repository, repository_clone_url=repository_clone_url, relative_install_dir=relative_install_dir, repository_elem=repository_elem, install_dependencies=install_dependencies ) self.app.sa_session.refresh( tool_shed_repository ) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision ) response = urllib2.urlopen( url ) text = response.read() response.close() if text: tool_version_dicts = from_json_string( text ) shed_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) else: # Set the tool versions since they seem to be missing for this repository in the tool shed. # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. for tool_dict in metadata_dict[ 'tools' ]: flush_needed = False tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] tool_version = tool_dict[ 'version' ] tool_version_using_old_id = shed_util.get_tool_version( self.app, old_tool_id ) tool_version_using_guid = shed_util.get_tool_version( self.app, tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, tool_shed_repository=tool_shed_repository ) self.app.sa_session.add( tool_version_using_old_id ) self.app.sa_session.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, tool_shed_repository=tool_shed_repository ) self.app.sa_session.add( tool_version_using_guid ) self.app.sa_session.flush() # Associate the two versions as parent / child. tool_version_association = shed_util.get_tool_version_association( self.app, tool_version_using_old_id, tool_version_using_guid ) if not tool_version_association: tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, parent_id=tool_version_using_old_id.id ) self.app.sa_session.add( tool_version_association ) self.app.sa_session.flush() shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED )
def install_repository(self, repository_elem, install_dependencies): # Install a single repository, loading contained tools into the tool panel. name = repository_elem.get('name') description = repository_elem.get('description') installed_changeset_revision = repository_elem.get( 'changeset_revision') # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> relative_clone_dir = os.path.join(self.tool_shed, 'repos', self.repository_owner, name, installed_changeset_revision) clone_dir = os.path.join(self.tool_path, relative_clone_dir) if self.__isinstalled(clone_dir): print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir else: tool_shed_url = self.__get_url_from_tool_shed(self.tool_shed) repository_clone_url = os.path.join(tool_shed_url, 'repos', self.repository_owner, name) relative_install_dir = os.path.join(relative_clone_dir, name) install_dir = os.path.join(clone_dir, name) ctx_rev = suc.get_ctx_rev(tool_shed_url, name, self.repository_owner, installed_changeset_revision) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, installed_changeset_revision=installed_changeset_revision, ctx_rev=ctx_rev, repository_clone_url=repository_clone_url, metadata_dict={}, status=self.app.model.ToolShedRepository.installation_status. NEW, current_changeset_revision=None, owner=self.repository_owner, dist_to_shed=True) shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING) cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath(install_dir), ctx_rev) if cloned_ok: self.handle_repository_contents( tool_shed_repository=tool_shed_repository, repository_clone_url=repository_clone_url, relative_install_dir=relative_install_dir, repository_elem=repository_elem, install_dependencies=install_dependencies) self.app.sa_session.refresh(tool_shed_repository) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status. SETTING_TOOL_VERSIONS) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision ) response = urllib2.urlopen(url) text = response.read() response.close() if text: tool_version_dicts = from_json_string(text) shed_util.handle_tool_versions(self.app, tool_version_dicts, tool_shed_repository) else: # Set the tool versions since they seem to be missing for this repository in the tool shed. # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. for tool_dict in metadata_dict['tools']: flush_needed = False tool_id = tool_dict['guid'] old_tool_id = tool_dict['id'] tool_version = tool_dict['version'] tool_version_using_old_id = shed_util.get_tool_version( self.app, old_tool_id) tool_version_using_guid = shed_util.get_tool_version( self.app, tool_id) if not tool_version_using_old_id: tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, tool_shed_repository=tool_shed_repository) self.app.sa_session.add( tool_version_using_old_id) self.app.sa_session.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, tool_shed_repository=tool_shed_repository) self.app.sa_session.add( tool_version_using_guid) self.app.sa_session.flush() # Associate the two versions as parent / child. tool_version_association = shed_util.get_tool_version_association( self.app, tool_version_using_old_id, tool_version_using_guid) if not tool_version_association: tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, parent_id=tool_version_using_old_id.id) self.app.sa_session.add( tool_version_association) self.app.sa_session.flush() shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model. ToolShedRepository.installation_status.INSTALLED)
def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies ): """Generate the metadata for the installed tool shed repository, among other things.""" tool_panel_dict_for_display = odict() if self.tool_path: repo_install_dir = os.path.join( self.tool_path, relative_install_dir ) else: repo_install_dir = relative_install_dir for tool_elem in repository_elem: # The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> tool_config = tool_elem.get( 'file' ) guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config ) # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) if is_displayed: tool_panel_dict_for_tool_config = shed_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=self.app, repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, shed_config_dict = self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=True ) tool_shed_repository.metadata = metadata_dict self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() if 'tool_dependencies' in metadata_dict: # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. tool_dependencies = shed_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True ) else: tool_dependencies = None if 'tools' in metadata_dict: sample_files = metadata_dict.get( 'sample_files', [] ) sample_files = [ str( s ) for s in sample_files ] tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) sample_files_copied = [ s for s in tool_index_sample_files ] repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = shed_util.handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups, sample_files_copied ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. shed_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied ) if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict: # Install tool dependencies. shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) installed_tool_dependencies = shed_util.handle_tool_dependencies( app=self.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.model.ToolDependency.installation_status.ERROR: print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' shed_util.add_to_tool_panel( self.app, tool_shed_repository.name, repository_clone_url, tool_shed_repository.installed_changeset_revision, repository_tools_tups, self.repository_owner, self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True ) if 'datatypes' in metadata_dict: tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() work_dir = tempfile.mkdtemp() datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', repo_install_dir ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, name=tool_shed_repository.name, owner=self.repository_owner, installed_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dicts=metadata_dict.get( 'tools', [] ), converter_path=converter_path, display_path=display_path ) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) try: shutil.rmtree( work_dir ) except: pass
def handle_repository_contents(self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies): """Generate the metadata for the installed tool shed repository, among other things.""" tool_panel_dict_for_display = odict() if self.tool_path: repo_install_dir = os.path.join(self.tool_path, relative_install_dir) else: repo_install_dir = relative_install_dir for tool_elem in repository_elem: # The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> tool_config = tool_elem.get('file') guid = self.get_guid(repository_clone_url, relative_install_dir, tool_config) # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. is_displayed, tool_sections = self.get_containing_tool_sections( tool_config) if is_displayed: tool_panel_dict_for_tool_config = shed_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections) for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[k] = v else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=self.app, repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, shed_config_dict=self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=True) tool_shed_repository.metadata = metadata_dict self.app.sa_session.add(tool_shed_repository) self.app.sa_session.flush() if 'tool_dependencies' in metadata_dict: # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. tool_dependencies = shed_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True) else: tool_dependencies = None if 'tools' in metadata_dict: sample_files = metadata_dict.get('sample_files', []) sample_files = [str(s) for s in sample_files] tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files) shed_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=self.tool_path) sample_files_copied = [s for s in tool_index_sample_files] repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = shed_util.handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups, sample_files_copied) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. shed_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied) if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict: # Install tool dependencies. shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status. INSTALLING_TOOL_DEPENDENCIES) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir) installed_tool_dependencies = shed_util.handle_tool_dependencies( app=self.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.model.ToolDependency.installation_status.ERROR: print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' shed_util.add_to_tool_panel( self.app, tool_shed_repository.name, repository_clone_url, tool_shed_repository.installed_changeset_revision, repository_tools_tups, self.repository_owner, self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True) if 'datatypes' in metadata_dict: tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True self.app.sa_session.add(tool_shed_repository) self.app.sa_session.flush() work_dir = tempfile.mkdtemp() datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', repo_install_dir) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False) #repo_install_dir was relative_install_dir if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, name=tool_shed_repository.name, owner=self.repository_owner, installed_changeset_revision=tool_shed_repository. installed_changeset_revision, tool_dicts=metadata_dict.get('tools', []), converter_path=converter_path, display_path=display_path) if converter_path: # Load proprietary datatype converters self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict) if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict) try: shutil.rmtree(work_dir) except: pass