def handle_command(self, tool_dependency, cmd, return_output=False, job_name=""): """Handle a command and log the results.""" command = str(cmd) output = self.handle_complex_command(command, job_name=job_name) self.log_results(cmd, output, os.path.join(self.install_dir, INSTALLATION_LOG)) stdout = output.stdout stderr = output.stderr if len(stdout) > DATABASE_MAX_STRING_SIZE: log.warning("Length of stdout > %s, so only a portion will be saved in the database." % str(DATABASE_MAX_STRING_SIZE_PRETTY)) stdout = shrink_string_by_size(stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) if len(stderr) > DATABASE_MAX_STRING_SIZE: log.warning("Length of stderr > %s, so only a portion will be saved in the database." % str(DATABASE_MAX_STRING_SIZE_PRETTY)) stderr = shrink_string_by_size(stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) if output.return_code not in [0]: status = self.app.install_model.ToolDependency.installation_status.ERROR if stderr: error_message = unicodify(stderr) elif stdout: error_message = unicodify(stdout) else: # We have a problem if there was no stdout and no stderr. error_message = "Unknown error occurred executing shell command %s, return_code: %s" % \ (str(cmd), str(output.return_code)) set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=status, error_message=error_message) if return_output: return output return output.return_code
def process_tag_set(self, tool_shed_repository, tool_dependency, package_elem, package_name, package_version, from_tool_migration_manager=False, tool_dependency_db_records=None): action_elem_tuples = [] proceed_with_install = False # Only install the tool_dependency if it is not already installed and it is associated with a database # record in the received tool_dependencies. if package_name and package_version: dependencies_ignored = not self.app.toolbox.dependency_manager.uses_tool_shed_dependencies( ) if dependencies_ignored: log.debug( "Skipping installation of tool dependency package %s because tool shed dependency resolver not enabled." % str(package_name)) # Tool dependency resolves have been configured and they do not include the tool shed. Do not install package. dep = self.app.toolbox.dependency_manager.find_dep( package_name, package_version, type='package') if not isinstance(dep, NullDependency): # TODO: Do something here such as marking it installed or configured externally. pass tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.ERROR) else: proceed_with_install = True return tool_dependency, proceed_with_install, action_elem_tuples
def set_environment(self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install): """ Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml file is:: <set_environment version="1.0"> <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> </set_environment> This method must also handle the sub-element tag:: <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> """ # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like # the following. See the implementation of support for this in the tool dependency package type's method above. # This function is only called for set environment actions as defined below, not within an <install version="1.0"> tool # dependency type. Here is an example of the tag set this function does handle: # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable> # </action> # Here is an example of the tag set this function does not handle: # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> env_manager = EnvManager(self.app) tool_dependencies = [] env_var_version = elem.get('version', '1.0') tool_shed_repository_install_dir = os.path.abspath( tool_shed_repository.repo_files_directory(self.app)) if elem.tag == 'environment_variable': # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> elems = [elem] else: # <set_environment version="1.0"> # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> # </set_environment> elems = [env_var_elem for env_var_elem in elem] for env_var_elem in elems: env_var_name = env_var_elem.get('name') if not env_var_name: raise Exception( 'The <environment_variable> tag must have a name attribute' ) # The value of env_var_name must match the text value of at least 1 <requirement> tag in the # tool config's <requirements> tag set whose "type" attribute is "set_environment" (e.g., # <requirement type="set_environment">R_SCRIPT_PATH</requirement>). env_var_action = env_var_elem.get('action') if not env_var_action: raise Exception( 'The <environment_variable> tag must have an action attribute' ) # Tool dependencies of type "set_environment" always have the version attribute set to None. attr_tup = (env_var_name, None, 'set_environment') if attr_tup in attr_tups_of_dependencies_for_install: install_dir = \ tool_dependency_util.get_tool_dependency_install_dir(app=self.app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='set_environment', tool_dependency_name=env_var_name, tool_dependency_version=None) install_environment = InstallEnvironment( app=self.app, tool_shed_repository_install_dir= tool_shed_repository_install_dir, install_dir=install_dir) env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem, install_environment=install_environment) if not os.path.exists(install_dir): os.makedirs(install_dir) status = self.app.install_model.ToolDependency.installation_status.INSTALLING tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency(app=self.app, tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type='set_environment', status=status, set_status=True) if env_var_version == '1.0': # Create this tool dependency's env.sh file. env_file_builder = EnvFileBuilder(install_dir) return_code = env_file_builder.append_line( make_executable=True, **env_var_dict) if return_code: error_message = 'Error creating env.sh file for tool dependency %s, return_code: %s' % \ (str(tool_dependency.name), str(return_code)) log.debug(error_message) status = self.app.install_model.ToolDependency.installation_status.ERROR tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=status, error_message=error_message) else: if tool_dependency.status not in [ self.app.install_model.ToolDependency. installation_status.ERROR, self.app.install_model.ToolDependency. installation_status.INSTALLED ]: status = self.app.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=status) log.debug( 'Environment variable %s set in %s for tool dependency %s.' % (str(env_var_name), str(install_dir), str(tool_dependency.name))) else: error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' status = self.app.install_model.ToolDependency.installation_status.ERROR tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=status, error_message=error_message) tool_dependencies.append(tool_dependency) return tool_dependencies
def create_tool_dependency_with_initialized_env_sh_file( self, dependent_install_dir, tool_shed_repository, required_repository, package_name, package_version, tool_dependencies_config): """ Create or get a tool_dependency record that is defined by the received package_name and package_version. An env.sh file will be created for the tool_dependency in the received dependent_install_dir. """ # The received required_repository refers to a tool_shed_repository record that is defined as a complex # repository dependency for this tool_dependency. The required_repository may or may not be currently # installed (it doesn't matter). If it is installed, it is associated with a tool_dependency that has # an env.sh file that this new tool_dependency must be able to locate and "source". If it is not installed, # we can still determine where that env.sh file will be, so we'll initialize this new tool_dependency's env.sh # file in either case. If the required repository ends up with an installation error, this new tool # dependency will still be fine because its containing repository will be defined as missing dependencies. tool_dependencies = [] if not os.path.exists(dependent_install_dir): os.makedirs(dependent_install_dir) required_tool_dependency_env_file_path = None if tool_dependencies_config: required_td_tree, error_message = parse_xml( tool_dependencies_config) if required_td_tree: required_td_root = required_td_tree.getroot() for required_td_elem in required_td_root: # Find the appropriate package name and version. if required_td_elem.tag == 'package': # <package name="bwa" version="0.5.9"> required_td_package_name = required_td_elem.get( 'name', None) required_td_package_version = required_td_elem.get( 'version', None) # Check the database to see if we have a record for the required tool dependency (we may not which is ok). If we # find a record, we need to see if it is in an error state and if so handle it appropriately. required_tool_dependency = \ tool_dependency_util.get_tool_dependency_by_name_version_type_repository(self.app, required_repository, required_td_package_name, required_td_package_version, 'package') if required_td_package_name == package_name and required_td_package_version == package_version: # Get or create a database tool_dependency record with which the installed package on disk will be associated. tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency(app=self.app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=self.app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, set_status=True) # Create an env.sh file for the tool_dependency whose first line will source the env.sh file located in # the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh # file currently exists.. required_tool_dependency_env_file_path = \ self.get_required_repository_package_env_sh_path(package_name, package_version, required_repository) env_file_builder = EnvFileBuilder( tool_dependency.installation_directory( self.app)) env_file_builder.append_line( action="source", value=required_tool_dependency_env_file_path) return_code = env_file_builder.return_code if return_code: error_message = 'Error defining env.sh file for package %s, return_code: %s' % \ (str(package_name), str(return_code)) tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.ERROR, error_message=error_message) elif required_tool_dependency is not None and required_tool_dependency.in_error_state: error_message = "This tool dependency's required tool dependency %s version %s has status %s." % \ (str(required_tool_dependency.name), str(required_tool_dependency.version), str(required_tool_dependency.status)) tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.ERROR, error_message=error_message) else: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes(self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency.installation_status.INSTALLED) tool_dependencies.append(tool_dependency) return tool_dependencies
def process_tag_set(self, tool_shed_repository, tool_dependency, package_elem, package_name, package_version, from_tool_migration_manager=False, tool_dependency_db_records=None): # <install version="1.0"> # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. actions_elem_tuples = [] proceed_with_install = False install_dir = \ tool_dependency_util.get_tool_dependency_install_dir(app=self.app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type='package', tool_dependency_name=package_name, tool_dependency_version=package_version) if os.path.exists(install_dir): # The tool_migration_manager handles tool migration stages and the sync_database_with_file_system() # method handles two scenarios: (1) where a Galaxy file system environment related to installed # Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy # database tables associated with these installed items, and (2) the Tool Shed's install and test # framework which installs repositories in 2 stages, those of type tool_dependency_definition # followed by those containing valid tools and tool functional test components. Neither of these # scenarios apply when the install manager is running. if from_tool_migration_manager: proceed_with_install = True else: # Notice that we'll throw away the following tool_dependency if it can be installed. tool_dependency, proceed_with_install = self.sync_database_with_file_system( self.app, tool_shed_repository, package_name, package_version, install_dir, tool_dependency_type='package') if not proceed_with_install: log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it." % (str(tool_dependency.name), str(tool_dependency.version))) return tool_dependency, proceed_with_install, actions_elem_tuples else: proceed_with_install = True if proceed_with_install: package_install_version = package_elem.get('version', '1.0') status = self.app.install_model.ToolDependency.installation_status.INSTALLING tool_dependency = \ tool_dependency_util.create_or_update_tool_dependency(app=self.app, tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', status=status, set_status=True) # Get the information about the current platform in case the tool dependency definition includes tag sets # for installing compiled binaries. platform_info_dict = tool_dependency_util.get_platform_info_dict() if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. actions_elem_tuples = tool_dependency_util.parse_package_elem( package_elem, platform_info_dict=platform_info_dict, include_after_install_actions=True) if not actions_elem_tuples: proceed_with_install = False error_message = f'Version {str(package_version)} of the {str(package_name)} package cannot be installed because ' error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> ' error_message += 'tag set.' # Since there was an installation error, update the tool dependency status to Error. # The remove_installation_path option must be left False here. tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=self.app.install_model.ToolDependency. installation_status.ERROR, error_message=error_message) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' ) return tool_dependency, proceed_with_install, actions_elem_tuples