def handle_command(self, tool_dependency, cmd, return_output=False, job_name=""):
"""Handle a command and log the results."""
command = str(cmd)
output = self.handle_complex_command(command, job_name=job_name)
self.log_results(cmd, output, os.path.join(self.install_dir, INSTALLATION_LOG))
stdout = output.stdout
stderr = output.stderr
if len(stdout) > DATABASE_MAX_STRING_SIZE:
log.warning("Length of stdout > %s, so only a portion will be saved in the database." % str(DATABASE_MAX_STRING_SIZE_PRETTY))
stdout = shrink_string_by_size(stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
if len(stderr) > DATABASE_MAX_STRING_SIZE:
log.warning("Length of stderr > %s, so only a portion will be saved in the database." % str(DATABASE_MAX_STRING_SIZE_PRETTY))
stderr = shrink_string_by_size(stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
if output.return_code not in [0]:
status = self.app.install_model.ToolDependency.installation_status.ERROR
if stderr:
error_message = unicodify(stderr)
elif stdout:
error_message = unicodify(stdout)
else:
# We have a problem if there was no stdout and no stderr.
error_message = "Unknown error occurred executing shell command %s, return_code: %s" % \
(str(cmd), str(output.return_code))
set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=status,
error_message=error_message)
if return_output:
return output
return output.return_code
def process_tag_set(self,
tool_shed_repository,
tool_dependency,
package_elem,
package_name,
package_version,
from_tool_migration_manager=False,
tool_dependency_db_records=None):
action_elem_tuples = []
proceed_with_install = False
# Only install the tool_dependency if it is not already installed and it is associated with a database
# record in the received tool_dependencies.
if package_name and package_version:
dependencies_ignored = not self.app.toolbox.dependency_manager.uses_tool_shed_dependencies(
)
if dependencies_ignored:
log.debug(
"Skipping installation of tool dependency package %s because tool shed dependency resolver not enabled."
% str(package_name))
# Tool dependency resolves have been configured and they do not include the tool shed. Do not install package.
dep = self.app.toolbox.dependency_manager.find_dep(
package_name, package_version, type='package')
if not isinstance(dep, NullDependency):
# TODO: Do something here such as marking it installed or configured externally.
pass
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.ERROR)
else:
proceed_with_install = True
return tool_dependency, proceed_with_install, action_elem_tuples
def set_environment(self, elem, tool_shed_repository,
attr_tups_of_dependencies_for_install):
"""
Create a ToolDependency to set an environment variable. This is different from the process used to
set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml
file is::
<set_environment version="1.0">
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
</set_environment>
This method must also handle the sub-element tag::
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
"""
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
# the following. See the implementation of support for this in the tool dependency package type's method above.
# This function is only called for set environment actions as defined below, not within an <install version="1.0"> tool
# dependency type. Here is an example of the tag set this function does handle:
# <action type="set_environment">
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable>
# </action>
# Here is an example of the tag set this function does not handle:
# <set_environment version="1.0">
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
# </set_environment>
env_manager = EnvManager(self.app)
tool_dependencies = []
env_var_version = elem.get('version', '1.0')
tool_shed_repository_install_dir = os.path.abspath(
tool_shed_repository.repo_files_directory(self.app))
if elem.tag == 'environment_variable':
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
elems = [elem]
else:
# <set_environment version="1.0">
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
# </set_environment>
elems = [env_var_elem for env_var_elem in elem]
for env_var_elem in elems:
env_var_name = env_var_elem.get('name')
if not env_var_name:
raise Exception(
'The <environment_variable> tag must have a name attribute'
)
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the
# tool config's <requirements> tag set whose "type" attribute is "set_environment" (e.g.,
# <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
env_var_action = env_var_elem.get('action')
if not env_var_action:
raise Exception(
'The <environment_variable> tag must have an action attribute'
)
# Tool dependencies of type "set_environment" always have the version attribute set to None.
attr_tup = (env_var_name, None, 'set_environment')
if attr_tup in attr_tups_of_dependencies_for_install:
install_dir = \
tool_dependency_util.get_tool_dependency_install_dir(app=self.app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='set_environment',
tool_dependency_name=env_var_name,
tool_dependency_version=None)
install_environment = InstallEnvironment(
app=self.app,
tool_shed_repository_install_dir=
tool_shed_repository_install_dir,
install_dir=install_dir)
env_var_dict = env_manager.create_env_var_dict(
elem=env_var_elem, install_environment=install_environment)
if not os.path.exists(install_dir):
os.makedirs(install_dir)
status = self.app.install_model.ToolDependency.installation_status.INSTALLING
tool_dependency = \
tool_dependency_util.create_or_update_tool_dependency(app=self.app,
tool_shed_repository=tool_shed_repository,
name=env_var_name,
version=None,
type='set_environment',
status=status,
set_status=True)
if env_var_version == '1.0':
# Create this tool dependency's env.sh file.
env_file_builder = EnvFileBuilder(install_dir)
return_code = env_file_builder.append_line(
make_executable=True, **env_var_dict)
if return_code:
error_message = 'Error creating env.sh file for tool dependency %s, return_code: %s' % \
(str(tool_dependency.name), str(return_code))
log.debug(error_message)
status = self.app.install_model.ToolDependency.installation_status.ERROR
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=status,
error_message=error_message)
else:
if tool_dependency.status not in [
self.app.install_model.ToolDependency.
installation_status.ERROR,
self.app.install_model.ToolDependency.
installation_status.INSTALLED
]:
status = self.app.install_model.ToolDependency.installation_status.INSTALLED
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=status)
log.debug(
'Environment variable %s set in %s for tool dependency %s.'
% (str(env_var_name), str(install_dir),
str(tool_dependency.name)))
else:
error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).'
status = self.app.install_model.ToolDependency.installation_status.ERROR
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=status,
error_message=error_message)
tool_dependencies.append(tool_dependency)
return tool_dependencies
def create_tool_dependency_with_initialized_env_sh_file(
self, dependent_install_dir, tool_shed_repository,
required_repository, package_name, package_version,
tool_dependencies_config):
"""
Create or get a tool_dependency record that is defined by the received package_name and package_version.
An env.sh file will be created for the tool_dependency in the received dependent_install_dir.
"""
# The received required_repository refers to a tool_shed_repository record that is defined as a complex
# repository dependency for this tool_dependency. The required_repository may or may not be currently
# installed (it doesn't matter). If it is installed, it is associated with a tool_dependency that has
# an env.sh file that this new tool_dependency must be able to locate and "source". If it is not installed,
# we can still determine where that env.sh file will be, so we'll initialize this new tool_dependency's env.sh
# file in either case. If the required repository ends up with an installation error, this new tool
# dependency will still be fine because its containing repository will be defined as missing dependencies.
tool_dependencies = []
if not os.path.exists(dependent_install_dir):
os.makedirs(dependent_install_dir)
required_tool_dependency_env_file_path = None
if tool_dependencies_config:
required_td_tree, error_message = parse_xml(
tool_dependencies_config)
if required_td_tree:
required_td_root = required_td_tree.getroot()
for required_td_elem in required_td_root:
# Find the appropriate package name and version.
if required_td_elem.tag == 'package':
# <package name="bwa" version="0.5.9">
required_td_package_name = required_td_elem.get(
'name', None)
required_td_package_version = required_td_elem.get(
'version', None)
# Check the database to see if we have a record for the required tool dependency (we may not which is ok). If we
# find a record, we need to see if it is in an error state and if so handle it appropriately.
required_tool_dependency = \
tool_dependency_util.get_tool_dependency_by_name_version_type_repository(self.app,
required_repository,
required_td_package_name,
required_td_package_version,
'package')
if required_td_package_name == package_name and required_td_package_version == package_version:
# Get or create a database tool_dependency record with which the installed package on disk will be associated.
tool_dependency = \
tool_dependency_util.create_or_update_tool_dependency(app=self.app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=self.app.install_model.ToolDependency.installation_status.NEVER_INSTALLED,
set_status=True)
# Create an env.sh file for the tool_dependency whose first line will source the env.sh file located in
# the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh
# file currently exists..
required_tool_dependency_env_file_path = \
self.get_required_repository_package_env_sh_path(package_name,
package_version,
required_repository)
env_file_builder = EnvFileBuilder(
tool_dependency.installation_directory(
self.app))
env_file_builder.append_line(
action="source",
value=required_tool_dependency_env_file_path)
return_code = env_file_builder.return_code
if return_code:
error_message = 'Error defining env.sh file for package %s, return_code: %s' % \
(str(package_name), str(return_code))
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.ERROR,
error_message=error_message)
elif required_tool_dependency is not None and required_tool_dependency.in_error_state:
error_message = "This tool dependency's required tool dependency %s version %s has status %s." % \
(str(required_tool_dependency.name), str(required_tool_dependency.version), str(required_tool_dependency.status))
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.ERROR,
error_message=error_message)
else:
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.INSTALLED)
tool_dependencies.append(tool_dependency)
return tool_dependencies
def process_tag_set(self,
tool_shed_repository,
tool_dependency,
package_elem,
package_name,
package_version,
from_tool_migration_manager=False,
tool_dependency_db_records=None):
# <install version="1.0">
# Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository.
actions_elem_tuples = []
proceed_with_install = False
install_dir = \
tool_dependency_util.get_tool_dependency_install_dir(app=self.app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
if os.path.exists(install_dir):
# The tool_migration_manager handles tool migration stages and the sync_database_with_file_system()
# method handles two scenarios: (1) where a Galaxy file system environment related to installed
# Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy
# database tables associated with these installed items, and (2) the Tool Shed's install and test
# framework which installs repositories in 2 stages, those of type tool_dependency_definition
# followed by those containing valid tools and tool functional test components. Neither of these
# scenarios apply when the install manager is running.
if from_tool_migration_manager:
proceed_with_install = True
else:
# Notice that we'll throw away the following tool_dependency if it can be installed.
tool_dependency, proceed_with_install = self.sync_database_with_file_system(
self.app,
tool_shed_repository,
package_name,
package_version,
install_dir,
tool_dependency_type='package')
if not proceed_with_install:
log.debug(
"Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it."
% (str(tool_dependency.name),
str(tool_dependency.version)))
return tool_dependency, proceed_with_install, actions_elem_tuples
else:
proceed_with_install = True
if proceed_with_install:
package_install_version = package_elem.get('version', '1.0')
status = self.app.install_model.ToolDependency.installation_status.INSTALLING
tool_dependency = \
tool_dependency_util.create_or_update_tool_dependency(app=self.app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=status,
set_status=True)
# Get the information about the current platform in case the tool dependency definition includes tag sets
# for installing compiled binaries.
platform_info_dict = tool_dependency_util.get_platform_info_dict()
if package_install_version == '1.0':
# Handle tool dependency installation using a fabric method included in the Galaxy framework.
actions_elem_tuples = tool_dependency_util.parse_package_elem(
package_elem,
platform_info_dict=platform_info_dict,
include_after_install_actions=True)
if not actions_elem_tuples:
proceed_with_install = False
error_message = f'Version {str(package_version)} of the {str(package_name)} package cannot be installed because '
error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> '
error_message += 'tag set.'
# Since there was an installation error, update the tool dependency status to Error.
# The remove_installation_path option must be left False here.
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.
installation_status.ERROR,
error_message=error_message)
else:
raise NotImplementedError(
'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).'
)
return tool_dependency, proceed_with_install, actions_elem_tuples
