def set_environment( app, elem, tool_shed_repository ):
"""
Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated
with a package. An example entry in a tool_dependencies.xml file is::
<set_environment version="1.0">
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
</set_environment>
"""
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
# the following. See the implementation of support for this in the tool dependency package type's method above.
# <set_environment version="1.0">
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
# </set_environment>
sa_session = app.model.context.current
tool_dependency = None
env_var_version = elem.get( 'version', '1.0' )
for env_var_elem in elem:
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose
# "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
env_var_name = env_var_elem.get( 'name', None )
env_var_action = env_var_elem.get( 'action', None )
if env_var_name and env_var_action:
install_dir = get_tool_dependency_install_dir( app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='set_environment',
tool_dependency_name=env_var_name,
tool_dependency_version=None )
tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository )
env_var_dict = common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir )
if env_var_dict:
if not os.path.exists( install_dir ):
os.makedirs( install_dir )
tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app,
tool_shed_repository=tool_shed_repository,
name=env_var_name,
version=None,
type='set_environment',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True )
cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict )
if env_var_version == '1.0':
# Handle setting environment variables using a fabric method.
fabric_util.handle_command( app, tool_dependency, install_dir, cmd )
sa_session.refresh( tool_dependency )
if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
sa_session.add( tool_dependency )
sa_session.flush()
print 'Environment variable ', env_var_name, 'set in', install_dir
else:
raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' )
def install_package( app, elem, tool_shed_repository, tool_dependencies=None ):
# The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository.
sa_session = app.model.context.current
tool_dependency = None
# The value of package_name should match the value of the "package" type in the tool config's <requirements> tag set, but it's not required.
package_name = elem.get( 'name', None )
package_version = elem.get( 'version', None )
if tool_dependencies and package_name and package_version:
for package_elem in elem:
if package_elem.tag == 'repository':
# We have a complex repository dependency definition.
tool_shed = package_elem.attrib[ 'toolshed' ]
required_repository_name = package_elem.attrib[ 'name' ]
required_repository_owner = package_elem.attrib[ 'owner' ]
default_required_repository_changeset_revision = package_elem.attrib[ 'changeset_revision' ]
required_repository = get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( app,
tool_shed,
required_repository_name,
required_repository_owner,
default_required_repository_changeset_revision )
tmp_filename = None
if required_repository:
required_repository_changeset_revision = required_repository.installed_changeset_revision
# Define the installation directory for the required tool dependency package in the required repository.
required_repository_package_install_dir = \
get_tool_dependency_install_dir( app=app,
repository_name=required_repository_name,
repository_owner=required_repository_owner,
repository_changeset_revision=required_repository_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
# Define the this dependent repository's tool dependency installation directory that will contain the env.sh file with a path to the
# required repository's installed tool dependency package.
dependent_install_dir = get_tool_dependency_install_dir( app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
# Set this dependent repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package.
# We can get everything we need from the discovered installed required_repository.
if required_repository.status in [ app.model.ToolShedRepository.installation_status.DEACTIVATED,
app.model.ToolShedRepository.installation_status.INSTALLED ]:
if not os.path.exists( required_repository_package_install_dir ):
print 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir )
repo_files_dir = required_repository.repo_files_directory( app )
tool_dependencies_config = get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' )
if tool_dependencies_config:
config_to_use = tool_dependencies_config
else:
message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \
( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) )
raise Exception( message )
else:
# Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision
# should be updated if it's not current.
text = get_updated_changeset_revisions_from_tool_shed( app=app,
tool_shed_url=tool_shed,
name=required_repository_name,
owner=required_repository_owner,
changeset_revision=required_repository_changeset_revision )
if text:
updated_changeset_revisions = listify( text )
# The list of changeset revisions is in reverse order, so the newest will be first.
required_repository_changeset_revision = updated_changeset_revisions[ 0 ]
# Make a call to the tool shed to get the required repository's tool_dependencies.xml file.
tmp_filename = create_temporary_tool_dependencies_config( app,
tool_shed,
required_repository_name,
required_repository_owner,
required_repository_changeset_revision )
config_to_use = tmp_filename
tool_dependency, actions_dict = populate_actions_dict( app=app,
dependent_install_dir=dependent_install_dir,
required_install_dir=required_repository_package_install_dir,
tool_shed_repository=tool_shed_repository,
required_repository=required_repository,
package_name=package_name,
package_version=package_version,
tool_dependencies_config=config_to_use )
if tmp_filename:
try:
os.remove( tmp_filename )
except:
pass
# Install and build the package via fabric.
install_and_build_package_via_fabric( app, tool_dependency, actions_dict )
else:
message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \
( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) )
raise Exception( message )
elif package_elem.tag == 'install':
# <install version="1.0">
# Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository.
install_dir = get_tool_dependency_install_dir( app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
if os.path.exists( install_dir ):
print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n'
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( app,
tool_shed_repository,
package_name,
package_version,
'package' )
tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
sa_session.add( tool_dependency )
sa_session.flush()
else:
package_install_version = package_elem.get( 'version', '1.0' )
tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True )
if package_install_version == '1.0':
# Handle tool dependency installation using a fabric method included in the Galaxy framework.
for actions_elem in package_elem:
install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=package_name )
sa_session.refresh( tool_dependency )
if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
print package_name, 'version', package_version, 'installed in', install_dir
else:
raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
elif package_elem.tag == 'readme':
# Nothing to be done.
continue
#elif package_elem.tag == 'proprietary_fabfile':
# # TODO: This is not yet supported or functionally correct...
# # Handle tool dependency installation where the repository includes one or more proprietary fabric scripts.
# if not fabric_version_checked:
# check_fabric_version()
# fabric_version_checked = True
# fabfile_name = package_elem.get( 'name', None )
# proprietary_fabfile_path = os.path.abspath( os.path.join( os.path.split( tool_dependencies_config )[ 0 ], fabfile_name ) )
# print 'Installing tool dependencies via fabric script ', proprietary_fabfile_path
return tool_dependency
def set_environment(app, elem, tool_shed_repository):
"""
Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated
with a package. An example entry in a tool_dependencies.xml file is::
<set_environment version="1.0">
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
</set_environment>
"""
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
# the following. See the implementation of support for this in the tool dependency package type's method above.
# <set_environment version="1.0">
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
# </set_environment>
sa_session = app.model.context.current
tool_dependency = None
env_var_version = elem.get('version', '1.0')
for env_var_elem in elem:
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose
# "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
env_var_name = env_var_elem.get('name', None)
env_var_action = env_var_elem.get('action', None)
if env_var_name and env_var_action:
install_dir = get_tool_dependency_install_dir(
app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.
installed_changeset_revision,
tool_dependency_type='set_environment',
tool_dependency_name=env_var_name,
tool_dependency_version=None)
tool_shed_repository_install_dir = get_tool_shed_repository_install_dir(
app, tool_shed_repository)
env_var_dict = common_util.create_env_var_dict(
env_var_elem,
tool_shed_repository_install_dir=
tool_shed_repository_install_dir)
if env_var_dict:
if not os.path.exists(install_dir):
os.makedirs(install_dir)
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=env_var_name,
version=None,
type='set_environment',
status=app.model.ToolDependency.installation_status.
INSTALLING,
set_status=True)
cmd = common_util.create_or_update_env_shell_file(
install_dir, env_var_dict)
if env_var_version == '1.0':
# Handle setting environment variables using a fabric method.
fabric_util.handle_command(app, tool_dependency,
install_dir, cmd)
sa_session.refresh(tool_dependency)
if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
sa_session.add(tool_dependency)
sa_session.flush()
print 'Environment variable ', env_var_name, 'set in', install_dir
else:
raise NotImplementedError(
'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).'
)
def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository ):
"""
Populate a list of actions for creating an env.sh file for a dependent repository. The received elem is the <package> tag set associated
with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency.
"""
action_dict = {}
actions = []
for package_elem in elem:
if package_elem.tag == 'install':
# Create the tool_dependency record in the database.
tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True )
# Get the installation method version from a tag like: <install version="1.0">
package_install_version = package_elem.get( 'version', '1.0' )
if package_install_version == '1.0':
# Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find
# the <action type="set_environment"> tag.
env_var_dicts = []
for actions_elem in package_elem:
for action_elem in actions_elem:
action_type = action_elem.get( 'type', 'shell_command' )
if action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir )
if env_var_dict:
if env_var_dict not in env_var_dicts:
env_var_dicts.append( env_var_dict )
elif action_type == 'setup_virtualenv':
# Add the virtualenv's site-packages to PYTHONPATH and bin to PATH. This is a bit hackish.
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python" )
output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True )
if output.return_code:
log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr )
elif not os.path.exists( output.stdout ):
log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout )
else:
env_var_dicts.append( dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) )
env_var_dicts.append( dict( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin' ) ) )
if env_var_dicts:
if required_repository.status in [ app.model.ToolShedRepository.installation_status.INSTALLED,
app.model.ToolShedRepository.installation_status.DEACTIVATED ]:
# Handle the case where we have an installed required repository due to the prior_installation_required = True
# setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's
# tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this,
# we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh
# file generated for the installed required repository. Each env_var_dict currently looks something like this:
# {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'}
# We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict
# with the associated value in the env.sh file.
new_env_var_dicts = []
env_sh_file_dir = get_tool_dependency_install_dir( app=app,
repository_name=required_repository.name,
repository_owner=required_repository.owner,
repository_changeset_revision=required_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' )
if os.path.exists( env_sh_file_path ):
for i, line in enumerate( open( env_sh_file_path, 'r' ) ):
env_var_dict = env_var_dicts[ i ]
action = env_var_dict.get( 'action', None )
name = env_var_dict.get( 'name', None )
value = env_var_dict.get( 'value', None )
if action and name and value:
new_value = parse_env_shell_entry( action, name, value, line )
env_var_dict[ 'value' ] = new_value
new_env_var_dicts.append( env_var_dict )
else:
log.debug( 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path )
action_dict[ 'environment_variable' ] = new_env_var_dicts
else:
action_dict[ 'environment_variable' ] = env_var_dicts
actions.append( ( 'set_environment', action_dict ) )
else:
raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
return tool_dependency, actions
return None, actions
def install_package(app, elem, tool_shed_repository, tool_dependencies=None):
# The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository.
sa_session = app.model.context.current
tool_dependency = None
# The value of package_name should match the value of the "package" type in the tool config's <requirements> tag set, but it's not required.
package_name = elem.get('name', None)
package_version = elem.get('version', None)
if tool_dependencies and package_name and package_version:
for package_elem in elem:
if package_elem.tag == 'repository':
# We have a complex repository dependency definition.
tool_shed = package_elem.attrib['toolshed']
required_repository_name = package_elem.attrib['name']
required_repository_owner = package_elem.attrib['owner']
default_required_repository_changeset_revision = package_elem.attrib[
'changeset_revision']
required_repository = get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision(
app, tool_shed, required_repository_name,
required_repository_owner,
default_required_repository_changeset_revision)
tmp_filename = None
if required_repository:
required_repository_changeset_revision = required_repository.installed_changeset_revision
# Define the installation directory for the required tool dependency package in the required repository.
required_repository_package_install_dir = \
get_tool_dependency_install_dir( app=app,
repository_name=required_repository_name,
repository_owner=required_repository_owner,
repository_changeset_revision=required_repository_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
# Define the this dependent repository's tool dependency installation directory that will contain the env.sh file with a path to the
# required repository's installed tool dependency package.
dependent_install_dir = get_tool_dependency_install_dir(
app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.
installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
# Set this dependent repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package.
# We can get everything we need from the discovered installed required_repository.
if required_repository.status in [
app.model.ToolShedRepository.installation_status.
DEACTIVATED, app.model.ToolShedRepository.
installation_status.INSTALLED
]:
if not os.path.exists(
required_repository_package_install_dir):
print 'Missing required tool dependency directory %s' % str(
required_repository_package_install_dir)
repo_files_dir = required_repository.repo_files_directory(
app)
tool_dependencies_config = get_absolute_path_to_file_in_repository(
repo_files_dir, 'tool_dependencies.xml')
if tool_dependencies_config:
config_to_use = tool_dependencies_config
else:
message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \
( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) )
raise Exception(message)
else:
# Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision
# should be updated if it's not current.
text = get_updated_changeset_revisions_from_tool_shed(
app=app,
tool_shed_url=tool_shed,
name=required_repository_name,
owner=required_repository_owner,
changeset_revision=
required_repository_changeset_revision)
if text:
updated_changeset_revisions = listify(text)
# The list of changeset revisions is in reverse order, so the newest will be first.
required_repository_changeset_revision = updated_changeset_revisions[
0]
# Make a call to the tool shed to get the required repository's tool_dependencies.xml file.
tmp_filename = create_temporary_tool_dependencies_config(
app, tool_shed, required_repository_name,
required_repository_owner,
required_repository_changeset_revision)
config_to_use = tmp_filename
tool_dependency, actions_dict = populate_actions_dict(
app=app,
dependent_install_dir=dependent_install_dir,
required_install_dir=
required_repository_package_install_dir,
tool_shed_repository=tool_shed_repository,
required_repository=required_repository,
package_name=package_name,
package_version=package_version,
tool_dependencies_config=config_to_use)
if tmp_filename:
try:
os.remove(tmp_filename)
except:
pass
# Install and build the package via fabric.
install_and_build_package_via_fabric(
app, tool_dependency, actions_dict)
else:
message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \
( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) )
raise Exception(message)
elif package_elem.tag == 'install':
# <install version="1.0">
# Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository.
install_dir = get_tool_dependency_install_dir(
app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.
installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
if os.path.exists(install_dir):
print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n'
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository(
app, tool_shed_repository, package_name,
package_version, 'package')
tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
sa_session.add(tool_dependency)
sa_session.flush()
else:
package_install_version = package_elem.get(
'version', '1.0')
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.
INSTALLING,
set_status=True)
if package_install_version == '1.0':
# Handle tool dependency installation using a fabric method included in the Galaxy framework.
for actions_elem in package_elem:
install_via_fabric(app,
tool_dependency,
actions_elem,
install_dir,
package_name=package_name)
sa_session.refresh(tool_dependency)
if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
print package_name, 'version', package_version, 'installed in', install_dir
else:
raise NotImplementedError(
'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).'
)
elif package_elem.tag == 'readme':
# Nothing to be done.
continue
#elif package_elem.tag == 'proprietary_fabfile':
# # TODO: This is not yet supported or functionally correct...
# # Handle tool dependency installation where the repository includes one or more proprietary fabric scripts.
# if not fabric_version_checked:
# check_fabric_version()
# fabric_version_checked = True
# fabfile_name = package_elem.get( 'name', None )
# proprietary_fabfile_path = os.path.abspath( os.path.join( os.path.split( tool_dependencies_config )[ 0 ], fabfile_name ) )
# print 'Installing tool dependencies via fabric script ', proprietary_fabfile_path
return tool_dependency
def handle_set_environment_entry_for_package(app, install_dir,
tool_shed_repository,
package_name, package_version,
elem, required_repository):
"""
Populate a list of actions for creating an env.sh file for a dependent repository. The received elem is the <package> tag set associated
with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency.
"""
action_dict = {}
actions = []
for package_elem in elem:
if package_elem.tag == 'install':
# Create the tool_dependency record in the database.
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True)
# Get the installation method version from a tag like: <install version="1.0">
package_install_version = package_elem.get('version', '1.0')
if package_install_version == '1.0':
# Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find
# the <action type="set_environment"> tag.
env_var_dicts = []
for actions_elem in package_elem:
for action_elem in actions_elem:
action_type = action_elem.get('type', 'shell_command')
if action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict(
env_elem,
tool_dependency_install_dir=install_dir
)
if env_var_dict:
if env_var_dict not in env_var_dicts:
env_var_dicts.append(env_var_dict)
elif action_type == 'setup_virtualenv':
# Add the virtualenv's site-packages to PYTHONPATH and bin to PATH. This is a bit hackish.
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join(
install_dir, "venv", "bin", "python")
output = fabric_util.handle_command(
app,
tool_dependency,
install_dir,
site_packages_command,
return_output=True)
if output.return_code:
log.error(
'Dependency includes a setup_virtualenv action but venv python is broken:',
output.stderr)
elif not os.path.exists(output.stdout):
log.error(
"virtualenv's site-packages directory '%s' does not exist",
output.stdout)
else:
env_var_dicts.append(
dict(name="PYTHONPATH",
action="prepend_to",
value=output.stdout))
env_var_dicts.append(
dict(name="PATH",
action="prepend_to",
value=os.path.join(
install_dir, 'venv', 'bin')))
if env_var_dicts:
if required_repository.status in [
app.model.ToolShedRepository.installation_status.
INSTALLED, app.model.ToolShedRepository.
installation_status.DEACTIVATED
]:
# Handle the case where we have an installed required repository due to the prior_installation_required = True
# setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's
# tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this,
# we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh
# file generated for the installed required repository. Each env_var_dict currently looks something like this:
# {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'}
# We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict
# with the associated value in the env.sh file.
new_env_var_dicts = []
env_sh_file_dir = get_tool_dependency_install_dir(
app=app,
repository_name=required_repository.name,
repository_owner=required_repository.owner,
repository_changeset_revision=required_repository.
installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
env_sh_file_path = os.path.join(
env_sh_file_dir, 'env.sh')
if os.path.exists(env_sh_file_path):
for i, line in enumerate(
open(env_sh_file_path, 'r')):
env_var_dict = env_var_dicts[i]
action = env_var_dict.get('action', None)
name = env_var_dict.get('name', None)
value = env_var_dict.get('value', None)
if action and name and value:
new_value = parse_env_shell_entry(
action, name, value, line)
env_var_dict['value'] = new_value
new_env_var_dicts.append(env_var_dict)
else:
log.debug(
'Invalid file %s specified, ignoring set_environment_for_install action.',
env_sh_file_path)
action_dict['environment_variable'] = new_env_var_dicts
else:
action_dict['environment_variable'] = env_var_dicts
actions.append(('set_environment', action_dict))
else:
raise NotImplementedError(
'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).'
)
return tool_dependency, actions
return None, actions
def set_environment( self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install ):
"""
Create a ToolDependency to set an environment variable. This is different from the process used to
set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml
file is::
<set_environment version="1.0">
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
</set_environment>
This method must also handle the sub-element tag::
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
"""
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
# the following. See the implementation of support for this in the tool dependency package type's method above.
# This function is only called for set environment actions as defined below, not within an <install version="1.0"> tool
# dependency type. Here is an example of the tag set this function does handle:
# <action type="set_environment">
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable>
# </action>
# Here is an example of the tag set this function does not handle:
# <set_environment version="1.0">
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
# </set_environment>
env_manager = EnvManager( self.app )
tool_dependencies = []
env_var_version = elem.get( 'version', '1.0' )
tool_shed_repository_install_dir = os.path.abspath( tool_shed_repository.repo_files_directory( self.app ) )
if elem.tag == 'environment_variable':
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
elems = [ elem ]
else:
# <set_environment version="1.0">
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
# </set_environment>
elems = [ env_var_elem for env_var_elem in elem ]
for env_var_elem in elems:
env_var_name = env_var_elem.get( 'name', None )
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the
# tool config's <requirements> tag set whose "type" attribute is "set_environment" (e.g.,
# <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
env_var_action = env_var_elem.get( 'action', None )
if env_var_name and env_var_action:
# Tool dependencies of type "set_environment" always have the version attribute set to None.
attr_tup = ( env_var_name, None, 'set_environment' )
if attr_tup in attr_tups_of_dependencies_for_install:
install_dir = \
tool_dependency_util.get_tool_dependency_install_dir( app=self.app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='set_environment',
tool_dependency_name=env_var_name,
tool_dependency_version=None )
install_environment = InstallEnvironment( app=self.app,
tool_shed_repository_install_dir=tool_shed_repository_install_dir,
install_dir=install_dir )
env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem,
install_environment=install_environment )
if env_var_dict:
if not os.path.exists( install_dir ):
os.makedirs( install_dir )
status = self.app.install_model.ToolDependency.installation_status.INSTALLING
tool_dependency = \
tool_dependency_util.create_or_update_tool_dependency( app=self.app,
tool_shed_repository=tool_shed_repository,
name=env_var_name,
version=None,
type='set_environment',
status=status,
set_status=True )
if env_var_version == '1.0':
# Create this tool dependency's env.sh file.
env_file_builder = EnvFileBuilder( install_dir )
return_code = env_file_builder.append_line( make_executable=True, **env_var_dict )
if return_code:
error_message = 'Error creating env.sh file for tool dependency %s, return_code: %s' % \
( str( tool_dependency.name ), str( return_code ) )
log.debug( error_message )
status = self.app.install_model.ToolDependency.installation_status.ERROR
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency=tool_dependency,
status=status,
error_message=error_message )
else:
if tool_dependency.status not in [ self.app.install_model.ToolDependency.installation_status.ERROR,
self.app.install_model.ToolDependency.installation_status.INSTALLED ]:
status = self.app.install_model.ToolDependency.installation_status.INSTALLED
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency=tool_dependency,
status=status )
log.debug( 'Environment variable %s set in %s for tool dependency %s.' %
( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) )
else:
error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).'
status = self.app.install_model.ToolDependency.installation_status.ERROR
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency=tool_dependency,
status=status,
error_message=error_message )
tool_dependencies.append( tool_dependency )
return tool_dependencies
def create_tool_dependency_with_initialized_env_sh_file( self, dependent_install_dir, tool_shed_repository,
required_repository, package_name, package_version,
tool_dependencies_config ):
"""
Create or get a tool_dependency record that is defined by the received package_name and package_version.
An env.sh file will be created for the tool_dependency in the received dependent_install_dir.
"""
# The received required_repository refers to a tool_shed_repository record that is defined as a complex
# repository dependency for this tool_dependency. The required_repository may or may not be currently
# installed (it doesn't matter). If it is installed, it is associated with a tool_dependency that has
# an env.sh file that this new tool_dependency must be able to locate and "source". If it is not installed,
# we can still determine where that env.sh file will be, so we'll initialize this new tool_dependency's env.sh
# file in either case. If the required repository ends up with an installation error, this new tool
# dependency will still be fine because its containing repository will be defined as missing dependencies.
tool_dependencies = []
if not os.path.exists( dependent_install_dir ):
os.makedirs( dependent_install_dir )
required_tool_dependency_env_file_path = None
if tool_dependencies_config:
required_td_tree, error_message = xml_util.parse_xml( tool_dependencies_config )
if required_td_tree:
required_td_root = required_td_tree.getroot()
for required_td_elem in required_td_root:
# Find the appropriate package name and version.
if required_td_elem.tag == 'package':
# <package name="bwa" version="0.5.9">
required_td_package_name = required_td_elem.get( 'name', None )
required_td_package_version = required_td_elem.get( 'version', None )
# Check the database to see if we have a record for the required tool dependency (we may not which is ok). If we
# find a record, we need to see if it is in an error state and if so handle it appropriately.
required_tool_dependency = \
tool_dependency_util.get_tool_dependency_by_name_version_type_repository( self.app,
required_repository,
required_td_package_name,
required_td_package_version,
'package' )
if required_td_package_name == package_name and required_td_package_version == package_version:
# Get or create a database tool_dependency record with which the installed package on disk will be associated.
tool_dependency = \
tool_dependency_util.create_or_update_tool_dependency( app=self.app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=self.app.install_model.ToolDependency.installation_status.NEVER_INSTALLED,
set_status=True )
# Create an env.sh file for the tool_dependency whose first line will source the env.sh file located in
# the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh
# file currently exists..
required_tool_dependency_env_file_path = \
self.get_required_repository_package_env_sh_path( package_name,
package_version,
required_repository )
env_file_builder = EnvFileBuilder( tool_dependency.installation_directory( self.app ) )
env_file_builder.append_line( action="source", value=required_tool_dependency_env_file_path )
return_code = env_file_builder.return_code
if return_code:
error_message = 'Error defining env.sh file for package %s, return_code: %s' % \
( str( package_name ), str( return_code ) )
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.ERROR,
error_message=error_message)
elif required_tool_dependency is not None and required_tool_dependency.in_error_state:
error_message = "This tool dependency's required tool dependency %s version %s has status %s." % \
( str( required_tool_dependency.name ), str( required_tool_dependency.version ), str( required_tool_dependency.status ) )
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.ERROR,
error_message=error_message)
else:
tool_dependency = \
tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.INSTALLED )
tool_dependencies.append( tool_dependency )
return tool_dependencies
def process_tag_set( self, tool_shed_repository, tool_dependency, package_elem, package_name, package_version,
from_tool_migration_manager=False, tool_dependency_db_records=None ):
# <install version="1.0">
# Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository.
actions_elem_tuples = []
proceed_with_install = False
install_dir = \
tool_dependency_util.get_tool_dependency_install_dir( app=self.app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
if os.path.exists( install_dir ):
# The tool_migration_manager handles tool migration stages and the sync_database_with_file_system()
# method handles two scenarios: (1) where a Galaxy file system environment related to installed
# Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy
# database tables associated with these installed items, and (2) the Tool Shed's install and test
# framework which installs repositories in 2 stages, those of type tool_dependency_definition
# followed by those containing valid tools and tool functional test components. Neither of these
# scenarios apply when the install manager is running.
if from_tool_migration_manager:
proceed_with_install = True
else:
# Notice that we'll throw away the following tool_dependency if it can be installed.
tool_dependency, proceed_with_install = self.sync_database_with_file_system( self.app,
tool_shed_repository,
package_name,
package_version,
install_dir,
tool_dependency_type='package' )
if not proceed_with_install:
log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it." %
( str( tool_dependency.name ), str( tool_dependency.version ) ) )
return tool_dependency, proceed_with_install, actions_elem_tuples
else:
proceed_with_install = True
if proceed_with_install:
package_install_version = package_elem.get( 'version', '1.0' )
status = self.app.install_model.ToolDependency.installation_status.INSTALLING
tool_dependency = \
tool_dependency_util.create_or_update_tool_dependency( app=self.app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=status,
set_status=True )
# Get the information about the current platform in case the tool dependency definition includes tag sets
# for installing compiled binaries.
platform_info_dict = tool_dependency_util.get_platform_info_dict()
if package_install_version == '1.0':
# Handle tool dependency installation using a fabric method included in the Galaxy framework.
actions_elem_tuples = tool_dependency_util.parse_package_elem( package_elem,
platform_info_dict=platform_info_dict,
include_after_install_actions=True )
if not actions_elem_tuples:
proceed_with_install = False
error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) )
error_message += 'the recipe for installing the package is missing either an <actions> tag set or an <actions_group> '
error_message += 'tag set.'
# Since there was an installation error, update the tool dependency status to Error.
# The remove_installation_path option must be left False here.
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(self.app,
tool_dependency=tool_dependency,
status=self.app.install_model.ToolDependency.installation_status.ERROR,
error_message=error_message)
else:
raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
return tool_dependency, proceed_with_install, actions_elem_tuples
