def set_environment(app, elem, tool_shed_repository):
"""
Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated
with a package. An example entry in a tool_dependencies.xml file is::
<set_environment version="1.0">
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
</set_environment>
"""
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
# the following. See the implementation of support for this in the tool dependency package type's method above.
# <set_environment version="1.0">
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
# </set_environment>
sa_session = app.model.context.current
tool_dependency = None
env_var_version = elem.get("version", "1.0")
for env_var_elem in elem:
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose
# "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
env_var_name = env_var_elem.get("name", None)
env_var_action = env_var_elem.get("action", None)
if env_var_name and env_var_action:
install_dir = get_tool_dependency_install_dir(
app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dependency_type="set_environment",
tool_dependency_name=env_var_name,
tool_dependency_version=None,
)
tool_shed_repository_install_dir = get_tool_shed_repository_install_dir(app, tool_shed_repository)
env_var_dict = common_util.create_env_var_dict(
env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir
)
if env_var_dict:
if not os.path.exists(install_dir):
os.makedirs(install_dir)
tool_dependency = create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=env_var_name,
version=None,
type="set_environment",
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True,
)
cmd = common_util.create_or_update_env_shell_file(install_dir, env_var_dict)
if env_var_version == "1.0":
# Handle setting environment variables using a fabric method.
fabric_util.handle_command(app, tool_dependency, install_dir, cmd)
sa_session.refresh(tool_dependency)
if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
sa_session.add(tool_dependency)
sa_session.flush()
print "Environment variable ", env_var_name, "set in", install_dir
def handle_set_environment_entry_for_package(app, install_dir,
tool_shed_repository,
package_name, package_version,
elem):
action_dict = {}
actions = []
for package_elem in elem:
if package_elem.tag == 'install':
# Create the tool_dependency record in the database.
tool_dependency = create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True)
# Get the installation method version from a tag like: <install version="1.0">
package_install_version = package_elem.get('version', '1.0')
if package_install_version == '1.0':
# Since the required tool dependency is installed for a repository dependency, all we need to do
# is inspect the <actions> tag set to find the <action type="set_environment"> tag.
for actions_elem in package_elem:
for action_elem in actions_elem:
action_type = action_elem.get('type', 'shell_command')
if action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
env_var_dicts = []
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict(
env_elem,
tool_dependency_install_dir=install_dir
)
if env_var_dict:
env_var_dicts.append(env_var_dict)
if env_var_dicts:
action_dict[env_elem.tag] = env_var_dicts
actions.append((action_type, action_dict))
return tool_dependency, actions
return None, actions
def handle_set_environment_entry_for_package(
app, install_dir, tool_shed_repository, package_name, package_version, elem
):
action_dict = {}
actions = []
for package_elem in elem:
if package_elem.tag == "install":
# Create the tool_dependency record in the database.
tool_dependency = create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type="package",
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True,
)
# Get the installation method version from a tag like: <install version="1.0">
package_install_version = package_elem.get("version", "1.0")
if package_install_version == "1.0":
# Since the required tool dependency is installed for a repository dependency, all we need to do
# is inspect the <actions> tag set to find the <action type="set_environment"> tag.
for actions_elem in package_elem:
for action_elem in actions_elem:
action_type = action_elem.get("type", "shell_command")
if action_type == "set_environment":
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
env_var_dicts = []
for env_elem in action_elem:
if env_elem.tag == "environment_variable":
env_var_dict = common_util.create_env_var_dict(
env_elem, tool_dependency_install_dir=install_dir
)
if env_var_dict:
env_var_dicts.append(env_var_dict)
if env_var_dicts:
action_dict[env_elem.tag] = env_var_dicts
actions.append((action_type, action_dict))
return tool_dependency, actions
return None, actions
def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd ):
"""Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
sa_session = app.model.context.current
def evaluate_template( text ):
""" Substitute variables defined in XML blocks from dependencies file."""
return Template( text ).safe_substitute( common_util.get_env_var_values( install_dir ) )
if not os.path.exists( install_dir ):
os.makedirs( install_dir )
actions_dict = dict( install_dir=install_dir )
if package_name:
actions_dict[ 'package_name' ] = package_name
actions = []
all_env_shell_file_paths = []
env_var_dicts = []
# Make sure to skip all comments, since they are now included in the XML tree.
for action_elem in actions_elem.findall( 'action' ):
action_dict = {}
action_type = action_elem.get( 'type', 'shell_command' )
if action_type == 'shell_command':
# <action type="shell_command">make</action>
action_elem_text = evaluate_template( action_elem.text )
if action_elem_text:
action_dict[ 'command' ] = action_elem_text
else:
continue
elif action_type == 'template_command':
# Default to Cheetah as it's the first template language supported.
language = action_elem.get( 'language', 'cheetah' ).lower()
if language == 'cheetah':
# Cheetah template syntax.
# <action type="template_command" language="cheetah">
# #if env.PATH:
# make
# #end if
# </action>
action_elem_text = action_elem.text.strip()
if action_elem_text:
action_dict[ 'language' ] = language
action_dict[ 'command' ] = action_elem_text
else:
continue
else:
log.debug( "Unsupported template language '%s'. Not proceeding." % str( language ) )
raise Exception( "Unsupported template language '%s' in tool dependency definition." % str( language ) )
elif action_type == 'download_by_url':
# <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
if action_elem.text:
action_dict[ 'url' ] = action_elem.text
target_filename = action_elem.get( 'target_filename', None )
if target_filename:
action_dict[ 'target_filename' ] = target_filename
else:
continue
elif action_type == 'download_file':
# <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
if action_elem.text:
action_dict[ 'url' ] = action_elem.text
target_filename = action_elem.get( 'target_filename', None )
if target_filename:
action_dict[ 'target_filename' ] = target_filename
action_dict[ 'extract' ] = asbool( action_elem.get( 'extract', False ) )
else:
continue
elif action_type == 'make_directory':
# <action type="make_directory">$INSTALL_DIR/lib/python</action>
if action_elem.text:
action_dict[ 'full_path' ] = evaluate_template( action_elem.text )
else:
continue
elif action_type == 'change_directory':
# <action type="change_directory">PHYLIP-3.6b</action>
if action_elem.text:
action_dict[ 'directory' ] = action_elem.text
else:
continue
elif action_type in [ 'move_directory_files', 'move_file' ]:
# <action type="move_file">
# <source>misc/some_file</source>
# <destination>$INSTALL_DIR/bin</destination>
# </action>
# <action type="move_directory_files">
# <source_directory>bin</source_directory>
# <destination_directory>$INSTALL_DIR/bin</destination_directory>
# </action>
for move_elem in action_elem:
move_elem_text = evaluate_template( move_elem.text )
if move_elem_text:
action_dict[ move_elem.tag ] = move_elem_text
elif action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir )
if env_var_dict:
env_var_dicts.append( env_var_dict )
if env_var_dicts:
# The last child of an <action type="set_environment"> might be a comment, so manually set it to be 'environment_variable'.
action_dict[ 'environment_variable' ] = env_var_dicts
else:
continue
elif action_type == 'set_environment_for_install':
# <action type="set_environment_for_install">
# <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
# <package name="numpy" version="1.7.1" />
# </repository>
# </action>
# This action type allows for defining an environment that will properly compile a tool dependency. Currently, tag set definitions like
# that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be
# supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current
# tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example.
for env_elem in action_elem:
if env_elem.tag == 'repository':
env_shell_file_paths = common_util.get_env_shell_file_paths( app, env_elem )
if env_shell_file_paths:
all_env_shell_file_paths.extend( env_shell_file_paths )
if all_env_shell_file_paths:
action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths
else:
continue
elif action_type == 'setup_virtualenv':
# <action type="setup_virtualenv" />
## Install requirements from file requirements.txt of downloaded bundle - or -
# <action type="setup_virtualenv">tools/requirements.txt</action>
## Install requirements from specified file from downloaded bundle -or -
# <action type="setup_virtualenv">pyyaml==3.2.0
# lxml==2.3.0</action>
## Manually specify contents of requirements.txt file to create dynamically.
action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' )
else:
log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) )
raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) )
action_tuple = ( action_type, action_dict )
if action_type == 'set_environment':
if action_tuple not in actions:
actions.append( action_tuple )
else:
actions.append( action_tuple )
if actions:
actions_dict[ 'actions' ] = actions
if proprietary_fabfile_path:
# TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
# run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
raise Exception( 'Tool dependency installation using proprietary fabric scripts is not yet supported.' )
else:
install_and_build_package_via_fabric( app, tool_dependency, actions_dict )
def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem, required_repository ):
"""
Populate a list of actions for creating an env.sh file for a dependent repository. The received elem is the <package> tag set associated
with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency.
"""
action_dict = {}
actions = []
for package_elem in elem:
if package_elem.tag == 'install':
# Create the tool_dependency record in the database.
tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True )
# Get the installation method version from a tag like: <install version="1.0">
package_install_version = package_elem.get( 'version', '1.0' )
if package_install_version == '1.0':
# Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find
# the <action type="set_environment"> tag.
env_var_dicts = []
for actions_elem in package_elem:
for action_elem in actions_elem:
action_type = action_elem.get( 'type', 'shell_command' )
if action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir )
if env_var_dict:
if env_var_dict not in env_var_dicts:
env_var_dicts.append( env_var_dict )
elif action_type == 'setup_virtualenv':
# Add the virtualenv's site-packages to PYTHONPATH and bin to PATH. This is a bit hackish.
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( install_dir, "venv", "bin", "python" )
output = fabric_util.handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True )
if output.return_code:
log.error( 'Dependency includes a setup_virtualenv action but venv python is broken:', output.stderr )
elif not os.path.exists( output.stdout ):
log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout )
else:
env_var_dicts.append( dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) )
env_var_dicts.append( dict( name="PATH", action="prepend_to", value=os.path.join( install_dir, 'venv', 'bin' ) ) )
if env_var_dicts:
if required_repository.status in [ app.model.ToolShedRepository.installation_status.INSTALLED,
app.model.ToolShedRepository.installation_status.DEACTIVATED ]:
# Handle the case where we have an installed required repository due to the prior_installation_required = True
# setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's
# tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this,
# we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh
# file generated for the installed required repository. Each env_var_dict currently looks something like this:
# {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'}
# We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict
# with the associated value in the env.sh file.
new_env_var_dicts = []
env_sh_file_dir = get_tool_dependency_install_dir( app=app,
repository_name=required_repository.name,
repository_owner=required_repository.owner,
repository_changeset_revision=required_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' )
if os.path.exists( env_sh_file_path ):
for i, line in enumerate( open( env_sh_file_path, 'r' ) ):
env_var_dict = env_var_dicts[ i ]
action = env_var_dict.get( 'action', None )
name = env_var_dict.get( 'name', None )
value = env_var_dict.get( 'value', None )
if action and name and value:
new_value = parse_env_shell_entry( action, name, value, line )
env_var_dict[ 'value' ] = new_value
new_env_var_dicts.append( env_var_dict )
else:
log.debug( 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path )
action_dict[ 'environment_variable' ] = new_env_var_dicts
else:
action_dict[ 'environment_variable' ] = env_var_dicts
actions.append( ( 'set_environment', action_dict ) )
else:
raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
return tool_dependency, actions
return None, actions
def set_environment(app, elem, tool_shed_repository):
"""
Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated
with a package. An example entry in a tool_dependencies.xml file is::
<set_environment version="1.0">
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
</set_environment>
"""
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
# the following. See the implementation of support for this in the tool dependency package type's method above.
# <set_environment version="1.0">
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
# </set_environment>
sa_session = app.model.context.current
tool_dependency = None
env_var_version = elem.get('version', '1.0')
for env_var_elem in elem:
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose
# "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
env_var_name = env_var_elem.get('name', None)
env_var_action = env_var_elem.get('action', None)
if env_var_name and env_var_action:
install_dir = get_tool_dependency_install_dir(
app=app,
repository_name=tool_shed_repository.name,
repository_owner=tool_shed_repository.owner,
repository_changeset_revision=tool_shed_repository.
installed_changeset_revision,
tool_dependency_type='set_environment',
tool_dependency_name=env_var_name,
tool_dependency_version=None)
tool_shed_repository_install_dir = get_tool_shed_repository_install_dir(
app, tool_shed_repository)
env_var_dict = common_util.create_env_var_dict(
env_var_elem,
tool_shed_repository_install_dir=
tool_shed_repository_install_dir)
if env_var_dict:
if not os.path.exists(install_dir):
os.makedirs(install_dir)
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=env_var_name,
version=None,
type='set_environment',
status=app.model.ToolDependency.installation_status.
INSTALLING,
set_status=True)
cmd = common_util.create_or_update_env_shell_file(
install_dir, env_var_dict)
if env_var_version == '1.0':
# Handle setting environment variables using a fabric method.
fabric_util.handle_command(app, tool_dependency,
install_dir, cmd)
sa_session.refresh(tool_dependency)
if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED
sa_session.add(tool_dependency)
sa_session.flush()
print 'Environment variable ', env_var_name, 'set in', install_dir
else:
raise NotImplementedError(
'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).'
)
def install_via_fabric(app,
tool_dependency,
actions_elem,
install_dir,
package_name=None,
proprietary_fabfile_path=None,
**kwd):
"""Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
sa_session = app.model.context.current
def evaluate_template(text):
""" Substitute variables defined in XML blocks from dependencies file."""
return Template(text).safe_substitute(
common_util.get_env_var_values(install_dir))
if not os.path.exists(install_dir):
os.makedirs(install_dir)
actions_dict = dict(install_dir=install_dir)
if package_name:
actions_dict['package_name'] = package_name
actions = []
all_env_shell_file_paths = []
env_var_dicts = []
# Make sure to skip all comments, since they are now included in the XML tree.
for action_elem in actions_elem.findall('action'):
action_dict = {}
action_type = action_elem.get('type', 'shell_command')
if action_type == 'shell_command':
# <action type="shell_command">make</action>
action_elem_text = evaluate_template(action_elem.text)
if action_elem_text:
action_dict['command'] = action_elem_text
else:
continue
elif action_type == 'template_command':
# Default to Cheetah as it's the first template language supported.
language = action_elem.get('language', 'cheetah').lower()
if language == 'cheetah':
# Cheetah template syntax.
# <action type="template_command" language="cheetah">
# #if env.PATH:
# make
# #end if
# </action>
action_elem_text = action_elem.text.strip()
if action_elem_text:
action_dict['language'] = language
action_dict['command'] = action_elem_text
else:
continue
else:
log.debug(
"Unsupported template language '%s'. Not proceeding." %
str(language))
raise Exception(
"Unsupported template language '%s' in tool dependency definition."
% str(language))
elif action_type == 'download_by_url':
# <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
if action_elem.text:
action_dict['url'] = action_elem.text
target_filename = action_elem.get('target_filename', None)
if target_filename:
action_dict['target_filename'] = target_filename
else:
continue
elif action_type == 'download_file':
# <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
if action_elem.text:
action_dict['url'] = action_elem.text
target_filename = action_elem.get('target_filename', None)
if target_filename:
action_dict['target_filename'] = target_filename
action_dict['extract'] = asbool(
action_elem.get('extract', False))
else:
continue
elif action_type == 'make_directory':
# <action type="make_directory">$INSTALL_DIR/lib/python</action>
if action_elem.text:
action_dict['full_path'] = evaluate_template(action_elem.text)
else:
continue
elif action_type == 'change_directory':
# <action type="change_directory">PHYLIP-3.6b</action>
if action_elem.text:
action_dict['directory'] = action_elem.text
else:
continue
elif action_type in ['move_directory_files', 'move_file']:
# <action type="move_file">
# <source>misc/some_file</source>
# <destination>$INSTALL_DIR/bin</destination>
# </action>
# <action type="move_directory_files">
# <source_directory>bin</source_directory>
# <destination_directory>$INSTALL_DIR/bin</destination_directory>
# </action>
for move_elem in action_elem:
move_elem_text = evaluate_template(move_elem.text)
if move_elem_text:
action_dict[move_elem.tag] = move_elem_text
elif action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict(
env_elem, tool_dependency_install_dir=install_dir)
if env_var_dict:
env_var_dicts.append(env_var_dict)
if env_var_dicts:
# The last child of an <action type="set_environment"> might be a comment, so manually set it to be 'environment_variable'.
action_dict['environment_variable'] = env_var_dicts
else:
continue
elif action_type == 'set_environment_for_install':
# <action type="set_environment_for_install">
# <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
# <package name="numpy" version="1.7.1" />
# </repository>
# </action>
# This action type allows for defining an environment that will properly compile a tool dependency. Currently, tag set definitions like
# that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be
# supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current
# tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example.
for env_elem in action_elem:
if env_elem.tag == 'repository':
env_shell_file_paths = common_util.get_env_shell_file_paths(
app, env_elem)
if env_shell_file_paths:
all_env_shell_file_paths.extend(env_shell_file_paths)
if all_env_shell_file_paths:
action_dict['env_shell_file_paths'] = all_env_shell_file_paths
else:
continue
elif action_type == 'setup_virtualenv':
# <action type="setup_virtualenv" />
## Install requirements from file requirements.txt of downloaded bundle - or -
# <action type="setup_virtualenv">tools/requirements.txt</action>
## Install requirements from specified file from downloaded bundle -or -
# <action type="setup_virtualenv">pyyaml==3.2.0
# lxml==2.3.0</action>
## Manually specify contents of requirements.txt file to create dynamically.
action_dict['requirements'] = evaluate_template(
action_elem.text or 'requirements.txt')
else:
log.debug("Unsupported action type '%s'. Not proceeding." %
str(action_type))
raise Exception(
"Unsupported action type '%s' in tool dependency definition." %
str(action_type))
action_tuple = (action_type, action_dict)
if action_type == 'set_environment':
if action_tuple not in actions:
actions.append(action_tuple)
else:
actions.append(action_tuple)
if actions:
actions_dict['actions'] = actions
if proprietary_fabfile_path:
# TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
# run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
raise Exception(
'Tool dependency installation using proprietary fabric scripts is not yet supported.'
)
else:
install_and_build_package_via_fabric(app, tool_dependency,
actions_dict)
def handle_set_environment_entry_for_package(app, install_dir,
tool_shed_repository,
package_name, package_version,
elem, required_repository):
"""
Populate a list of actions for creating an env.sh file for a dependent repository. The received elem is the <package> tag set associated
with the tool-dependencies.xml file for one of the received tool_shed_repository's repository dependency.
"""
action_dict = {}
actions = []
for package_elem in elem:
if package_elem.tag == 'install':
# Create the tool_dependency record in the database.
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
app=app,
tool_shed_repository=tool_shed_repository,
name=package_name,
version=package_version,
type='package',
status=app.model.ToolDependency.installation_status.INSTALLING,
set_status=True)
# Get the installation method version from a tag like: <install version="1.0">
package_install_version = package_elem.get('version', '1.0')
if package_install_version == '1.0':
# Since the required tool dependency is installed for a repository dependency, we first need to inspect the <actions> tag set to find
# the <action type="set_environment"> tag.
env_var_dicts = []
for actions_elem in package_elem:
for action_elem in actions_elem:
action_type = action_elem.get('type', 'shell_command')
if action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict(
env_elem,
tool_dependency_install_dir=install_dir
)
if env_var_dict:
if env_var_dict not in env_var_dicts:
env_var_dicts.append(env_var_dict)
elif action_type == 'setup_virtualenv':
# Add the virtualenv's site-packages to PYTHONPATH and bin to PATH. This is a bit hackish.
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join(
install_dir, "venv", "bin", "python")
output = fabric_util.handle_command(
app,
tool_dependency,
install_dir,
site_packages_command,
return_output=True)
if output.return_code:
log.error(
'Dependency includes a setup_virtualenv action but venv python is broken:',
output.stderr)
elif not os.path.exists(output.stdout):
log.error(
"virtualenv's site-packages directory '%s' does not exist",
output.stdout)
else:
env_var_dicts.append(
dict(name="PYTHONPATH",
action="prepend_to",
value=output.stdout))
env_var_dicts.append(
dict(name="PATH",
action="prepend_to",
value=os.path.join(
install_dir, 'venv', 'bin')))
if env_var_dicts:
if required_repository.status in [
app.model.ToolShedRepository.installation_status.
INSTALLED, app.model.ToolShedRepository.
installation_status.DEACTIVATED
]:
# Handle the case where we have an installed required repository due to the prior_installation_required = True
# setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's
# tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this,
# we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh
# file generated for the installed required repository. Each env_var_dict currently looks something like this:
# {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'}
# We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict
# with the associated value in the env.sh file.
new_env_var_dicts = []
env_sh_file_dir = get_tool_dependency_install_dir(
app=app,
repository_name=required_repository.name,
repository_owner=required_repository.owner,
repository_changeset_revision=required_repository.
installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version)
env_sh_file_path = os.path.join(
env_sh_file_dir, 'env.sh')
if os.path.exists(env_sh_file_path):
for i, line in enumerate(
open(env_sh_file_path, 'r')):
env_var_dict = env_var_dicts[i]
action = env_var_dict.get('action', None)
name = env_var_dict.get('name', None)
value = env_var_dict.get('value', None)
if action and name and value:
new_value = parse_env_shell_entry(
action, name, value, line)
env_var_dict['value'] = new_value
new_env_var_dicts.append(env_var_dict)
else:
log.debug(
'Invalid file %s specified, ignoring set_environment_for_install action.',
env_sh_file_path)
action_dict['environment_variable'] = new_env_var_dicts
else:
action_dict['environment_variable'] = env_var_dicts
actions.append(('set_environment', action_dict))
else:
raise NotImplementedError(
'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).'
)
return tool_dependency, actions
return None, actions
def install_via_fabric(app,
tool_dependency,
actions_elem,
install_dir,
package_name=None,
proprietary_fabfile_path=None,
**kwd):
"""Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
sa_session = app.model.context.current
if not os.path.exists(install_dir):
os.makedirs(install_dir)
actions_dict = dict(install_dir=install_dir)
if package_name:
actions_dict['package_name'] = package_name
actions = []
for action_elem in actions_elem:
action_dict = {}
action_type = action_elem.get('type', 'shell_command')
if action_type == 'shell_command':
# <action type="shell_command">make</action>
action_elem_text = action_elem.text.replace(
'$INSTALL_DIR', install_dir)
if action_elem_text:
action_dict['command'] = action_elem_text
else:
continue
elif action_type == 'download_by_url':
# <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
if action_elem.text:
action_dict['url'] = action_elem.text
if 'target_filename' in action_elem.attrib:
action_dict['target_filename'] = action_elem.attrib[
'target_filename']
else:
continue
elif action_type == 'make_directory':
# <action type="make_directory">$INSTALL_DIR/lib/python</action>
if action_elem.text:
action_dict['full_path'] = action_elem.text.replace(
'$INSTALL_DIR', install_dir)
else:
continue
elif action_type in ['move_directory_files', 'move_file']:
# <action type="move_file">
# <source>misc/some_file</source>
# <destination>$INSTALL_DIR/bin</destination>
# </action>
# <action type="move_directory_files">
# <source_directory>bin</source_directory>
# <destination_directory>$INSTALL_DIR/bin</destination_directory>
# </action>
for move_elem in action_elem:
move_elem_text = move_elem.text.replace(
'$INSTALL_DIR', install_dir)
if move_elem_text:
action_dict[move_elem.tag] = move_elem_text
elif action_type == 'set_environment':
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
env_var_dicts = []
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = common_util.create_env_var_dict(
env_elem, tool_dependency_install_dir=install_dir)
if env_var_dict:
env_var_dicts.append(env_var_dict)
if env_var_dicts:
action_dict[env_elem.tag] = env_var_dicts
else:
continue
else:
continue
actions.append((action_type, action_dict))
if actions:
actions_dict['actions'] = actions
if proprietary_fabfile_path:
# TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
# run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
raise Exception(
'Tool dependency installation using proprietary fabric scripts is not yet supported.'
)
else:
install_and_build_package_via_fabric(app, tool_dependency,
actions_dict)
def install_via_fabric(
app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd
):
"""Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
sa_session = app.model.context.current
if not os.path.exists(install_dir):
os.makedirs(install_dir)
actions_dict = dict(install_dir=install_dir)
if package_name:
actions_dict["package_name"] = package_name
actions = []
for action_elem in actions_elem:
action_dict = {}
action_type = action_elem.get("type", "shell_command")
if action_type == "shell_command":
# <action type="shell_command">make</action>
action_elem_text = action_elem.text.replace("$INSTALL_DIR", install_dir)
if action_elem_text:
action_dict["command"] = action_elem_text
else:
continue
elif action_type == "download_by_url":
# <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
if action_elem.text:
action_dict["url"] = action_elem.text
if "target_filename" in action_elem.attrib:
action_dict["target_filename"] = action_elem.attrib["target_filename"]
else:
continue
elif action_type == "make_directory":
# <action type="make_directory">$INSTALL_DIR/lib/python</action>
if action_elem.text:
action_dict["full_path"] = action_elem.text.replace("$INSTALL_DIR", install_dir)
else:
continue
elif action_type in ["move_directory_files", "move_file"]:
# <action type="move_file">
# <source>misc/some_file</source>
# <destination>$INSTALL_DIR/bin</destination>
# </action>
# <action type="move_directory_files">
# <source_directory>bin</source_directory>
# <destination_directory>$INSTALL_DIR/bin</destination_directory>
# </action>
for move_elem in action_elem:
move_elem_text = move_elem.text.replace("$INSTALL_DIR", install_dir)
if move_elem_text:
action_dict[move_elem.tag] = move_elem_text
elif action_type == "set_environment":
# <action type="set_environment">
# <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable>
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
env_var_dicts = []
for env_elem in action_elem:
if env_elem.tag == "environment_variable":
env_var_dict = common_util.create_env_var_dict(env_elem, tool_dependency_install_dir=install_dir)
if env_var_dict:
env_var_dicts.append(env_var_dict)
if env_var_dicts:
action_dict[env_elem.tag] = env_var_dicts
else:
continue
else:
continue
actions.append((action_type, action_dict))
if actions:
actions_dict["actions"] = actions
if proprietary_fabfile_path:
# TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
# run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name )
raise Exception("Tool dependency installation using proprietary fabric scripts is not yet supported.")
else:
install_and_build_package_via_fabric(app, tool_dependency, actions_dict)
