def install_and_build_package( app, tool_dependency, actions_dict ):
"""Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
sa_session = app.model.context.current
install_dir = actions_dict[ 'install_dir' ]
package_name = actions_dict[ 'package_name' ]
actions = actions_dict.get( 'actions', None )
filtered_actions = []
env_shell_file_paths = []
if actions:
with make_tmp_dir() as work_dir:
with lcd( work_dir ):
# The first action in the list of actions will be the one that defines the installation process. There
# are currently only two supported processes; download_by_url and clone via a "shell_command" action type.
action_type, action_dict = actions[ 0 ]
if action_type == 'download_by_url':
# Eliminate the download_by_url action so remaining actions can be processed correctly.
filtered_actions = actions[ 1: ]
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this
# attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
downloaded_filename = action_dict[ 'target_filename' ]
else:
downloaded_filename = os.path.split( url )[ -1 ]
dir = common_util.url_download( work_dir, downloaded_filename, url, extract=True )
elif action_type == 'shell_command':
# <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action>
# Eliminate the shell_command clone action so remaining actions can be processed correctly.
filtered_actions = actions[ 1: ]
return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] )
if return_code:
return
dir = package_name
elif action_type == 'download_file':
# <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
# Download a single file to the working directory.
filtered_actions = actions[ 1: ]
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this
# attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
filename = action_dict[ 'target_filename' ]
else:
filename = url.split( '/' )[ -1 ]
common_util.url_download( work_dir, filename, url )
dir = os.path.curdir
else:
# We're handling a complex repository dependency where we only have a set_environment tag set.
# <action type="set_environment">
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
filtered_actions = [ a for a in actions ]
dir = install_dir
# We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local
# environment and the python environment. Checking the path as follows should work.
full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) )
if not os.path.exists( full_path_to_dir ):
os.makedirs( full_path_to_dir )
# The package has been down-loaded, so we can now perform all of the actions defined for building it.
for action_tup in filtered_actions:
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
action_type, action_dict = action_tup
if action_type == 'make_directory':
common_util.make_directory( full_path=action_dict[ 'full_path' ] )
elif action_type == 'move_directory_files':
common_util.move_directory_files( current_dir=current_dir,
source_dir=os.path.join( action_dict[ 'source_directory' ] ),
destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) )
elif action_type == 'move_file':
# TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found.
# current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9'
common_util.move_file( current_dir=current_dir,
source=os.path.join( action_dict[ 'source' ] ),
destination_dir=os.path.join( action_dict[ 'destination' ] ) )
elif action_type == 'set_environment':
# Currently the only action supported in this category is "environment_variable".
# Build a command line from the prior_installation_required, in case an environment variable is referenced
# in the set_environment action.
cmds = []
for env_shell_file_path in env_shell_file_paths:
if os.path.exists( env_shell_file_path ):
for env_setting in open( env_shell_file_path ):
cmds.append( env_setting.strip( '\n' ) )
else:
log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path )
env_var_dicts = action_dict[ 'environment_variable' ]
for env_var_dict in env_var_dicts:
# Check for the presence of the $ENV[] key string and populate it if possible.
env_var_dict = handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, cmds )
env_command = common_util.create_or_update_env_shell_file( install_dir, env_var_dict )
return_code = handle_command( app, tool_dependency, install_dir, env_command )
if return_code:
return
elif action_type == 'set_environment_for_install':
# Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files,
# the environment setting in each of which will be injected into the environment for all <action type="shell_command">
# tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file.
env_shell_file_paths = action_dict[ 'env_shell_file_paths' ]
elif action_type == 'setup_virtualenv':
# TODO: maybe should be configurable
venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) )
if not install_virtualenv( app, venv_src_directory ):
log.error( 'Unable to install virtualenv' )
return
requirements = action_dict[ 'requirements' ]
if os.path.exists( os.path.join( dir, requirements ) ):
# requirements specified as path to a file
requirements_path = requirements
else:
# requirements specified directly in XML, create a file with these for pip.
requirements_path = os.path.join( install_dir, "requirements.txt" )
with open( requirements_path, "w" ) as f:
f.write( requirements )
venv_directory = os.path.join( install_dir, "venv" )
# TODO: Consider making --no-site-packages optional.
setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % (venv_src_directory, venv_directory)
# POSIXLY_CORRECT forces shell commands . and source to have the same
# and well defined behavior in bash/zsh.
activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" )
install_command = "python '%s' install -r '%s'" % ( os.path.join( venv_directory, "bin", "pip" ), requirements_path )
full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command )
return_code = handle_command( app, tool_dependency, install_dir, full_setup_command )
if return_code:
return
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python" )
output = handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True )
if output.return_code:
return
if not os.path.exists( output.stdout ):
log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout )
return
modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) )
return_code = handle_command( app, tool_dependency, install_dir, modify_env_command )
if return_code:
return
modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) )
return_code = handle_command( app, tool_dependency, install_dir, modify_env_command )
if return_code:
return
elif action_type == 'shell_command':
with settings( warn_only=True ):
cmd = ''
for env_shell_file_path in env_shell_file_paths:
if os.path.exists( env_shell_file_path ):
for env_setting in open( env_shell_file_path ):
cmd += '%s\n' % env_setting
else:
log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path )
cmd += action_dict[ 'command' ]
return_code = handle_command( app, tool_dependency, install_dir, cmd )
if return_code:
return
elif action_type == 'template_command':
env_vars = dict()
for env_shell_file_path in env_shell_file_paths:
if os.path.exists( env_shell_file_path ):
for env_setting in open( env_shell_file_path ):
env_string = env_setting.split( ';' )[ 0 ]
env_name, env_path = env_string.split( '=' )
env_vars[ env_name ] = env_path
else:
log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path )
env_vars.update( common_util.get_env_var_values( install_dir ) )
language = action_dict[ 'language' ]
with settings( warn_only=True, **env_vars ):
if language == 'cheetah':
# We need to import fabric.api.env so that we can access all collected environment variables.
cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars )
return_code = handle_command( app, tool_dependency, install_dir, cmd )
if return_code:
return
elif action_type == 'download_file':
# Download a single file to the current working directory.
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
filename = action_dict[ 'target_filename' ]
else:
filename = url.split( '/' )[ -1 ]
extract = action_dict.get( 'extract', False )
common_util.url_download( current_dir, filename, url, extract=extract )
elif action_type == 'change_directory':
target_directory = os.path.realpath( os.path.normpath( os.path.join( current_dir, action_dict[ 'directory' ] ) ) )
if target_directory.startswith( os.path.realpath( current_dir ) ) and os.path.exists( target_directory ):
# Change directory to a directory within the current working directory.
dir = target_directory
elif target_directory.startswith( os.path.realpath( work_dir ) ) and os.path.exists( target_directory ):
# Change directory to a directory above the current working directory, but within the defined work_dir.
dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' )
else:
log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory )
def evaluate_template(text):
""" Substitute variables defined in XML blocks from dependencies file."""
return Template(text).safe_substitute(
common_util.get_env_var_values(install_dir))
def evaluate_template( text ):
""" Substitute variables defined in XML blocks from dependencies file."""
return Template( text ).safe_substitute( common_util.get_env_var_values( install_dir ) )
def install_and_build_package(app, tool_dependency, actions_dict):
"""Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
sa_session = app.model.context.current
install_dir = actions_dict['install_dir']
package_name = actions_dict['package_name']
actions = actions_dict.get('actions', None)
filtered_actions = []
env_shell_file_paths = []
if actions:
with make_tmp_dir() as work_dir:
with lcd(work_dir):
# The first action in the list of actions will be the one that defines the installation process. There
# are currently only two supported processes; download_by_url and clone via a "shell_command" action type.
action_type, action_dict = actions[0]
if action_type == 'download_by_url':
# Eliminate the download_by_url action so remaining actions can be processed correctly.
filtered_actions = actions[1:]
url = action_dict['url']
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this
# attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
downloaded_filename = action_dict['target_filename']
else:
downloaded_filename = os.path.split(url)[-1]
dir = common_util.url_download(work_dir,
downloaded_filename,
url,
extract=True)
elif action_type == 'shell_command':
# <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action>
# Eliminate the shell_command clone action so remaining actions can be processed correctly.
filtered_actions = actions[1:]
return_code = handle_command(app, tool_dependency,
install_dir,
action_dict['command'])
if return_code:
return
dir = package_name
elif action_type == 'download_file':
# <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
# Download a single file to the working directory.
filtered_actions = actions[1:]
url = action_dict['url']
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this
# attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
filename = action_dict['target_filename']
else:
filename = url.split('/')[-1]
common_util.url_download(work_dir, filename, url)
dir = os.path.curdir
else:
# We're handling a complex repository dependency where we only have a set_environment tag set.
# <action type="set_environment">
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
# </action>
filtered_actions = [a for a in actions]
dir = install_dir
# We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local
# environment and the python environment. Checking the path as follows should work.
full_path_to_dir = os.path.abspath(os.path.join(work_dir, dir))
if not os.path.exists(full_path_to_dir):
os.makedirs(full_path_to_dir)
# The package has been down-loaded, so we can now perform all of the actions defined for building it.
for action_tup in filtered_actions:
current_dir = os.path.abspath(os.path.join(work_dir, dir))
with lcd(current_dir):
action_type, action_dict = action_tup
if action_type == 'make_directory':
common_util.make_directory(
full_path=action_dict['full_path'])
elif action_type == 'move_directory_files':
common_util.move_directory_files(
current_dir=current_dir,
source_dir=os.path.join(
action_dict['source_directory']),
destination_dir=os.path.join(
action_dict['destination_directory']))
elif action_type == 'move_file':
# TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found.
# current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9'
common_util.move_file(
current_dir=current_dir,
source=os.path.join(action_dict['source']),
destination_dir=os.path.join(
action_dict['destination']))
elif action_type == 'set_environment':
# Currently the only action supported in this category is "environment_variable".
# Build a command line from the prior_installation_required, in case an environment variable is referenced
# in the set_environment action.
cmds = []
for env_shell_file_path in env_shell_file_paths:
if os.path.exists(env_shell_file_path):
for env_setting in open(
env_shell_file_path):
cmds.append(env_setting.strip('\n'))
else:
log.debug(
'Invalid file %s specified, ignoring set_environment action.',
env_shell_file_path)
env_var_dicts = action_dict['environment_variable']
for env_var_dict in env_var_dicts:
# Check for the presence of the $ENV[] key string and populate it if possible.
env_var_dict = handle_environment_variables(
app, tool_dependency, install_dir,
env_var_dict, cmds)
env_command = common_util.create_or_update_env_shell_file(
install_dir, env_var_dict)
return_code = handle_command(
app, tool_dependency, install_dir,
env_command)
if return_code:
return
elif action_type == 'set_environment_for_install':
# Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files,
# the environment setting in each of which will be injected into the environment for all <action type="shell_command">
# tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file.
env_shell_file_paths = action_dict[
'env_shell_file_paths']
elif action_type == 'setup_virtualenv':
# TODO: maybe should be configurable
venv_src_directory = os.path.abspath(
os.path.join(app.config.tool_dependency_dir,
'__virtualenv_src'))
if not install_virtualenv(app, venv_src_directory):
log.error('Unable to install virtualenv')
return
requirements = action_dict['requirements']
if os.path.exists(os.path.join(dir, requirements)):
# requirements specified as path to a file
requirements_path = requirements
else:
# requirements specified directly in XML, create a file with these for pip.
requirements_path = os.path.join(
install_dir, "requirements.txt")
with open(requirements_path, "w") as f:
f.write(requirements)
venv_directory = os.path.join(install_dir, "venv")
# TODO: Consider making --no-site-packages optional.
setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % (
venv_src_directory, venv_directory)
# POSIXLY_CORRECT forces shell commands . and source to have the same
# and well defined behavior in bash/zsh.
activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join(
venv_directory, "bin", "activate")
install_command = "python '%s' install -r '%s'" % (
os.path.join(venv_directory, "bin",
"pip"), requirements_path)
full_setup_command = "%s; %s; %s" % (
setup_command, activate_command,
install_command)
return_code = handle_command(
app, tool_dependency, install_dir,
full_setup_command)
if return_code:
return
site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join(
venv_directory, "bin", "python")
output = handle_command(app,
tool_dependency,
install_dir,
site_packages_command,
return_output=True)
if output.return_code:
return
if not os.path.exists(output.stdout):
log.error(
"virtualenv's site-packages directory '%s' does not exist",
output.stdout)
return
modify_env_command = common_util.create_or_update_env_shell_file(
install_dir,
dict(name="PYTHONPATH",
action="prepend_to",
value=output.stdout))
return_code = handle_command(
app, tool_dependency, install_dir,
modify_env_command)
if return_code:
return
modify_env_command = common_util.create_or_update_env_shell_file(
install_dir,
dict(name="PATH",
action="prepend_to",
value=os.path.join(venv_directory,
"bin")))
return_code = handle_command(
app, tool_dependency, install_dir,
modify_env_command)
if return_code:
return
elif action_type == 'shell_command':
with settings(warn_only=True):
cmd = ''
for env_shell_file_path in env_shell_file_paths:
if os.path.exists(env_shell_file_path):
for env_setting in open(
env_shell_file_path):
cmd += '%s\n' % env_setting
else:
log.debug(
'Invalid file %s specified, ignoring shell_command action.',
env_shell_file_path)
cmd += action_dict['command']
return_code = handle_command(
app, tool_dependency, install_dir, cmd)
if return_code:
return
elif action_type == 'template_command':
env_vars = dict()
for env_shell_file_path in env_shell_file_paths:
if os.path.exists(env_shell_file_path):
for env_setting in open(
env_shell_file_path):
env_string = env_setting.split(';')[0]
env_name, env_path = env_string.split(
'=')
env_vars[env_name] = env_path
else:
log.debug(
'Invalid file %s specified, ignoring template_command action.',
env_shell_file_path)
env_vars.update(
common_util.get_env_var_values(install_dir))
language = action_dict['language']
with settings(warn_only=True, **env_vars):
if language == 'cheetah':
# We need to import fabric.api.env so that we can access all collected environment variables.
cmd = fill_template(
'#from fabric.api import env\n%s' %
action_dict['command'],
context=env_vars)
return_code = handle_command(
app, tool_dependency, install_dir, cmd)
if return_code:
return
elif action_type == 'download_file':
# Download a single file to the current working directory.
url = action_dict['url']
if 'target_filename' in action_dict:
filename = action_dict['target_filename']
else:
filename = url.split('/')[-1]
extract = action_dict.get('extract', False)
common_util.url_download(current_dir,
filename,
url,
extract=extract)
elif action_type == 'change_directory':
target_directory = os.path.realpath(
os.path.normpath(
os.path.join(current_dir,
action_dict['directory'])))
if target_directory.startswith(
os.path.realpath(current_dir)
) and os.path.exists(target_directory):
# Change directory to a directory within the current working directory.
dir = target_directory
elif target_directory.startswith(
os.path.realpath(work_dir)
) and os.path.exists(target_directory):
# Change directory to a directory above the current working directory, but within the defined work_dir.
dir = target_directory.replace(
os.path.realpath(work_dir), '').lstrip('/')
else:
log.error(
'Invalid or nonexistent directory %s specified, ignoring change_directory action.',
target_directory)