def set_environment(app, elem, tool_shed_repository): """ Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml file is:: <set_environment version="1.0"> <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> </set_environment> """ # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like # the following. See the implementation of support for this in the tool dependency package type's method above. # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> sa_session = app.model.context.current tool_dependency = None env_var_version = elem.get("version", "1.0") for env_var_elem in elem: # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>). env_var_name = env_var_elem.get("name", None) env_var_action = env_var_elem.get("action", None) if env_var_name and env_var_action: install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dependency_type="set_environment", tool_dependency_name=env_var_name, tool_dependency_version=None, ) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir(app, tool_shed_repository) env_var_dict = common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) if env_var_dict: if not os.path.exists(install_dir): os.makedirs(install_dir) tool_dependency = create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type="set_environment", status=app.model.ToolDependency.installation_status.INSTALLING, set_status=True, ) cmd = common_util.create_or_update_env_shell_file(install_dir, env_var_dict) if env_var_version == "1.0": # Handle setting environment variables using a fabric method. fabric_util.handle_command(app, tool_dependency, install_dir, cmd) sa_session.refresh(tool_dependency) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add(tool_dependency) sa_session.flush() print "Environment variable ", env_var_name, "set in", install_dir
def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current install_dir = actions_dict[ 'install_dir' ] package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] env_shell_file_paths = [] if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): # The first action in the list of actions will be the one that defines the installation process. There # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[ 0 ] if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] if 'target_filename' in action_dict: # Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction. downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] dir = common_util.url_download( work_dir, downloaded_filename, url, extract=True ) elif action_type == 'shell_command': # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> # Eliminate the shell_command clone action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: return dir = package_name elif action_type == 'download_file': # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> # Download a single file to the working directory. filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] if 'target_filename' in action_dict: # Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction. filename = action_dict[ 'target_filename' ] else: filename = url.split( '/' )[ -1 ] common_util.url_download( work_dir, filename, url ) dir = os.path.curdir else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> filtered_actions = [ a for a in actions ] dir = install_dir # We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local # environment and the python environment. Checking the path as follows should work. full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) ) if not os.path.exists( full_path_to_dir ): os.makedirs( full_path_to_dir ) # The package has been down-loaded, so we can now perform all of the actions defined for building it. for action_tup in filtered_actions: current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): action_type, action_dict = action_tup if action_type == 'make_directory': common_util.make_directory( full_path=action_dict[ 'full_path' ] ) elif action_type == 'move_directory_files': common_util.move_directory_files( current_dir=current_dir, source_dir=os.path.join( action_dict[ 'source_directory' ] ), destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) elif action_type == 'move_file': # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found. # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9' common_util.move_file( current_dir=current_dir, source=os.path.join( action_dict[ 'source' ] ), destination_dir=os.path.join( action_dict[ 'destination' ] ) ) elif action_type == 'set_environment': # Currently the only action supported in this category is "environment_variable". # Build a command line from the prior_installation_required, in case an environment variable is referenced # in the set_environment action. cmds = [] for env_shell_file_path in env_shell_file_paths: if os.path.exists( env_shell_file_path ): for env_setting in open( env_shell_file_path ): cmds.append( env_setting.strip( '\n' ) ) else: log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path ) env_var_dicts = action_dict[ 'environment_variable' ] for env_var_dict in env_var_dicts: # Check for the presence of the $ENV[] key string and populate it if possible. env_var_dict = handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, cmds ) env_command = common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) return_code = handle_command( app, tool_dependency, install_dir, env_command ) if return_code: return elif action_type == 'set_environment_for_install': # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] elif action_type == 'setup_virtualenv': # TODO: maybe should be configurable venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) ) if not install_virtualenv( app, venv_src_directory ): log.error( 'Unable to install virtualenv' ) return requirements = action_dict[ 'requirements' ] if os.path.exists( os.path.join( dir, requirements ) ): # requirements specified as path to a file requirements_path = requirements else: # requirements specified directly in XML, create a file with these for pip. requirements_path = os.path.join( install_dir, "requirements.txt" ) with open( requirements_path, "w" ) as f: f.write( requirements ) venv_directory = os.path.join( install_dir, "venv" ) # TODO: Consider making --no-site-packages optional. setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % (venv_src_directory, venv_directory) # POSIXLY_CORRECT forces shell commands . and source to have the same # and well defined behavior in bash/zsh. activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" ) install_command = "python '%s' install -r '%s'" % ( os.path.join( venv_directory, "bin", "pip" ), requirements_path ) full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command ) return_code = handle_command( app, tool_dependency, install_dir, full_setup_command ) if return_code: return site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python" ) output = handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True ) if output.return_code: return if not os.path.exists( output.stdout ): log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout ) return modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) ) return_code = handle_command( app, tool_dependency, install_dir, modify_env_command ) if return_code: return modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) ) return_code = handle_command( app, tool_dependency, install_dir, modify_env_command ) if return_code: return elif action_type == 'shell_command': with settings( warn_only=True ): cmd = '' for env_shell_file_path in env_shell_file_paths: if os.path.exists( env_shell_file_path ): for env_setting in open( env_shell_file_path ): cmd += '%s\n' % env_setting else: log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path ) cmd += action_dict[ 'command' ] return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return elif action_type == 'template_command': env_vars = dict() for env_shell_file_path in env_shell_file_paths: if os.path.exists( env_shell_file_path ): for env_setting in open( env_shell_file_path ): env_string = env_setting.split( ';' )[ 0 ] env_name, env_path = env_string.split( '=' ) env_vars[ env_name ] = env_path else: log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path ) env_vars.update( common_util.get_env_var_values( install_dir ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): if language == 'cheetah': # We need to import fabric.api.env so that we can access all collected environment variables. cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return elif action_type == 'download_file': # Download a single file to the current working directory. url = action_dict[ 'url' ] if 'target_filename' in action_dict: filename = action_dict[ 'target_filename' ] else: filename = url.split( '/' )[ -1 ] extract = action_dict.get( 'extract', False ) common_util.url_download( current_dir, filename, url, extract=extract ) elif action_type == 'change_directory': target_directory = os.path.realpath( os.path.normpath( os.path.join( current_dir, action_dict[ 'directory' ] ) ) ) if target_directory.startswith( os.path.realpath( current_dir ) ) and os.path.exists( target_directory ): # Change directory to a directory within the current working directory. dir = target_directory elif target_directory.startswith( os.path.realpath( work_dir ) ) and os.path.exists( target_directory ): # Change directory to a directory above the current working directory, but within the defined work_dir. dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' ) else: log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory )
def set_environment(app, elem, tool_shed_repository): """ Create a ToolDependency to set an environment variable. This is different from the process used to set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml file is:: <set_environment version="1.0"> <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> </set_environment> """ # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like # the following. See the implementation of support for this in the tool dependency package type's method above. # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> sa_session = app.model.context.current tool_dependency = None env_var_version = elem.get('version', '1.0') for env_var_elem in elem: # The value of env_var_name must match the text value of at least 1 <requirement> tag in the tool config's <requirements> tag set whose # "type" attribute is "set_environment" (e.g., <requirement type="set_environment">R_SCRIPT_PATH</requirement>). env_var_name = env_var_elem.get('name', None) env_var_action = env_var_elem.get('action', None) if env_var_name and env_var_action: install_dir = get_tool_dependency_install_dir( app=app, repository_name=tool_shed_repository.name, repository_owner=tool_shed_repository.owner, repository_changeset_revision=tool_shed_repository. installed_changeset_revision, tool_dependency_type='set_environment', tool_dependency_name=env_var_name, tool_dependency_version=None) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository) env_var_dict = common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir= tool_shed_repository_install_dir) if env_var_dict: if not os.path.exists(install_dir): os.makedirs(install_dir) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type='set_environment', status=app.model.ToolDependency.installation_status. INSTALLING, set_status=True) cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict) if env_var_version == '1.0': # Handle setting environment variables using a fabric method. fabric_util.handle_command(app, tool_dependency, install_dir, cmd) sa_session.refresh(tool_dependency) if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add(tool_dependency) sa_session.flush() print 'Environment variable ', env_var_name, 'set in', install_dir else: raise NotImplementedError( 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' )
def install_and_build_package( app, tool_dependency, actions_dict ): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current install_dir = actions_dict[ 'install_dir' ] package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): # The first action in the list of actions will be the one that defines the installation process. There # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[ 0 ] if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] if 'target_filename' in action_dict: downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] downloaded_file_path = common_util.url_download( work_dir, downloaded_filename, url ) if common_util.istar( downloaded_file_path ): # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action> common_util.extract_tar( downloaded_file_path, work_dir ) dir = common_util.tar_extraction_directory( work_dir, downloaded_filename ) elif common_util.iszip( downloaded_file_path ): # <action type="download_by_url">http://downloads.sourceforge.net/project/picard/picard-tools/1.56/picard-tools-1.56.zip</action> zip_archive_extracted = common_util.extract_zip( downloaded_file_path, work_dir ) dir = common_util.zip_extraction_directory( work_dir, downloaded_filename ) else: dir = work_dir elif action_type == 'shell_command': # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> # Eliminate the shell_command clone action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: return dir = package_name else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> filtered_actions = [ a for a in actions ] dir = install_dir if not os.path.exists( dir ): os.makedirs( dir ) # The package has been down-loaded, so we can now perform all of the actions defined for building it. with lcd( dir ): for action_tup in filtered_actions: action_type, action_dict = action_tup current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) if action_type == 'make_directory': common_util.make_directory( full_path=action_dict[ 'full_path' ] ) elif action_type == 'move_directory_files': common_util.move_directory_files( current_dir=current_dir, source_dir=os.path.join( action_dict[ 'source_directory' ] ), destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) elif action_type == 'move_file': # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found. # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9' common_util.move_file( current_dir=current_dir, source=os.path.join( action_dict[ 'source' ] ), destination_dir=os.path.join( action_dict[ 'destination' ] ) ) elif action_type == 'set_environment': # Currently the only action supported in this category is "environment_variable". env_var_dicts = action_dict[ 'environment_variable' ] for env_var_dict in env_var_dicts: cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return elif action_type == 'shell_command': with settings( warn_only=True ): return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: return
def install_and_build_package(app, tool_dependency, actions_dict): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current install_dir = actions_dict['install_dir'] package_name = actions_dict['package_name'] actions = actions_dict.get('actions', None) filtered_actions = [] env_shell_file_paths = [] if actions: with make_tmp_dir() as work_dir: with lcd(work_dir): # The first action in the list of actions will be the one that defines the installation process. There # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[0] if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[1:] url = action_dict['url'] if 'target_filename' in action_dict: # Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction. downloaded_filename = action_dict['target_filename'] else: downloaded_filename = os.path.split(url)[-1] dir = common_util.url_download(work_dir, downloaded_filename, url, extract=True) elif action_type == 'shell_command': # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> # Eliminate the shell_command clone action so remaining actions can be processed correctly. filtered_actions = actions[1:] return_code = handle_command(app, tool_dependency, install_dir, action_dict['command']) if return_code: return dir = package_name elif action_type == 'download_file': # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> # Download a single file to the working directory. filtered_actions = actions[1:] url = action_dict['url'] if 'target_filename' in action_dict: # Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction. filename = action_dict['target_filename'] else: filename = url.split('/')[-1] common_util.url_download(work_dir, filename, url) dir = os.path.curdir else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> filtered_actions = [a for a in actions] dir = install_dir # We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local # environment and the python environment. Checking the path as follows should work. full_path_to_dir = os.path.abspath(os.path.join(work_dir, dir)) if not os.path.exists(full_path_to_dir): os.makedirs(full_path_to_dir) # The package has been down-loaded, so we can now perform all of the actions defined for building it. for action_tup in filtered_actions: current_dir = os.path.abspath(os.path.join(work_dir, dir)) with lcd(current_dir): action_type, action_dict = action_tup if action_type == 'make_directory': common_util.make_directory( full_path=action_dict['full_path']) elif action_type == 'move_directory_files': common_util.move_directory_files( current_dir=current_dir, source_dir=os.path.join( action_dict['source_directory']), destination_dir=os.path.join( action_dict['destination_directory'])) elif action_type == 'move_file': # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found. # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9' common_util.move_file( current_dir=current_dir, source=os.path.join(action_dict['source']), destination_dir=os.path.join( action_dict['destination'])) elif action_type == 'set_environment': # Currently the only action supported in this category is "environment_variable". # Build a command line from the prior_installation_required, in case an environment variable is referenced # in the set_environment action. cmds = [] for env_shell_file_path in env_shell_file_paths: if os.path.exists(env_shell_file_path): for env_setting in open( env_shell_file_path): cmds.append(env_setting.strip('\n')) else: log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path) env_var_dicts = action_dict['environment_variable'] for env_var_dict in env_var_dicts: # Check for the presence of the $ENV[] key string and populate it if possible. env_var_dict = handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, cmds) env_command = common_util.create_or_update_env_shell_file( install_dir, env_var_dict) return_code = handle_command( app, tool_dependency, install_dir, env_command) if return_code: return elif action_type == 'set_environment_for_install': # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. env_shell_file_paths = action_dict[ 'env_shell_file_paths'] elif action_type == 'setup_virtualenv': # TODO: maybe should be configurable venv_src_directory = os.path.abspath( os.path.join(app.config.tool_dependency_dir, '__virtualenv_src')) if not install_virtualenv(app, venv_src_directory): log.error('Unable to install virtualenv') return requirements = action_dict['requirements'] if os.path.exists(os.path.join(dir, requirements)): # requirements specified as path to a file requirements_path = requirements else: # requirements specified directly in XML, create a file with these for pip. requirements_path = os.path.join( install_dir, "requirements.txt") with open(requirements_path, "w") as f: f.write(requirements) venv_directory = os.path.join(install_dir, "venv") # TODO: Consider making --no-site-packages optional. setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % ( venv_src_directory, venv_directory) # POSIXLY_CORRECT forces shell commands . and source to have the same # and well defined behavior in bash/zsh. activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate") install_command = "python '%s' install -r '%s'" % ( os.path.join(venv_directory, "bin", "pip"), requirements_path) full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command) return_code = handle_command( app, tool_dependency, install_dir, full_setup_command) if return_code: return site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python") output = handle_command(app, tool_dependency, install_dir, site_packages_command, return_output=True) if output.return_code: return if not os.path.exists(output.stdout): log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout) return modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict(name="PYTHONPATH", action="prepend_to", value=output.stdout)) return_code = handle_command( app, tool_dependency, install_dir, modify_env_command) if return_code: return modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict(name="PATH", action="prepend_to", value=os.path.join(venv_directory, "bin"))) return_code = handle_command( app, tool_dependency, install_dir, modify_env_command) if return_code: return elif action_type == 'shell_command': with settings(warn_only=True): cmd = '' for env_shell_file_path in env_shell_file_paths: if os.path.exists(env_shell_file_path): for env_setting in open( env_shell_file_path): cmd += '%s\n' % env_setting else: log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path) cmd += action_dict['command'] return_code = handle_command( app, tool_dependency, install_dir, cmd) if return_code: return elif action_type == 'template_command': env_vars = dict() for env_shell_file_path in env_shell_file_paths: if os.path.exists(env_shell_file_path): for env_setting in open( env_shell_file_path): env_string = env_setting.split(';')[0] env_name, env_path = env_string.split( '=') env_vars[env_name] = env_path else: log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path) env_vars.update( common_util.get_env_var_values(install_dir)) language = action_dict['language'] with settings(warn_only=True, **env_vars): if language == 'cheetah': # We need to import fabric.api.env so that we can access all collected environment variables. cmd = fill_template( '#from fabric.api import env\n%s' % action_dict['command'], context=env_vars) return_code = handle_command( app, tool_dependency, install_dir, cmd) if return_code: return elif action_type == 'download_file': # Download a single file to the current working directory. url = action_dict['url'] if 'target_filename' in action_dict: filename = action_dict['target_filename'] else: filename = url.split('/')[-1] extract = action_dict.get('extract', False) common_util.url_download(current_dir, filename, url, extract=extract) elif action_type == 'change_directory': target_directory = os.path.realpath( os.path.normpath( os.path.join(current_dir, action_dict['directory']))) if target_directory.startswith( os.path.realpath(current_dir) ) and os.path.exists(target_directory): # Change directory to a directory within the current working directory. dir = target_directory elif target_directory.startswith( os.path.realpath(work_dir) ) and os.path.exists(target_directory): # Change directory to a directory above the current working directory, but within the defined work_dir. dir = target_directory.replace( os.path.realpath(work_dir), '').lstrip('/') else: log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory)
def install_and_build_package(app, tool_dependency, actions_dict): """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" sa_session = app.model.context.current install_dir = actions_dict['install_dir'] package_name = actions_dict['package_name'] actions = actions_dict.get('actions', None) filtered_actions = [] if actions: with make_tmp_dir() as work_dir: with lcd(work_dir): # The first action in the list of actions will be the one that defines the installation process. There # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[0] if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[1:] url = action_dict['url'] if 'target_filename' in action_dict: downloaded_filename = action_dict['target_filename'] else: downloaded_filename = os.path.split(url)[-1] downloaded_file_path = common_util.url_download( work_dir, downloaded_filename, url) if common_util.istar(downloaded_file_path): # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action> common_util.extract_tar(downloaded_file_path, work_dir) dir = common_util.tar_extraction_directory( work_dir, downloaded_filename) elif common_util.iszip(downloaded_file_path): # <action type="download_by_url">http://downloads.sourceforge.net/project/picard/picard-tools/1.56/picard-tools-1.56.zip</action> zip_archive_extracted = common_util.extract_zip( downloaded_file_path, work_dir) dir = common_util.zip_extraction_directory( work_dir, downloaded_filename) else: dir = work_dir elif action_type == 'shell_command': # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> # Eliminate the shell_command clone action so remaining actions can be processed correctly. filtered_actions = actions[1:] return_code = handle_command(app, tool_dependency, install_dir, action_dict['command']) if return_code: return dir = package_name else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> filtered_actions = [a for a in actions] dir = install_dir if not os.path.exists(dir): os.makedirs(dir) # The package has been down-loaded, so we can now perform all of the actions defined for building it. with lcd(dir): for action_tup in filtered_actions: action_type, action_dict = action_tup current_dir = os.path.abspath( os.path.join(work_dir, dir)) if action_type == 'make_directory': common_util.make_directory( full_path=action_dict['full_path']) elif action_type == 'move_directory_files': common_util.move_directory_files( current_dir=current_dir, source_dir=os.path.join( action_dict['source_directory']), destination_dir=os.path.join( action_dict['destination_directory'])) elif action_type == 'move_file': # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found. # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9' common_util.move_file( current_dir=current_dir, source=os.path.join(action_dict['source']), destination_dir=os.path.join( action_dict['destination'])) elif action_type == 'set_environment': # Currently the only action supported in this category is "environment_variable". env_var_dicts = action_dict['environment_variable'] for env_var_dict in env_var_dicts: cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict) return_code = handle_command( app, tool_dependency, install_dir, cmd) if return_code: return elif action_type == 'shell_command': with settings(warn_only=True): return_code = handle_command( app, tool_dependency, install_dir, action_dict['command']) if return_code: return