Пример #1
0
def install_and_build_package( app, tool_dependency, actions_dict ):
    """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
    sa_session = app.model.context.current
    install_dir = actions_dict[ 'install_dir' ]
    package_name = actions_dict[ 'package_name' ]
    actions = actions_dict.get( 'actions', None )
    filtered_actions = []
    env_shell_file_paths = []
    if actions:
        with make_tmp_dir() as work_dir:
            with lcd( work_dir ):
                # The first action in the list of actions will be the one that defines the installation process.  There
                # are currently only two supported processes; download_by_url and clone via a "shell_command" action type.
                action_type, action_dict = actions[ 0 ]
                if action_type == 'download_by_url':
                    # Eliminate the download_by_url action so remaining actions can be processed correctly.
                    filtered_actions = actions[ 1: ]
                    url = action_dict[ 'url' ]
                    if 'target_filename' in action_dict:
                        # Sometimes compressed archives extracts their content to a folder other than the default defined file name.  Using this
                        # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
                        downloaded_filename = action_dict[ 'target_filename' ]
                    else:
                        downloaded_filename = os.path.split( url )[ -1 ]
                    dir = common_util.url_download( work_dir, downloaded_filename, url, extract=True )
                elif action_type == 'shell_command':
                    # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action>
                    # Eliminate the shell_command clone action so remaining actions can be processed correctly.
                    filtered_actions = actions[ 1: ]
                    return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] )
                    if return_code:
                        return
                    dir = package_name
                elif action_type == 'download_file':
                    # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
                    # Download a single file to the working directory.
                    filtered_actions = actions[ 1: ]
                    url = action_dict[ 'url' ]
                    if 'target_filename' in action_dict:
                        # Sometimes compressed archives extracts their content to a folder other than the default defined file name.  Using this
                        # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
                        filename = action_dict[ 'target_filename' ]
                    else:
                        filename = url.split( '/' )[ -1 ]
                    common_util.url_download( work_dir, filename, url )
                    dir = os.path.curdir
                else:
                    # We're handling a complex repository dependency where we only have a set_environment tag set.
                    # <action type="set_environment">
                    #    <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
                    # </action>
                    filtered_actions = [ a for a in actions ]
                    dir = install_dir
                # We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local
                # environment and the python environment.  Checking the path as follows should work.
                full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) )
                if not os.path.exists( full_path_to_dir ):
                    os.makedirs( full_path_to_dir )
                # The package has been down-loaded, so we can now perform all of the actions defined for building it.
                for action_tup in filtered_actions:
                    current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
                    with lcd( current_dir ):
                        action_type, action_dict = action_tup
                        if action_type == 'make_directory':
                            common_util.make_directory( full_path=action_dict[ 'full_path' ] )
                        elif action_type == 'move_directory_files':
                            common_util.move_directory_files( current_dir=current_dir,
                                                              source_dir=os.path.join( action_dict[ 'source_directory' ] ),
                                                              destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) )
                        elif action_type == 'move_file':
                            # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found.
                            # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9'
                            common_util.move_file( current_dir=current_dir,
                                                   source=os.path.join( action_dict[ 'source' ] ),
                                                   destination_dir=os.path.join( action_dict[ 'destination' ] ) )
                        elif action_type == 'set_environment':
                            # Currently the only action supported in this category is "environment_variable".
                            # Build a command line from the prior_installation_required, in case an environment variable is referenced
                            # in the set_environment action.
                            cmds = []
                            for env_shell_file_path in env_shell_file_paths:
                                if os.path.exists( env_shell_file_path ):
                                    for env_setting in open( env_shell_file_path ):
                                        cmds.append( env_setting.strip( '\n' ) )
                                else:
                                    log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path )
                            env_var_dicts = action_dict[ 'environment_variable' ]
                            for env_var_dict in env_var_dicts:
                                # Check for the presence of the $ENV[] key string and populate it if possible.
                                env_var_dict = handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, cmds )
                                env_command = common_util.create_or_update_env_shell_file( install_dir, env_var_dict )
                                return_code = handle_command( app, tool_dependency, install_dir, env_command )
                                if return_code:
                                    return
                        elif action_type == 'set_environment_for_install':
                            # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files,
                            # the environment setting in each of which will be injected into the environment for all <action type="shell_command">
                            # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file.
                            env_shell_file_paths = action_dict[ 'env_shell_file_paths' ]
                        elif action_type == 'setup_virtualenv':
                            # TODO: maybe should be configurable
                            venv_src_directory = os.path.abspath( os.path.join( app.config.tool_dependency_dir, '__virtualenv_src' ) )
                            if not install_virtualenv( app, venv_src_directory ):
                                log.error( 'Unable to install virtualenv' )
                                return
                            requirements = action_dict[ 'requirements' ]
                            if os.path.exists( os.path.join( dir, requirements ) ):
                                # requirements specified as path to a file
                                requirements_path = requirements
                            else:
                                # requirements specified directly in XML, create a file with these for pip.
                                requirements_path = os.path.join( install_dir, "requirements.txt" )
                                with open( requirements_path, "w" ) as f:
                                    f.write( requirements )
                            venv_directory = os.path.join( install_dir, "venv" )
                            # TODO: Consider making --no-site-packages optional.
                            setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % (venv_src_directory, venv_directory)
                            # POSIXLY_CORRECT forces shell commands . and source to have the same
                            # and well defined behavior in bash/zsh.
                            activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" )
                            install_command = "python '%s' install -r '%s'" % ( os.path.join( venv_directory, "bin", "pip" ), requirements_path )
                            full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command )
                            return_code = handle_command( app, tool_dependency, install_dir, full_setup_command )
                            if return_code:
                                return
                            site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join( venv_directory, "bin", "python" )
                            output = handle_command( app, tool_dependency, install_dir, site_packages_command, return_output=True )
                            if output.return_code:
                                return
                            if not os.path.exists( output.stdout ):
                                log.error( "virtualenv's site-packages directory '%s' does not exist", output.stdout )
                                return
                            modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PYTHONPATH", action="prepend_to", value=output.stdout ) )
                            return_code = handle_command( app, tool_dependency, install_dir, modify_env_command )
                            if return_code:
                                return
                            modify_env_command = common_util.create_or_update_env_shell_file( install_dir, dict( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) ) )
                            return_code = handle_command( app, tool_dependency, install_dir, modify_env_command )
                            if return_code:
                                return
                        elif action_type == 'shell_command':
                            with settings( warn_only=True ):
                                cmd = ''
                                for env_shell_file_path in env_shell_file_paths:
                                    if os.path.exists( env_shell_file_path ):
                                        for env_setting in open( env_shell_file_path ):
                                            cmd += '%s\n' % env_setting
                                    else:
                                        log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path )
                                cmd += action_dict[ 'command' ]
                                return_code = handle_command( app, tool_dependency, install_dir, cmd )
                                if return_code:
                                    return
                        elif action_type == 'template_command':
                            env_vars = dict()
                            for env_shell_file_path in env_shell_file_paths:
                                if os.path.exists( env_shell_file_path ):
                                    for env_setting in open( env_shell_file_path ):
                                        env_string = env_setting.split( ';' )[ 0 ]
                                        env_name, env_path = env_string.split( '=' )
                                        env_vars[ env_name ] = env_path
                                else:
                                    log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path )
                            env_vars.update( common_util.get_env_var_values( install_dir ) )
                            language = action_dict[ 'language' ]
                            with settings( warn_only=True, **env_vars ):
                                if language == 'cheetah':
                                    # We need to import fabric.api.env so that we can access all collected environment variables.
                                    cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars )
                                    return_code = handle_command( app, tool_dependency, install_dir, cmd )
                                    if return_code:
                                        return
                        elif action_type == 'download_file':
                            # Download a single file to the current working directory.
                            url = action_dict[ 'url' ]
                            if 'target_filename' in action_dict:
                                filename = action_dict[ 'target_filename' ]
                            else:
                                filename = url.split( '/' )[ -1 ]
                            extract = action_dict.get( 'extract', False )
                            common_util.url_download( current_dir, filename, url, extract=extract )
                        elif action_type == 'change_directory':
                            target_directory = os.path.realpath( os.path.normpath( os.path.join( current_dir, action_dict[ 'directory' ] ) ) )
                            if target_directory.startswith( os.path.realpath( current_dir ) ) and os.path.exists( target_directory ):
                                # Change directory to a directory within the current working directory.
                                dir = target_directory
                            elif target_directory.startswith( os.path.realpath( work_dir ) ) and os.path.exists( target_directory ):
                                # Change directory to a directory above the current working directory, but within the defined work_dir.
                                dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' )
                            else:
                                log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory )
Пример #2
0
 def evaluate_template(text):
     """ Substitute variables defined in XML blocks from dependencies file."""
     return Template(text).safe_substitute(
         common_util.get_env_var_values(install_dir))
Пример #3
0
 def evaluate_template( text ):
     """ Substitute variables defined in XML blocks from dependencies file."""
     return Template( text ).safe_substitute( common_util.get_env_var_values( install_dir ) )
Пример #4
0
def install_and_build_package(app, tool_dependency, actions_dict):
    """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
    sa_session = app.model.context.current
    install_dir = actions_dict['install_dir']
    package_name = actions_dict['package_name']
    actions = actions_dict.get('actions', None)
    filtered_actions = []
    env_shell_file_paths = []
    if actions:
        with make_tmp_dir() as work_dir:
            with lcd(work_dir):
                # The first action in the list of actions will be the one that defines the installation process.  There
                # are currently only two supported processes; download_by_url and clone via a "shell_command" action type.
                action_type, action_dict = actions[0]
                if action_type == 'download_by_url':
                    # Eliminate the download_by_url action so remaining actions can be processed correctly.
                    filtered_actions = actions[1:]
                    url = action_dict['url']
                    if 'target_filename' in action_dict:
                        # Sometimes compressed archives extracts their content to a folder other than the default defined file name.  Using this
                        # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
                        downloaded_filename = action_dict['target_filename']
                    else:
                        downloaded_filename = os.path.split(url)[-1]
                    dir = common_util.url_download(work_dir,
                                                   downloaded_filename,
                                                   url,
                                                   extract=True)
                elif action_type == 'shell_command':
                    # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action>
                    # Eliminate the shell_command clone action so remaining actions can be processed correctly.
                    filtered_actions = actions[1:]
                    return_code = handle_command(app, tool_dependency,
                                                 install_dir,
                                                 action_dict['command'])
                    if return_code:
                        return
                    dir = package_name
                elif action_type == 'download_file':
                    # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action>
                    # Download a single file to the working directory.
                    filtered_actions = actions[1:]
                    url = action_dict['url']
                    if 'target_filename' in action_dict:
                        # Sometimes compressed archives extracts their content to a folder other than the default defined file name.  Using this
                        # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
                        filename = action_dict['target_filename']
                    else:
                        filename = url.split('/')[-1]
                    common_util.url_download(work_dir, filename, url)
                    dir = os.path.curdir
                else:
                    # We're handling a complex repository dependency where we only have a set_environment tag set.
                    # <action type="set_environment">
                    #    <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
                    # </action>
                    filtered_actions = [a for a in actions]
                    dir = install_dir
                # We need to be careful in determining if the value of dir is a valid directory because we're dealing with 2 environments, the fabric local
                # environment and the python environment.  Checking the path as follows should work.
                full_path_to_dir = os.path.abspath(os.path.join(work_dir, dir))
                if not os.path.exists(full_path_to_dir):
                    os.makedirs(full_path_to_dir)
                # The package has been down-loaded, so we can now perform all of the actions defined for building it.
                for action_tup in filtered_actions:
                    current_dir = os.path.abspath(os.path.join(work_dir, dir))
                    with lcd(current_dir):
                        action_type, action_dict = action_tup
                        if action_type == 'make_directory':
                            common_util.make_directory(
                                full_path=action_dict['full_path'])
                        elif action_type == 'move_directory_files':
                            common_util.move_directory_files(
                                current_dir=current_dir,
                                source_dir=os.path.join(
                                    action_dict['source_directory']),
                                destination_dir=os.path.join(
                                    action_dict['destination_directory']))
                        elif action_type == 'move_file':
                            # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found.
                            # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9'
                            common_util.move_file(
                                current_dir=current_dir,
                                source=os.path.join(action_dict['source']),
                                destination_dir=os.path.join(
                                    action_dict['destination']))
                        elif action_type == 'set_environment':
                            # Currently the only action supported in this category is "environment_variable".
                            # Build a command line from the prior_installation_required, in case an environment variable is referenced
                            # in the set_environment action.
                            cmds = []
                            for env_shell_file_path in env_shell_file_paths:
                                if os.path.exists(env_shell_file_path):
                                    for env_setting in open(
                                            env_shell_file_path):
                                        cmds.append(env_setting.strip('\n'))
                                else:
                                    log.debug(
                                        'Invalid file %s specified, ignoring set_environment action.',
                                        env_shell_file_path)
                            env_var_dicts = action_dict['environment_variable']
                            for env_var_dict in env_var_dicts:
                                # Check for the presence of the $ENV[] key string and populate it if possible.
                                env_var_dict = handle_environment_variables(
                                    app, tool_dependency, install_dir,
                                    env_var_dict, cmds)
                                env_command = common_util.create_or_update_env_shell_file(
                                    install_dir, env_var_dict)
                                return_code = handle_command(
                                    app, tool_dependency, install_dir,
                                    env_command)
                                if return_code:
                                    return
                        elif action_type == 'set_environment_for_install':
                            # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files,
                            # the environment setting in each of which will be injected into the environment for all <action type="shell_command">
                            # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file.
                            env_shell_file_paths = action_dict[
                                'env_shell_file_paths']
                        elif action_type == 'setup_virtualenv':
                            # TODO: maybe should be configurable
                            venv_src_directory = os.path.abspath(
                                os.path.join(app.config.tool_dependency_dir,
                                             '__virtualenv_src'))
                            if not install_virtualenv(app, venv_src_directory):
                                log.error('Unable to install virtualenv')
                                return
                            requirements = action_dict['requirements']
                            if os.path.exists(os.path.join(dir, requirements)):
                                # requirements specified as path to a file
                                requirements_path = requirements
                            else:
                                # requirements specified directly in XML, create a file with these for pip.
                                requirements_path = os.path.join(
                                    install_dir, "requirements.txt")
                                with open(requirements_path, "w") as f:
                                    f.write(requirements)
                            venv_directory = os.path.join(install_dir, "venv")
                            # TODO: Consider making --no-site-packages optional.
                            setup_command = "python %s/virtualenv.py --no-site-packages '%s'" % (
                                venv_src_directory, venv_directory)
                            # POSIXLY_CORRECT forces shell commands . and source to have the same
                            # and well defined behavior in bash/zsh.
                            activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join(
                                venv_directory, "bin", "activate")
                            install_command = "python '%s' install -r '%s'" % (
                                os.path.join(venv_directory, "bin",
                                             "pip"), requirements_path)
                            full_setup_command = "%s; %s; %s" % (
                                setup_command, activate_command,
                                install_command)
                            return_code = handle_command(
                                app, tool_dependency, install_dir,
                                full_setup_command)
                            if return_code:
                                return
                            site_packages_command = "%s -c 'import os, sys; print os.path.join(sys.prefix, \"lib\", \"python\" + sys.version[:3], \"site-packages\")'" % os.path.join(
                                venv_directory, "bin", "python")
                            output = handle_command(app,
                                                    tool_dependency,
                                                    install_dir,
                                                    site_packages_command,
                                                    return_output=True)
                            if output.return_code:
                                return
                            if not os.path.exists(output.stdout):
                                log.error(
                                    "virtualenv's site-packages directory '%s' does not exist",
                                    output.stdout)
                                return
                            modify_env_command = common_util.create_or_update_env_shell_file(
                                install_dir,
                                dict(name="PYTHONPATH",
                                     action="prepend_to",
                                     value=output.stdout))
                            return_code = handle_command(
                                app, tool_dependency, install_dir,
                                modify_env_command)
                            if return_code:
                                return
                            modify_env_command = common_util.create_or_update_env_shell_file(
                                install_dir,
                                dict(name="PATH",
                                     action="prepend_to",
                                     value=os.path.join(venv_directory,
                                                        "bin")))
                            return_code = handle_command(
                                app, tool_dependency, install_dir,
                                modify_env_command)
                            if return_code:
                                return
                        elif action_type == 'shell_command':
                            with settings(warn_only=True):
                                cmd = ''
                                for env_shell_file_path in env_shell_file_paths:
                                    if os.path.exists(env_shell_file_path):
                                        for env_setting in open(
                                                env_shell_file_path):
                                            cmd += '%s\n' % env_setting
                                    else:
                                        log.debug(
                                            'Invalid file %s specified, ignoring shell_command action.',
                                            env_shell_file_path)
                                cmd += action_dict['command']
                                return_code = handle_command(
                                    app, tool_dependency, install_dir, cmd)
                                if return_code:
                                    return
                        elif action_type == 'template_command':
                            env_vars = dict()
                            for env_shell_file_path in env_shell_file_paths:
                                if os.path.exists(env_shell_file_path):
                                    for env_setting in open(
                                            env_shell_file_path):
                                        env_string = env_setting.split(';')[0]
                                        env_name, env_path = env_string.split(
                                            '=')
                                        env_vars[env_name] = env_path
                                else:
                                    log.debug(
                                        'Invalid file %s specified, ignoring template_command action.',
                                        env_shell_file_path)
                            env_vars.update(
                                common_util.get_env_var_values(install_dir))
                            language = action_dict['language']
                            with settings(warn_only=True, **env_vars):
                                if language == 'cheetah':
                                    # We need to import fabric.api.env so that we can access all collected environment variables.
                                    cmd = fill_template(
                                        '#from fabric.api import env\n%s' %
                                        action_dict['command'],
                                        context=env_vars)
                                    return_code = handle_command(
                                        app, tool_dependency, install_dir, cmd)
                                    if return_code:
                                        return
                        elif action_type == 'download_file':
                            # Download a single file to the current working directory.
                            url = action_dict['url']
                            if 'target_filename' in action_dict:
                                filename = action_dict['target_filename']
                            else:
                                filename = url.split('/')[-1]
                            extract = action_dict.get('extract', False)
                            common_util.url_download(current_dir,
                                                     filename,
                                                     url,
                                                     extract=extract)
                        elif action_type == 'change_directory':
                            target_directory = os.path.realpath(
                                os.path.normpath(
                                    os.path.join(current_dir,
                                                 action_dict['directory'])))
                            if target_directory.startswith(
                                    os.path.realpath(current_dir)
                            ) and os.path.exists(target_directory):
                                # Change directory to a directory within the current working directory.
                                dir = target_directory
                            elif target_directory.startswith(
                                    os.path.realpath(work_dir)
                            ) and os.path.exists(target_directory):
                                # Change directory to a directory above the current working directory, but within the defined work_dir.
                                dir = target_directory.replace(
                                    os.path.realpath(work_dir), '').lstrip('/')
                            else:
                                log.error(
                                    'Invalid or nonexistent directory %s specified, ignoring change_directory action.',
                                    target_directory)